diff --git a/frontend/sync-client/src/sync-operations/sync-event-queue.test.ts b/frontend/sync-client/src/sync-operations/sync-event-queue.test.ts index 3e644057..ee9967a0 100644 --- a/frontend/sync-client/src/sync-operations/sync-event-queue.test.ts +++ b/frontend/sync-client/src/sync-operations/sync-event-queue.test.ts @@ -410,7 +410,7 @@ describe("SyncEventQueue", () => { assert.strictEqual(queue.syncedDocumentCount, 2); assert.strictEqual(queue.getSettledDocumentByPath("a.md")?.documentId, "A"); assert.strictEqual(queue.getSettledDocumentByPath("b.md")?.documentId, "B"); - assert.strictEqual(queue.lastSeenUpdateId, 5); + assert.strictEqual(queue._lastSeenUpdateId, 5); }); it("trackedPaths combines documents and pending events", () => { diff --git a/frontend/sync-client/src/sync-operations/sync-event-queue.ts b/frontend/sync-client/src/sync-operations/sync-event-queue.ts index 72ed56fe..5fb9c8c6 100644 --- a/frontend/sync-client/src/sync-operations/sync-event-queue.ts +++ b/frontend/sync-client/src/sync-operations/sync-event-queue.ts @@ -14,6 +14,7 @@ import { type SyncEvent, type VaultUpdateId, } from "./types"; +import { MinCovered } from "../utils/data-structures/min-covered"; export class SyncEventQueue { @@ -39,9 +40,7 @@ export class SyncEventQueue { // file creations for paths matching any of these patterns will be ignored private ignorePatterns: RegExp[]; - - - public readonly lastSeenUpdateId: VaultUpdateId; + public _lastSeenUpdateId: MinCovered; public constructor( private readonly settings: Settings, @@ -71,9 +70,17 @@ export class SyncEventQueue { this.documents.set(relativePath, record); } } - this.lastSeenUpdateId = initialState.lastSeenUpdateId ?? -1; + this._lastSeenUpdateId = new MinCovered(initialState.lastSeenUpdateId ?? 0); - this.logger.debug(`Loaded ${this.documents.size} documents and lastSeenUpdateId=${this.lastSeenUpdateId} from storage`); + this.logger.debug(`Loaded ${this.documents.size} documents and lastSeenUpdateId=${this._lastSeenUpdateId} from storage`); + } + + public get lastSeenUpdateId(): VaultUpdateId { + return this._lastSeenUpdateId.min; + } + + public set lastSeenUpdateId(id: VaultUpdateId) { + this._lastSeenUpdateId.add(id); } public get pendingUpdateCount(): number { @@ -214,7 +221,7 @@ export class SyncEventQueue { ...record }) ), - lastSeenUpdateId: this.lastSeenUpdateId + lastSeenUpdateId: this._lastSeenUpdateId }); } @@ -261,7 +268,7 @@ export class SyncEventQueue { public async clearAllState(): Promise { this.clearPending(); this.documents.clear(); - this.lastSeenUpdateId = -1; + this._lastSeenUpdateId.reset() await this.save(); } diff --git a/frontend/sync-client/src/sync-operations/syncer.ts b/frontend/sync-client/src/sync-operations/syncer.ts index 77144462..a44c6efb 100644 --- a/frontend/sync-client/src/sync-operations/syncer.ts +++ b/frontend/sync-client/src/sync-operations/syncer.ts @@ -378,6 +378,7 @@ export class Syncer { createEvent: event }); + this.history.addHistoryEntry({ status: SyncStatus.SUCCESS, details: { type: SyncType.CREATE, relativePath: effectivePath }, @@ -403,6 +404,8 @@ export class Syncer { }); await this.queue.removeDocument(doc.path); + this.queue.lastSeenUpdateId = response.vaultUpdateId; + this.history.addHistoryEntry({ status: SyncStatus.SUCCESS, @@ -443,6 +446,9 @@ export class Syncer { } ); + this.queue.lastSeenUpdateId = response.vaultUpdateId; + + await this.handleMaybeMergingResponse({ path: diskPath, response, @@ -530,6 +536,8 @@ export class Syncer { remoteHash }); } + + this.queue.lastSeenUpdateId = response.vaultUpdateId; } @@ -549,7 +557,13 @@ export class Syncer { return this.processRemoteDelete(documentWithPath.path, remoteVersion); } - + if (documentWithPath?.record.parentVersionId ?? 0 >= remoteVersion.vaultUpdateId) { + this.queue.lastSeenUpdateId = remoteVersion.vaultUpdateId; + this.logger.debug( + `Document ${remoteVersion.relativePath} is already up-to-date or has newer local changes; skipping remote update` + ); + return; + } if (documentWithPath !== undefined) { // must be the update to an existing doc @@ -570,6 +584,9 @@ export class Syncer { await this.operations.delete(path); await this.queue.removeDocument(path); + this.queue.lastSeenUpdateId = remoteVersion.vaultUpdateId; + + this.history.addHistoryEntry({ status: SyncStatus.SUCCESS, details: { @@ -599,12 +616,30 @@ export class Syncer { const currentContent = await this.operations.read(path); const remoteContent = await this.syncService.getDocumentVersionContent({ documentId: remoteVersion.documentId, vaultUpdateId: remoteVersion.vaultUpdateId }); this.operations.write(path, currentContent, remoteContent); - // todo: update last seen id + + await this.updateCache( + remoteVersion.vaultUpdateId, + remoteContent, + path + ); + this.queue.lastSeenUpdateId = remoteVersion.vaultUpdateId; } // else we don't need to update the content, a subsequent local update will do that + this.syncRemotelyUpdatedFile({ // schedule it so that the lastSeenUpdateId remains consistent + document: + remoteVersion + }) + + + // wait for a local edit to do the actual updating here, so we can't even update the lastSeenUpdateId here this.ensurePath(path, remoteVersion.relativePath); + this.queue.setDocument(remoteVersion.relativePath, { + ...record, + remoteRelativePath: remoteVersion.relativePath + }); + this.history.addHistoryEntry({ status: SyncStatus.SUCCESS, details: { @@ -642,6 +677,8 @@ export class Syncer { remoteRelativePath: remoteVersion.relativePath }); + this.queue.lastSeenUpdateId = remoteVersion.vaultUpdateId; + this.history.addHistoryEntry({ status: SyncStatus.SUCCESS, details: { @@ -682,12 +719,6 @@ export class Syncer { if (canMergeText) { const currentContent = await this.operations.read(pendingCreateEvent.path); - this.queue.resolveCreate(pendingCreateEvent, { - documentId: remoteVersion.documentId, - parentVersionId: remoteVersion.vaultUpdateId, - remoteHash, - remoteRelativePath: path - }); const merged = reconcile("", new TextDecoder().decode(currentContent), new TextDecoder().decode(remoteContent)).text; @@ -698,6 +729,14 @@ export class Syncer { path ); + await this.queue.resolveCreate(pendingCreateEvent, { + documentId: remoteVersion.documentId, + parentVersionId: remoteVersion.vaultUpdateId, + remoteHash, + remoteRelativePath: path + }); + this.queue.lastSeenUpdateId = remoteVersion.vaultUpdateId; + this.history.addHistoryEntry({ status: SyncStatus.SUCCESS, details: { @@ -719,6 +758,8 @@ export class Syncer { remoteHash, remoteRelativePath: path }); + this.queue.lastSeenUpdateId = remoteVersion.vaultUpdateId; + this.history.addHistoryEntry({ status: SyncStatus.SUCCESS, details: { diff --git a/frontend/sync-client/src/utils/data-structures/min-covered.test.ts b/frontend/sync-client/src/utils/data-structures/min-covered.test.ts new file mode 100644 index 00000000..8ebc94a4 --- /dev/null +++ b/frontend/sync-client/src/utils/data-structures/min-covered.test.ts @@ -0,0 +1,76 @@ +import { describe, it } from "node:test"; +import assert from "node:assert"; +import { MinCovered } from "./min-covered"; + +describe("MinCovered", () => { + it("should initialize with the given min value", () => { + const covered = new MinCovered(5); + assert.strictEqual(covered.min, 5); + }); + + it("should add values greater than min", () => { + const covered = new MinCovered(0); + covered.add(3); + assert.strictEqual(covered.min, 0); + covered.add(1); + assert.strictEqual(covered.min, 1); + covered.add(4); + assert.strictEqual(covered.min, 1); + covered.add(2); + assert.strictEqual(covered.min, 4); + }); + + it("should ignore duplicate values", () => { + const covered = new MinCovered(0); + covered.add(3); + covered.add(3); + covered.add(3); + assert.strictEqual(covered.min, 0); + covered.add(1); + covered.add(2); + assert.strictEqual(covered.min, 3); + }); + + it("should handle multiple consecutive values", () => { + const covered = new MinCovered(132); + for (let i = 250; i > 132; i--) { + assert.strictEqual(covered.min, 132); + covered.add(i); + } + assert.strictEqual(covered.min, 250); + }); + + it("should handle adding values lower than current min", () => { + const covered = new MinCovered(5); + covered.add(3); + assert.strictEqual(covered.min, 5); + covered.add(6); + assert.strictEqual(covered.min, 6); + }); + + it("should auto-advance when setting min value", () => { + const covered = new MinCovered(5); + covered.add(7); + covered.add(8); + covered.add(9); + assert.strictEqual(covered.min, 5); + // Setting min to 6 should auto-advance through 7, 8, 9 + covered.min = 6; + assert.strictEqual(covered.min, 9); + covered.add(10); + assert.strictEqual(covered.min, 10); + }); + + it("should handle setting min value with no consecutive values", () => { + const covered = new MinCovered(5); + covered.add(10); + covered.add(15); + assert.strictEqual(covered.min, 5); + // Setting min to 8 should not auto-advance (no consecutive values) + covered.min = 8; + assert.strictEqual(covered.min, 8); + // Add 9 to trigger auto-advance to 10 + covered.add(9); + assert.strictEqual(covered.min, 10); + }); +}); diff --git a/frontend/sync-client/src/utils/data-structures/min-covered.ts b/frontend/sync-client/src/utils/data-structures/min-covered.ts new file mode 100644 index 00000000..720e20a3 --- /dev/null +++ b/frontend/sync-client/src/utils/data-structures/min-covered.ts @@ -0,0 +1,66 @@ +/** + * A class that tracks the minimum covered value in a sequence of numbers. + * It keeps track of a minimum value based on the seen values. + * + * It expects integers slightly out of order and makes sure that the value of `min` is + * always the minimum of the seen values. This is done with bounded memory usage. + * + * @example + * ```typescript + * const covered = new MinCovered(0); + * covered.add(2); // seenValues = [2], min = 0 + * covered.add(1); // seenValues = [], min = 2 + * covered.min; // returns 2 + * ``` + */ +export class MinCovered { + private seenValues: number[] = []; + + public constructor(private minValue: number) { } + + public get min(): number { + return this.minValue; + } + + public set min(value: number) { + this.minValue = Math.max(value, this.minValue); + this.seenValues = this.seenValues.filter((v) => v > this.minValue); + this.advanceMinWhilePossible(); + } + + public add(value: number | undefined): void { + if (value === undefined || value < this.minValue) { + return; + } + + let i = 0; + while (i < this.seenValues.length && this.seenValues[i] < value) { + i++; + } + + if (i === this.seenValues.length) { + this.seenValues.push(value); + } else if (this.seenValues[i] === value) { + return; + } else { + this.seenValues.splice(i, 0, value); + } + + this.advanceMinWhilePossible(); + } + + public reset(minValue?: number): void { + this.minValue = minValue ?? 0; + this.seenValues = []; + } + + private advanceMinWhilePossible(): void { + while ( + this.seenValues.length > 0 && + this.seenValues[0] === this.minValue + 1 + ) { + this.seenValues.shift(); + this.minValue++; + } + } +}