wip better queue
This commit is contained in:
parent
d034ad5cb3
commit
1a4e39d57a
3 changed files with 414 additions and 219 deletions
|
|
@ -30,48 +30,47 @@ function fakeRemoteVersion(
|
|||
}
|
||||
|
||||
describe("SyncEventQueue", () => {
|
||||
it("sync-local followed by delete for the same document returns only the delete", () => {
|
||||
it("sync-local followed by delete for the same document returns only the delete", async () => {
|
||||
const queue = createQueue();
|
||||
queue.setDocument("a.md", {
|
||||
documentId: "A",
|
||||
parentVersionId: 1,
|
||||
hash: "hash-a"
|
||||
remoteHash: "hash-a"
|
||||
});
|
||||
|
||||
queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A" });
|
||||
queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A" });
|
||||
queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A", path: "a.md", originalPath: "a.md" });
|
||||
queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A", path: "a.md", originalPath: "a.md" });
|
||||
queue.enqueue({
|
||||
type: SyncEventType.Delete,
|
||||
documentId: "A",
|
||||
path: "a.md",
|
||||
});
|
||||
|
||||
const event = queue.next();
|
||||
const event = await queue.next();
|
||||
assert.strictEqual(event?.type, SyncEventType.Delete);
|
||||
if (event?.type === SyncEventType.Delete) {
|
||||
assert.strictEqual(event.documentId, "A");
|
||||
}
|
||||
assert.strictEqual(queue.next(), undefined);
|
||||
assert.strictEqual(await queue.next(), undefined);
|
||||
});
|
||||
|
||||
it("sync-local events for the same document coalesce to one", () => {
|
||||
it("sync-local events for the same document coalesce to one", async () => {
|
||||
const queue = createQueue();
|
||||
queue.setDocument("a.md", {
|
||||
documentId: "A",
|
||||
parentVersionId: 1,
|
||||
hash: "hash-a"
|
||||
remoteHash: "hash-a"
|
||||
});
|
||||
|
||||
queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A" });
|
||||
queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A" });
|
||||
queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A" });
|
||||
queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A", path: "a.md", originalPath: "a.md" });
|
||||
queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A", path: "a.md", originalPath: "a.md" });
|
||||
queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A", path: "a.md", originalPath: "a.md" });
|
||||
|
||||
const event = queue.next();
|
||||
const event = await queue.next();
|
||||
assert.strictEqual(event?.type, SyncEventType.SyncLocal);
|
||||
assert.strictEqual(queue.next(), undefined);
|
||||
assert.strictEqual(await queue.next(), undefined);
|
||||
});
|
||||
|
||||
it("sync-remote events for the same documentId coalesce to the last one", () => {
|
||||
it("sync-remote events for the same documentId coalesce to the last one", async () => {
|
||||
const queue = createQueue();
|
||||
|
||||
queue.enqueue({
|
||||
|
|
@ -87,116 +86,63 @@ describe("SyncEventQueue", () => {
|
|||
remoteVersion: fakeRemoteVersion("A", { vaultUpdateId: 3 })
|
||||
});
|
||||
|
||||
const event = queue.next();
|
||||
const event = await queue.next();
|
||||
assert.strictEqual(event?.type, SyncEventType.SyncRemote);
|
||||
if (event?.type === SyncEventType.SyncRemote) {
|
||||
assert.strictEqual(event.remoteVersion.vaultUpdateId, 3);
|
||||
}
|
||||
assert.strictEqual(queue.next(), undefined);
|
||||
assert.strictEqual(await queue.next(), undefined);
|
||||
});
|
||||
|
||||
it("create events are returned FIFO", () => {
|
||||
it("create events are returned FIFO", async () => {
|
||||
const queue = createQueue();
|
||||
queue.enqueue({ type: SyncEventType.Create, path: "a.md" });
|
||||
queue.enqueue({ type: SyncEventType.Create, path: "b.md" });
|
||||
queue.enqueue({ type: SyncEventType.Create, path: "a.md", originalPath: "a.md" });
|
||||
queue.enqueue({ type: SyncEventType.Create, path: "b.md", originalPath: "b.md" });
|
||||
|
||||
const first = queue.next();
|
||||
const first = await queue.next();
|
||||
assert.strictEqual(first?.type, SyncEventType.Create);
|
||||
if (first?.type === SyncEventType.Create) {
|
||||
assert.strictEqual(first.path, "a.md");
|
||||
}
|
||||
|
||||
const second = queue.next();
|
||||
const second = await queue.next();
|
||||
assert.strictEqual(second?.type, SyncEventType.Create);
|
||||
if (second?.type === SyncEventType.Create) {
|
||||
assert.strictEqual(second.path, "b.md");
|
||||
}
|
||||
});
|
||||
|
||||
it("duplicate creates for the same path are skipped", () => {
|
||||
const queue = createQueue();
|
||||
queue.enqueue({ type: SyncEventType.Create, path: "a.md" });
|
||||
queue.enqueue({ type: SyncEventType.Create, path: "a.md" });
|
||||
assert.strictEqual(queue.size, 1);
|
||||
});
|
||||
|
||||
it("create is skipped if the path already has a tracked document", () => {
|
||||
const queue = createQueue();
|
||||
queue.setDocument("a.md", {
|
||||
documentId: "A",
|
||||
parentVersionId: 1,
|
||||
hash: "hash-a"
|
||||
});
|
||||
|
||||
queue.enqueue({ type: SyncEventType.Create, path: "a.md" });
|
||||
assert.strictEqual(queue.size, 0);
|
||||
});
|
||||
|
||||
it("delete uses the provided documentId", () => {
|
||||
it("delete uses the provided documentId", async () => {
|
||||
const queue = createQueue();
|
||||
|
||||
queue.enqueue({
|
||||
type: SyncEventType.Delete,
|
||||
documentId: "A",
|
||||
path: "a.md",
|
||||
});
|
||||
|
||||
const event = queue.next();
|
||||
const event = await queue.next();
|
||||
assert.strictEqual(event?.type, SyncEventType.Delete);
|
||||
if (event?.type === SyncEventType.Delete) {
|
||||
assert.strictEqual(event.documentId, "A");
|
||||
}
|
||||
});
|
||||
|
||||
it("updateCreatePath updates the path of a create event in the queue", () => {
|
||||
const queue = createQueue();
|
||||
queue.enqueue({ type: SyncEventType.Create, path: "old.md" });
|
||||
|
||||
const updated = queue.updateCreatePath("old.md", "new.md");
|
||||
assert.strictEqual(updated, true);
|
||||
assert.strictEqual(queue.hasCreateEvent("old.md"), false);
|
||||
assert.strictEqual(queue.hasCreateEvent("new.md"), true);
|
||||
|
||||
const event = queue.next();
|
||||
assert.strictEqual(event?.type, SyncEventType.Create);
|
||||
if (event?.type === SyncEventType.Create) {
|
||||
assert.strictEqual(event.path, "new.md");
|
||||
}
|
||||
});
|
||||
|
||||
it("updateCreatePath returns false when no create event exists", () => {
|
||||
const queue = createQueue();
|
||||
const updated = queue.updateCreatePath("old.md", "new.md");
|
||||
assert.strictEqual(updated, false);
|
||||
});
|
||||
|
||||
it("hasCreateEvent detects pending creates", () => {
|
||||
const queue = createQueue();
|
||||
assert.strictEqual(queue.hasCreateEvent("a.md"), false);
|
||||
|
||||
queue.enqueue({ type: SyncEventType.Create, path: "a.md" });
|
||||
assert.strictEqual(queue.hasCreateEvent("a.md"), true);
|
||||
|
||||
queue.next();
|
||||
assert.strictEqual(queue.hasCreateEvent("a.md"), false);
|
||||
});
|
||||
|
||||
it("document store CRUD operations work correctly", () => {
|
||||
const queue = createQueue();
|
||||
|
||||
assert.strictEqual(queue.getDocument("a.md"), undefined);
|
||||
assert.strictEqual(queue.getSettledDocumentByPath("a.md"), undefined);
|
||||
assert.strictEqual(queue.documentCount, 0);
|
||||
|
||||
queue.setDocument("a.md", {
|
||||
documentId: "A",
|
||||
parentVersionId: 1,
|
||||
hash: "hash-a"
|
||||
remoteHash: "hash-a"
|
||||
});
|
||||
assert.strictEqual(queue.documentCount, 1);
|
||||
assert.deepStrictEqual(queue.getDocument("a.md"), {
|
||||
assert.deepStrictEqual(queue.getSettledDocumentByPath("a.md"), {
|
||||
documentId: "A",
|
||||
parentVersionId: 1,
|
||||
hash: "hash-a"
|
||||
remoteHash: "hash-a"
|
||||
});
|
||||
|
||||
const found = queue.getDocumentByDocumentId("A");
|
||||
|
|
@ -205,7 +151,7 @@ describe("SyncEventQueue", () => {
|
|||
|
||||
queue.removeDocument("a.md");
|
||||
assert.strictEqual(queue.documentCount, 0);
|
||||
assert.strictEqual(queue.getDocument("a.md"), undefined);
|
||||
assert.strictEqual(queue.getSettledDocumentByPath("a.md"), undefined);
|
||||
});
|
||||
|
||||
it("moveDocument moves a document and returns displaced documentId", () => {
|
||||
|
|
@ -213,18 +159,18 @@ describe("SyncEventQueue", () => {
|
|||
queue.setDocument("a.md", {
|
||||
documentId: "A",
|
||||
parentVersionId: 1,
|
||||
hash: "hash-a"
|
||||
remoteHash: "hash-a"
|
||||
});
|
||||
queue.setDocument("b.md", {
|
||||
documentId: "B",
|
||||
parentVersionId: 2,
|
||||
hash: "hash-b"
|
||||
remoteHash: "hash-b"
|
||||
});
|
||||
|
||||
const displacedId = queue.moveDocument("a.md", "b.md");
|
||||
assert.strictEqual(displacedId, "B");
|
||||
assert.strictEqual(queue.getDocument("a.md"), undefined);
|
||||
assert.strictEqual(queue.getDocument("b.md")?.documentId, "A");
|
||||
assert.strictEqual(queue.getSettledDocumentByPath("a.md"), undefined);
|
||||
assert.strictEqual(queue.getSettledDocumentByPath("b.md")?.documentId, "A");
|
||||
assert.strictEqual(queue.documentCount, 1);
|
||||
});
|
||||
|
||||
|
|
@ -233,140 +179,193 @@ describe("SyncEventQueue", () => {
|
|||
queue.setDocument("a.md", {
|
||||
documentId: "A",
|
||||
parentVersionId: 1,
|
||||
hash: "hash-a"
|
||||
remoteHash: "hash-a"
|
||||
});
|
||||
|
||||
const displacedId = queue.moveDocument("a.md", "b.md");
|
||||
assert.strictEqual(displacedId, undefined);
|
||||
assert.strictEqual(queue.getDocument("b.md")?.documentId, "A");
|
||||
assert.strictEqual(queue.getSettledDocumentByPath("b.md")?.documentId, "A");
|
||||
});
|
||||
|
||||
it("interleaved events for different documents are not confused", () => {
|
||||
it("interleaved events for different documents are not confused", async () => {
|
||||
const queue = createQueue();
|
||||
queue.setDocument("a.md", {
|
||||
documentId: "A",
|
||||
parentVersionId: 1,
|
||||
hash: "hash-a"
|
||||
remoteHash: "hash-a"
|
||||
});
|
||||
queue.setDocument("b.md", {
|
||||
documentId: "B",
|
||||
parentVersionId: 2,
|
||||
hash: "hash-b"
|
||||
remoteHash: "hash-b"
|
||||
});
|
||||
|
||||
queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A" });
|
||||
queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "B" });
|
||||
queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A", path: "a.md", originalPath: "a.md" });
|
||||
queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "B", path: "b.md", originalPath: "b.md" });
|
||||
queue.enqueue({
|
||||
type: SyncEventType.Delete,
|
||||
documentId: "A",
|
||||
path: "a.md",
|
||||
});
|
||||
queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "B" });
|
||||
queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "B", path: "b.md", originalPath: "b.md" });
|
||||
|
||||
// First next() should see the delete for A (coalescing sync-local + delete)
|
||||
const first = queue.next();
|
||||
const first = await queue.next();
|
||||
assert.strictEqual(first?.type, SyncEventType.Delete);
|
||||
if (first?.type === SyncEventType.Delete) {
|
||||
assert.strictEqual(first.documentId, "A");
|
||||
}
|
||||
|
||||
// Remaining should be the coalesced sync-local for B
|
||||
const second = queue.next();
|
||||
const second = await queue.next();
|
||||
assert.strictEqual(second?.type, SyncEventType.SyncLocal);
|
||||
if (second?.type === SyncEventType.SyncLocal) {
|
||||
assert.strictEqual(second.documentId, "B");
|
||||
}
|
||||
|
||||
assert.strictEqual(queue.next(), undefined);
|
||||
assert.strictEqual(await queue.next(), undefined);
|
||||
});
|
||||
|
||||
it("delete discards subsequent sync-remote events for the same document", () => {
|
||||
it("delete discards subsequent sync-remote events for the same document", async () => {
|
||||
const queue = createQueue();
|
||||
|
||||
queue.enqueue({
|
||||
type: SyncEventType.Delete,
|
||||
documentId: "A",
|
||||
path: "a.md",
|
||||
});
|
||||
queue.enqueue({
|
||||
type: SyncEventType.SyncRemote,
|
||||
remoteVersion: fakeRemoteVersion("A", { vaultUpdateId: 5 })
|
||||
});
|
||||
|
||||
const event = queue.next();
|
||||
const event = await queue.next();
|
||||
assert.strictEqual(event?.type, SyncEventType.Delete);
|
||||
assert.strictEqual(queue.next(), undefined);
|
||||
assert.strictEqual(await queue.next(), undefined);
|
||||
});
|
||||
|
||||
it("delete discards subsequent sync-local and sync-remote for the same document", () => {
|
||||
it("delete discards subsequent sync-local and sync-remote for the same document", async () => {
|
||||
const queue = createQueue();
|
||||
queue.setDocument("a.md", {
|
||||
documentId: "A",
|
||||
parentVersionId: 1,
|
||||
hash: "hash-a"
|
||||
remoteHash: "hash-a"
|
||||
});
|
||||
|
||||
queue.enqueue({
|
||||
type: SyncEventType.Delete,
|
||||
documentId: "A",
|
||||
path: "a.md",
|
||||
});
|
||||
queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A" });
|
||||
queue.enqueue({ type: SyncEventType.Create, path: "b.md" });
|
||||
queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A", path: "a.md", originalPath: "a.md" });
|
||||
queue.enqueue({ type: SyncEventType.Create, path: "b.md", originalPath: "b.md" });
|
||||
queue.enqueue({
|
||||
type: SyncEventType.SyncRemote,
|
||||
remoteVersion: fakeRemoteVersion("A", { vaultUpdateId: 5 })
|
||||
});
|
||||
|
||||
const first = queue.next();
|
||||
const first = await queue.next();
|
||||
assert.strictEqual(first?.type, SyncEventType.Delete);
|
||||
|
||||
// Only the unrelated create should remain
|
||||
const second = queue.next();
|
||||
const second = await queue.next();
|
||||
assert.strictEqual(second?.type, SyncEventType.Create);
|
||||
assert.strictEqual(queue.next(), undefined);
|
||||
assert.strictEqual(await queue.next(), undefined);
|
||||
});
|
||||
|
||||
it("delete with empty documentId does not discard other events", () => {
|
||||
it("delete with promise documentId does not discard other events", async () => {
|
||||
const queue = createQueue();
|
||||
queue.setDocument("a.md", {
|
||||
documentId: "A",
|
||||
parentVersionId: 1,
|
||||
hash: "hash-a"
|
||||
remoteHash: "hash-a"
|
||||
});
|
||||
|
||||
queue.enqueue({ type: SyncEventType.Create, path: "unknown.md", originalPath: "unknown.md" });
|
||||
const createPromise = queue.getCreatePromise("unknown.md");
|
||||
assert.ok(createPromise !== undefined);
|
||||
const event = await queue.next(); // dequeue the create
|
||||
assert.ok(event?.type === SyncEventType.Create);
|
||||
// Resolve so the delete's await doesn't hang
|
||||
event.resolvers!.resolve("NEW");
|
||||
|
||||
queue.enqueue({
|
||||
type: SyncEventType.Delete,
|
||||
documentId: "",
|
||||
path: "unknown.md",
|
||||
documentId: createPromise,
|
||||
});
|
||||
queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A" });
|
||||
queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A", path: "a.md", originalPath: "a.md" });
|
||||
|
||||
queue.next();
|
||||
const second = queue.next();
|
||||
await queue.next(); // delete
|
||||
const second = await queue.next();
|
||||
assert.strictEqual(second?.type, SyncEventType.SyncLocal);
|
||||
});
|
||||
|
||||
it("create can be re-enqueued after being dequeued", () => {
|
||||
it("getCreatePromise returns a promise resolved by the event's resolvers", async () => {
|
||||
const queue = createQueue();
|
||||
queue.enqueue({ type: SyncEventType.Create, path: "a.md" });
|
||||
queue.next();
|
||||
queue.enqueue({ type: SyncEventType.Create, path: "a.md", originalPath: "a.md" });
|
||||
|
||||
queue.enqueue({ type: SyncEventType.Create, path: "a.md" });
|
||||
const promise = queue.getCreatePromise("a.md");
|
||||
assert.ok(promise !== undefined);
|
||||
|
||||
// The syncer resolves via event.resolvers after dequeuing
|
||||
const event = await queue.next();
|
||||
assert.ok(event?.type === SyncEventType.Create);
|
||||
assert.ok(event.resolvers !== undefined);
|
||||
event.resolvers.resolve("resolved-id");
|
||||
|
||||
assert.strictEqual(await promise, "resolved-id");
|
||||
});
|
||||
|
||||
it("rejecting the event's resolvers rejects the create promise", async () => {
|
||||
const queue = createQueue();
|
||||
queue.enqueue({ type: SyncEventType.Create, path: "a.md", originalPath: "a.md" });
|
||||
|
||||
const promise = queue.getCreatePromise("a.md");
|
||||
assert.ok(promise !== undefined);
|
||||
|
||||
const event = await queue.next();
|
||||
assert.ok(event?.type === SyncEventType.Create);
|
||||
assert.ok(event.resolvers !== undefined);
|
||||
event.resolvers.promise.catch(() => { });
|
||||
event.resolvers.reject(new Error("cancelled"));
|
||||
|
||||
await assert.rejects(promise);
|
||||
});
|
||||
|
||||
it("clear rejects all pending create promises", async () => {
|
||||
const queue = createQueue();
|
||||
queue.enqueue({ type: SyncEventType.Create, path: "a.md", originalPath: "a.md" });
|
||||
queue.enqueue({ type: SyncEventType.Create, path: "b.md", originalPath: "b.md" });
|
||||
|
||||
const promiseA = queue.getCreatePromise("a.md");
|
||||
const promiseB = queue.getCreatePromise("b.md");
|
||||
assert.ok(promiseA !== undefined);
|
||||
assert.ok(promiseB !== undefined);
|
||||
|
||||
queue.clear();
|
||||
|
||||
await assert.rejects(promiseA);
|
||||
await assert.rejects(promiseB);
|
||||
});
|
||||
|
||||
it("create can be re-enqueued after being dequeued", async () => {
|
||||
const queue = createQueue();
|
||||
queue.enqueue({ type: SyncEventType.Create, path: "a.md", originalPath: "a.md" });
|
||||
await queue.next();
|
||||
|
||||
queue.enqueue({ type: SyncEventType.Create, path: "a.md", originalPath: "a.md" });
|
||||
assert.strictEqual(queue.size, 1);
|
||||
});
|
||||
|
||||
it("silently ignores create events matching ignore patterns", () => {
|
||||
const queue = createQueue(["*.tmp", ".hidden/**"]);
|
||||
|
||||
queue.enqueue({ type: SyncEventType.Create, path: "scratch.tmp" });
|
||||
queue.enqueue({ type: SyncEventType.Create, path: "scratch.tmp", originalPath: "scratch.tmp" });
|
||||
queue.enqueue({
|
||||
type: SyncEventType.Create,
|
||||
path: ".hidden/secret.md",
|
||||
originalPath: ".hidden/secret.md",
|
||||
});
|
||||
assert.strictEqual(queue.size, 0);
|
||||
|
||||
queue.enqueue({ type: SyncEventType.Create, path: "notes-new.md" });
|
||||
queue.enqueue({ type: SyncEventType.Create, path: "notes-new.md", originalPath: "notes-new.md" });
|
||||
assert.strictEqual(queue.size, 1);
|
||||
|
||||
queue.enqueue({
|
||||
|
|
@ -381,10 +380,10 @@ describe("SyncEventQueue", () => {
|
|||
queue.setDocument("a.md", {
|
||||
documentId: "A",
|
||||
parentVersionId: 1,
|
||||
hash: "hash-a"
|
||||
remoteHash: "hash-a"
|
||||
});
|
||||
queue.enqueue({ type: SyncEventType.Create, path: "b.md" });
|
||||
queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A" });
|
||||
queue.enqueue({ type: SyncEventType.Create, path: "b.md", originalPath: "b.md" });
|
||||
queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A", path: "a.md", originalPath: "a.md" });
|
||||
|
||||
assert.strictEqual(queue.size, 2);
|
||||
|
||||
|
|
@ -392,7 +391,7 @@ describe("SyncEventQueue", () => {
|
|||
|
||||
assert.strictEqual(queue.size, 0);
|
||||
assert.strictEqual(queue.documentCount, 1);
|
||||
assert.strictEqual(queue.getDocument("a.md")?.documentId, "A");
|
||||
assert.strictEqual(queue.getSettledDocumentByPath("a.md")?.documentId, "A");
|
||||
});
|
||||
|
||||
it("allDocuments returns all tracked documents", () => {
|
||||
|
|
@ -400,15 +399,15 @@ describe("SyncEventQueue", () => {
|
|||
queue.setDocument("a.md", {
|
||||
documentId: "A",
|
||||
parentVersionId: 1,
|
||||
hash: "hash-a"
|
||||
remoteHash: "hash-a"
|
||||
});
|
||||
queue.setDocument("b.md", {
|
||||
documentId: "B",
|
||||
parentVersionId: 2,
|
||||
hash: "hash-b"
|
||||
remoteHash: "hash-b"
|
||||
});
|
||||
|
||||
const docs = queue.allDocuments();
|
||||
const docs = queue.allSettledDocuments();
|
||||
assert.strictEqual(docs.length, 2);
|
||||
const paths = docs.map(([p]) => p).sort();
|
||||
assert.deepStrictEqual(paths, ["a.md", "b.md"]);
|
||||
|
|
@ -423,21 +422,150 @@ describe("SyncEventQueue", () => {
|
|||
relativePath: "a.md",
|
||||
documentId: "A",
|
||||
parentVersionId: 5,
|
||||
hash: "hash-a"
|
||||
remoteHash: "hash-a"
|
||||
},
|
||||
{
|
||||
relativePath: "b.md",
|
||||
documentId: "B",
|
||||
parentVersionId: 3,
|
||||
hash: "hash-b"
|
||||
remoteHash: "hash-b"
|
||||
}
|
||||
],
|
||||
lastSeenUpdateId: 4
|
||||
}, async () => { });
|
||||
|
||||
assert.strictEqual(queue.documentCount, 2);
|
||||
assert.strictEqual(queue.getDocument("a.md")?.documentId, "A");
|
||||
assert.strictEqual(queue.getDocument("b.md")?.documentId, "B");
|
||||
assert.strictEqual(queue.getLastSeenUpdateId(), 5);
|
||||
assert.strictEqual(queue.getSettledDocumentByPath("a.md")?.documentId, "A");
|
||||
assert.strictEqual(queue.getSettledDocumentByPath("b.md")?.documentId, "B");
|
||||
assert.strictEqual(queue.lastSeenUpdateId, 5);
|
||||
});
|
||||
|
||||
it("trackedPaths combines documents and pending events", () => {
|
||||
const queue = createQueue();
|
||||
queue.setDocument("a.md", {
|
||||
documentId: "A",
|
||||
parentVersionId: 1,
|
||||
remoteHash: "hash-a"
|
||||
});
|
||||
queue.setDocument("b.md", {
|
||||
documentId: "B",
|
||||
parentVersionId: 2,
|
||||
remoteHash: "hash-b"
|
||||
});
|
||||
|
||||
// Pending create adds a path
|
||||
queue.enqueue({ type: SyncEventType.Create, path: "c.md", originalPath: "c.md" });
|
||||
// Pending delete removes a path
|
||||
queue.enqueue({
|
||||
type: SyncEventType.Delete,
|
||||
documentId: "A",
|
||||
});
|
||||
|
||||
const paths = queue.trackedPaths();
|
||||
assert.deepStrictEqual(
|
||||
[...paths].sort(),
|
||||
["b.md", "c.md"]
|
||||
);
|
||||
});
|
||||
|
||||
it("trackedPaths handles create-delete-create for the same path", () => {
|
||||
const queue = createQueue();
|
||||
|
||||
queue.enqueue({ type: SyncEventType.Create, path: "a.md", originalPath: "a.md" });
|
||||
queue.enqueue({
|
||||
type: SyncEventType.Delete,
|
||||
documentId: Promise.resolve("X"),
|
||||
});
|
||||
queue.enqueue({ type: SyncEventType.Create, path: "a.md", originalPath: "a.md" });
|
||||
|
||||
const paths = queue.trackedPaths();
|
||||
assert.ok(paths.has("a.md"));
|
||||
});
|
||||
|
||||
it("trackedPaths applies moves for promise-based SyncLocal events", () => {
|
||||
const queue = createQueue();
|
||||
|
||||
queue.enqueue({ type: SyncEventType.Create, path: "a.md", originalPath: "a.md" });
|
||||
const createPromise = queue.getCreatePromise("a.md")!;
|
||||
|
||||
// File was renamed from a.md to b.md
|
||||
queue.enqueue({
|
||||
type: SyncEventType.SyncLocal,
|
||||
documentId: createPromise,
|
||||
path: "b.md",
|
||||
originalPath: "a.md",
|
||||
});
|
||||
|
||||
const paths = queue.trackedPaths();
|
||||
assert.ok(!paths.has("a.md"));
|
||||
assert.ok(paths.has("b.md"));
|
||||
});
|
||||
|
||||
it("trackedPaths tracks multiple moves for the same pending create", () => {
|
||||
const queue = createQueue();
|
||||
|
||||
queue.enqueue({ type: SyncEventType.Create, path: "a.md", originalPath: "a.md" });
|
||||
const createPromise = queue.getCreatePromise("a.md")!;
|
||||
|
||||
queue.enqueue({
|
||||
type: SyncEventType.SyncLocal,
|
||||
documentId: createPromise,
|
||||
path: "b.md",
|
||||
originalPath: "a.md",
|
||||
});
|
||||
queue.enqueue({
|
||||
type: SyncEventType.SyncLocal,
|
||||
documentId: createPromise,
|
||||
path: "c.md",
|
||||
originalPath: "a.md",
|
||||
});
|
||||
|
||||
const paths = queue.trackedPaths();
|
||||
assert.ok(!paths.has("a.md"));
|
||||
assert.ok(!paths.has("b.md"));
|
||||
assert.ok(paths.has("c.md"));
|
||||
});
|
||||
|
||||
it("resolveCreate settles the document and replaces promise documentIds in the queue", async () => {
|
||||
const queue = createQueue();
|
||||
|
||||
queue.enqueue({ type: SyncEventType.Create, path: "a.md", originalPath: "a.md" });
|
||||
const createPromise = queue.getCreatePromise("a.md")!;
|
||||
|
||||
// Dependent events enqueued while create is in flight
|
||||
queue.enqueue({
|
||||
type: SyncEventType.SyncLocal,
|
||||
documentId: createPromise,
|
||||
path: "a.md",
|
||||
originalPath: "a.md",
|
||||
});
|
||||
queue.enqueue({
|
||||
type: SyncEventType.Delete,
|
||||
documentId: createPromise,
|
||||
});
|
||||
|
||||
const event = await queue.next(); // dequeue the create
|
||||
assert.ok(event?.type === SyncEventType.Create);
|
||||
|
||||
queue.resolveCreate(event, {
|
||||
documentId: "DOC-1",
|
||||
parentVersionId: 5,
|
||||
remoteHash: "hash-1",
|
||||
});
|
||||
|
||||
// Document is now settled
|
||||
assert.strictEqual(queue.getSettledDocumentByPath("a.md")?.documentId, "DOC-1");
|
||||
|
||||
// Promise was resolved
|
||||
assert.strictEqual(await createPromise, "DOC-1");
|
||||
|
||||
// Remaining events have string documentIds instead of promises.
|
||||
// The SyncLocal + Delete for "DOC-1" coalesce: sync-local is
|
||||
// discarded and the delete is returned (standard coalescing).
|
||||
const deleteEvt = await queue.next();
|
||||
assert.ok(deleteEvt?.type === SyncEventType.Delete);
|
||||
assert.strictEqual(deleteEvt.documentId, "DOC-1");
|
||||
|
||||
assert.strictEqual(await queue.next(), undefined);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -14,9 +14,19 @@ import {
|
|||
} from "./types";
|
||||
|
||||
export class SyncEventQueue {
|
||||
private readonly events: SyncEvent[] = [];
|
||||
// latest state of the filesystem as we know it, excluding
|
||||
// unconfirmed creates but including pending deletes,
|
||||
// it's always indexed by the latest path on disk
|
||||
private readonly documents = new Map<RelativePath, DocumentRecord>();
|
||||
private readonly recentlyDeletedDocumentIds = new Set<DocumentId>();
|
||||
|
||||
// all outstanding operations in order of occurrence,
|
||||
// can include multiple generations of the same document,
|
||||
// e.g.: a create, delete, create sequence for the same path.
|
||||
// The paths for the events must always correspond to the latest
|
||||
// path on disk, so the path of each event may be updated multiple
|
||||
// times.
|
||||
private readonly events: SyncEvent[] = [];
|
||||
|
||||
private lastSeenUpdateIds: CoveredValues;
|
||||
private ignorePatterns: RegExp[];
|
||||
|
||||
|
|
@ -55,7 +65,7 @@ export class SyncEventQueue {
|
|||
this.lastSeenUpdateIds.add(record.parentVersionId);
|
||||
}
|
||||
|
||||
this.logger.debug(`Loaded ${this.documents.size} documents`);
|
||||
this.logger.debug(`Loaded ${this.documents.size} documents and lastSeenUpdateId=${this.lastSeenUpdateIds.min}`);
|
||||
}
|
||||
|
||||
public get size(): number {
|
||||
|
|
@ -66,10 +76,15 @@ export class SyncEventQueue {
|
|||
return this.documents.size;
|
||||
}
|
||||
|
||||
public getLastSeenUpdateId(): VaultUpdateId {
|
||||
public get lastSeenUpdateId(): VaultUpdateId {
|
||||
return this.lastSeenUpdateIds.min;
|
||||
}
|
||||
|
||||
public set lastSeenUpdateId(value: number) {
|
||||
this.lastSeenUpdateIds.min = value;
|
||||
this.saveInTheBackground();
|
||||
}
|
||||
|
||||
public addSeenUpdateId(value: number): void {
|
||||
const previousMin = this.lastSeenUpdateIds.min;
|
||||
this.lastSeenUpdateIds.add(value);
|
||||
|
|
@ -78,12 +93,8 @@ export class SyncEventQueue {
|
|||
}
|
||||
}
|
||||
|
||||
public setLastSeenUpdateId(value: number): void {
|
||||
this.lastSeenUpdateIds.min = value;
|
||||
this.saveInTheBackground();
|
||||
}
|
||||
|
||||
public getDocument(path: RelativePath): DocumentRecord | undefined {
|
||||
public getSettledDocumentByPath(path: RelativePath): DocumentRecord | undefined {
|
||||
return this.documents.get(path);
|
||||
}
|
||||
|
||||
|
|
@ -104,86 +115,96 @@ export class SyncEventQueue {
|
|||
}
|
||||
|
||||
public removeDocument(path: RelativePath): void {
|
||||
const record = this.documents.get(path);
|
||||
if (record !== undefined) {
|
||||
this.recentlyDeletedDocumentIds.add(record.documentId);
|
||||
}
|
||||
this.documents.delete(path);
|
||||
this.saveInTheBackground();
|
||||
}
|
||||
|
||||
/**
|
||||
* Move a document from oldPath to newPath.
|
||||
* If the target path is occupied by a different document, it is removed
|
||||
* and its documentId is returned so the caller can handle the displacement.
|
||||
* Settle a Create event: add the document to the settled map,
|
||||
* resolve the create promise, and replace promise-based documentId
|
||||
* references in the event queue with the actual string documentId.
|
||||
*/
|
||||
public moveDocument(
|
||||
oldPath: RelativePath,
|
||||
newPath: RelativePath
|
||||
): DocumentId | undefined {
|
||||
const record = this.documents.get(oldPath);
|
||||
if (record === undefined) return undefined;
|
||||
public resolveCreate(
|
||||
event: Extract<SyncEvent, { type: SyncEventType.Create }>,
|
||||
record: DocumentRecord
|
||||
): void {
|
||||
const promise = event.resolvers?.promise;
|
||||
|
||||
let displacedDocumentId: DocumentId | undefined = undefined;
|
||||
const existingAtTarget = this.documents.get(newPath);
|
||||
this.documents.set(event.path, record);
|
||||
event.resolvers?.resolve(record.documentId);
|
||||
|
||||
if (promise !== undefined) {
|
||||
for (const e of this.events) {
|
||||
if (
|
||||
existingAtTarget !== undefined &&
|
||||
existingAtTarget.documentId !== record.documentId
|
||||
(e.type === SyncEventType.SyncLocal || e.type === SyncEventType.Delete) &&
|
||||
e.documentId === promise
|
||||
) {
|
||||
displacedDocumentId = existingAtTarget.documentId;
|
||||
this.recentlyDeletedDocumentIds.add(displacedDocumentId);
|
||||
this.documents.delete(newPath);
|
||||
(e as { documentId: DocumentId | Promise<DocumentId> }).documentId = record.documentId;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.documents.delete(oldPath);
|
||||
this.documents.set(newPath, record);
|
||||
this.saveInTheBackground();
|
||||
return displacedDocumentId;
|
||||
}
|
||||
|
||||
public wasRecentlyDeleted(documentId: DocumentId): boolean {
|
||||
return this.recentlyDeletedDocumentIds.has(documentId);
|
||||
public getCreatePromise(path: RelativePath): Promise<DocumentId> | undefined {
|
||||
const event = this.findLastCreate(path);
|
||||
if (event === undefined) return undefined;
|
||||
event.resolvers ??= Promise.withResolvers<DocumentId>();
|
||||
return event.resolvers.promise;
|
||||
}
|
||||
|
||||
public unmarkRecentlyDeleted(documentId: DocumentId): void {
|
||||
this.recentlyDeletedDocumentIds.delete(documentId);
|
||||
}
|
||||
|
||||
|
||||
public allDocuments(): [RelativePath, DocumentRecord][] {
|
||||
public allSettledDocuments(): [RelativePath, DocumentRecord][] {
|
||||
return Array.from(this.documents.entries());
|
||||
}
|
||||
|
||||
public hasCreateEvent(path: RelativePath): boolean {
|
||||
return this.events.some(
|
||||
(e) => e.type === SyncEventType.Create && e.path === path
|
||||
);
|
||||
}
|
||||
/**
|
||||
* Returns the set of paths we expect to exist on disk by replaying
|
||||
* the event queue on top of the settled documents map.
|
||||
*/
|
||||
public trackedPaths(): Set<RelativePath> {
|
||||
const paths = new Set(this.documents.keys());
|
||||
// Track current path for each pending create so moves can be applied
|
||||
const pendingPaths = new Map<Promise<DocumentId>, RelativePath>();
|
||||
|
||||
public updateCreatePath(
|
||||
oldPath: RelativePath,
|
||||
newPath: RelativePath
|
||||
): boolean {
|
||||
for (const event of this.events) {
|
||||
if (event.type === SyncEventType.Create && event.path === oldPath) {
|
||||
event.path = newPath;
|
||||
return true;
|
||||
if (event.type === SyncEventType.Create) {
|
||||
paths.add(event.path);
|
||||
if (event.resolvers !== undefined) {
|
||||
pendingPaths.set(event.resolvers.promise, event.path);
|
||||
}
|
||||
} else if (event.type === SyncEventType.Delete) {
|
||||
if (typeof event.documentId === "string") {
|
||||
const path = this.getDocumentByDocumentId(event.documentId)?.path;
|
||||
if (path) {
|
||||
paths.delete(path);
|
||||
} else {
|
||||
throw new Error(`Delete event for unknown documentId ${event.documentId}`);
|
||||
}
|
||||
return false;
|
||||
} else {
|
||||
const path = pendingPaths.get(event.documentId);
|
||||
if (!path) {
|
||||
throw new Error(`Delete event with unresolved documentId promise`);
|
||||
}
|
||||
paths.delete(path);
|
||||
}
|
||||
} // no need to handle SyncLocal as path updates are applied to this.documents immediately when the event is enqueued
|
||||
}
|
||||
return paths;
|
||||
}
|
||||
|
||||
public hasPendingEventsForPath(path: RelativePath): boolean {
|
||||
const record = this.documents.get(path);
|
||||
const docId = record?.documentId;
|
||||
if (!record) {
|
||||
return true; // if we don't know about this path, it must be pending creation
|
||||
}
|
||||
const docId = record.documentId;
|
||||
return this.events.some(
|
||||
(e) =>
|
||||
(e.type === SyncEventType.Create && e.path === path) ||
|
||||
(e.type === SyncEventType.SyncLocal &&
|
||||
docId !== undefined &&
|
||||
e.documentId === docId) ||
|
||||
(e.type === SyncEventType.Delete &&
|
||||
docId !== undefined &&
|
||||
e.documentId === docId) ||
|
||||
(e.type === SyncEventType.SyncRemote &&
|
||||
e.remoteVersion.relativePath === path)
|
||||
|
|
@ -203,31 +224,26 @@ export class SyncEventQueue {
|
|||
}
|
||||
|
||||
public resetState(): void {
|
||||
this.rejectAllPendingCreates();
|
||||
this.documents.clear();
|
||||
this.recentlyDeletedDocumentIds.clear();
|
||||
this.lastSeenUpdateIds = new CoveredValues(0);
|
||||
this.saveInTheBackground();
|
||||
}
|
||||
|
||||
public clear(): void {
|
||||
this.rejectAllPendingCreates();
|
||||
this.events.length = 0;
|
||||
this.recentlyDeletedDocumentIds.clear();
|
||||
}
|
||||
|
||||
public enqueue(event: SyncEvent): void {
|
||||
if (this.isIgnored(event)) return;
|
||||
|
||||
if (event.type === SyncEventType.Create) {
|
||||
if (this.documents.has(event.path)) return;
|
||||
if (this.hasCreateEvent(event.path)) return;
|
||||
}
|
||||
|
||||
this.events.push(event);
|
||||
}
|
||||
|
||||
|
||||
|
||||
public next(): SyncEvent | undefined {
|
||||
public async next(): Promise<SyncEvent | undefined> {
|
||||
if (this.events.length === 0) return undefined;
|
||||
|
||||
const [first] = this.events;
|
||||
|
|
@ -244,9 +260,7 @@ export class SyncEventQueue {
|
|||
if (first.type === SyncEventType.Delete) {
|
||||
this.events.shift();
|
||||
const { documentId } = first;
|
||||
if (documentId !== "") {
|
||||
this.removeAllEventsForDocumentId(documentId);
|
||||
}
|
||||
this.removeAllEventsForDocumentId(await documentId);
|
||||
return first;
|
||||
}
|
||||
|
||||
|
|
@ -261,16 +275,18 @@ export class SyncEventQueue {
|
|||
e.documentId === documentId
|
||||
);
|
||||
if (deleteEvent !== undefined) {
|
||||
this.removeAllSyncLocalsForDocumentId(documentId);
|
||||
this.removeAllSyncLocalsForDocumentId(await documentId);
|
||||
removeFromArray(this.events, deleteEvent);
|
||||
return deleteEvent;
|
||||
}
|
||||
|
||||
// Coalesce multiple sync-locals for the same documentId to the last one
|
||||
// Coalesce multiple sync-locals for the same documentId and
|
||||
// original path to the last one
|
||||
const matching = this.events.filter(
|
||||
(e) =>
|
||||
e.type === SyncEventType.SyncLocal &&
|
||||
e.documentId === documentId
|
||||
e.documentId === documentId &&
|
||||
e.originalPath === first.originalPath
|
||||
);
|
||||
const result = matching[matching.length - 1];
|
||||
for (const item of matching) {
|
||||
|
|
@ -328,6 +344,49 @@ export class SyncEventQueue {
|
|||
}
|
||||
}
|
||||
|
||||
public updatePendingCreatePath(
|
||||
oldPath: RelativePath,
|
||||
newPath: RelativePath
|
||||
): void {
|
||||
const createEvent = this.findLastCreate(oldPath);
|
||||
if (createEvent === undefined) return;
|
||||
|
||||
const promise = createEvent.resolvers?.promise;
|
||||
createEvent.path = newPath;
|
||||
|
||||
if (promise !== undefined) {
|
||||
for (const e of this.events) {
|
||||
if (
|
||||
e.type === SyncEventType.SyncLocal &&
|
||||
e.documentId === promise
|
||||
) {
|
||||
e.path = newPath;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private findLastCreate(
|
||||
path: RelativePath
|
||||
): Extract<SyncEvent, { type: SyncEventType.Create }> | undefined {
|
||||
for (let i = this.events.length - 1; i >= 0; i--) {
|
||||
const e = this.events[i];
|
||||
if (e.type === SyncEventType.Create && e.path === path) {
|
||||
return e;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
private rejectAllPendingCreates(): void {
|
||||
for (const event of this.events) {
|
||||
if (event.type === SyncEventType.Create && event.resolvers !== undefined) {
|
||||
event.resolvers.promise.catch(() => { /* suppressed — consumer may not be listening */ });
|
||||
event.resolvers.reject(new Error("Create was cancelled"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private saveInTheBackground(): void {
|
||||
void this.save().catch((error: unknown) => {
|
||||
this.logger.error(`Error saving sync state: ${error}`);
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ export type RelativePath = string;
|
|||
export interface DocumentRecord {
|
||||
documentId: DocumentId;
|
||||
parentVersionId: VaultUpdateId;
|
||||
hash: string;
|
||||
remoteHash: string;
|
||||
remoteRelativePath?: RelativePath;
|
||||
}
|
||||
|
||||
|
|
@ -23,18 +23,26 @@ export interface StoredSyncState {
|
|||
export enum SyncEventType {
|
||||
Create = "create",
|
||||
SyncLocal = "sync-local",
|
||||
SyncRemote = "sync-remote",
|
||||
Delete = "delete",
|
||||
SyncRemote = "sync-remote",
|
||||
}
|
||||
|
||||
export type SyncEvent =
|
||||
| { type: SyncEventType.Create; path: RelativePath }
|
||||
| { type: SyncEventType.SyncLocal; documentId: DocumentId }
|
||||
| {
|
||||
type: SyncEventType.Create;
|
||||
path: RelativePath; // current path on disk
|
||||
originalPath: RelativePath; // original path on disk when the event was created
|
||||
resolvers?: PromiseWithResolvers<DocumentId>
|
||||
}
|
||||
| {
|
||||
type: SyncEventType.SyncLocal;
|
||||
documentId: DocumentId | Promise<DocumentId>; // if it's a promise, the promise is fulfilled once the document's create event is processed
|
||||
path: RelativePath; // current path on disk
|
||||
originalPath: RelativePath; // original path on disk when the event was created
|
||||
}
|
||||
| {
|
||||
type: SyncEventType.Delete;
|
||||
documentId: DocumentId;
|
||||
path: RelativePath;
|
||||
displacedAtVersion?: VaultUpdateId;
|
||||
documentId: DocumentId | Promise<DocumentId>; // if it's a promise, the promise is fulfilled once the document's create event is processed
|
||||
}
|
||||
| {
|
||||
type: SyncEventType.SyncRemote;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue