.
This commit is contained in:
parent
a7b588da97
commit
19d5dc1999
11 changed files with 352 additions and 349 deletions
|
|
@ -8,7 +8,7 @@ import { assertSetContainsExactly } from "../utils/assert-set-contains-exactly";
|
||||||
import type { FileSystemOperations } from "./filesystem-operations";
|
import type { FileSystemOperations } from "./filesystem-operations";
|
||||||
import type { TextWithCursors } from "reconcile-text";
|
import type { TextWithCursors } from "reconcile-text";
|
||||||
import type { ServerConfig, ServerConfigData } from "../services/server-config";
|
import type { ServerConfig, ServerConfigData } from "../services/server-config";
|
||||||
import { isConflictPath } from "../utils/conflict-path";
|
import { isConflictPath } from "../sync-operations/conflict-path";
|
||||||
|
|
||||||
class MockServerConfig implements Pick<ServerConfig, "getConfig"> {
|
class MockServerConfig implements Pick<ServerConfig, "getConfig"> {
|
||||||
public async getConfig(): Promise<ServerConfigData> {
|
public async getConfig(): Promise<ServerConfigData> {
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@ import type { TextWithCursors } from "reconcile-text";
|
||||||
import { reconcile } from "reconcile-text";
|
import { reconcile } from "reconcile-text";
|
||||||
import { isFileTypeMergable } from "../utils/is-file-type-mergable";
|
import { isFileTypeMergable } from "../utils/is-file-type-mergable";
|
||||||
import { isBinary } from "../utils/is-binary";
|
import { isBinary } from "../utils/is-binary";
|
||||||
import { buildConflictFileName } from "../utils/conflict-path";
|
import { buildConflictFileName } from "../sync-operations/conflict-path";
|
||||||
import type { ServerConfig } from "../services/server-config";
|
import type { ServerConfig } from "../services/server-config";
|
||||||
|
|
||||||
export class FileOperations {
|
export class FileOperations {
|
||||||
|
|
|
||||||
|
|
@ -57,8 +57,8 @@ export class SyncClient {
|
||||||
>
|
>
|
||||||
) { }
|
) { }
|
||||||
|
|
||||||
public get documentCount(): number {
|
public get syncedDocumentCount(): number {
|
||||||
return this.syncEventQueue.documentCount;
|
return this.syncEventQueue.syncedDocumentCount;
|
||||||
}
|
}
|
||||||
|
|
||||||
public get isWebSocketConnected(): boolean {
|
public get isWebSocketConnected(): boolean {
|
||||||
|
|
@ -390,7 +390,7 @@ export class SyncClient {
|
||||||
|
|
||||||
public get hasPendingWork(): boolean {
|
public get hasPendingWork(): boolean {
|
||||||
return (
|
return (
|
||||||
this.syncEventQueue.size > 0 ||
|
this.syncEventQueue.pendingUpdateCount > 0 ||
|
||||||
this.webSocketManager.hasOutstandingWork
|
this.webSocketManager.hasOutstandingWork
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
@ -408,7 +408,7 @@ export class SyncClient {
|
||||||
return DocumentSyncStatus.SYNCING;
|
return DocumentSyncStatus.SYNCING;
|
||||||
}
|
}
|
||||||
|
|
||||||
return this.syncer.hasPendingOperationsForDocument(relativePath)
|
return this.syncEventQueue.hasPendingEventsForPath(relativePath)
|
||||||
? DocumentSyncStatus.SYNCING
|
? DocumentSyncStatus.SYNCING
|
||||||
: DocumentSyncStatus.UP_TO_DATE;
|
: DocumentSyncStatus.UP_TO_DATE;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
import type { RelativePath } from "../sync-operations/types";
|
import type { RelativePath } from "./types";
|
||||||
|
|
||||||
// Local-only files displaced by `FileOperations.ensureClearPath` are named
|
// Local-only files displaced by `FileOperations.ensureClearPath` are named
|
||||||
// `conflict-<uuid>-<originalName>`. The UUID is a full RFC-4122 v4 value so
|
// `conflict-<uuid>-<originalName>`. The UUID is a full RFC-4122 v4 value so
|
||||||
|
|
@ -89,7 +89,7 @@ function enqueueRenamedDocuments(
|
||||||
const hasLocalRename = remoteRelPath !== undefined && remoteRelPath !== path;
|
const hasLocalRename = remoteRelPath !== undefined && remoteRelPath !== path;
|
||||||
|
|
||||||
if (hasLocalRename) {
|
if (hasLocalRename) {
|
||||||
queue.enqueue({ type: SyncEventType.SyncLocal, path });
|
queue.enqueue({ type: SyncEventType.LocalUpdate, path });
|
||||||
locallyRenamedPaths.add(path);
|
locallyRenamedPaths.add(path);
|
||||||
logger.debug(`Document ${path} was renamed locally (from ${remoteRelPath}), scheduling sync`);
|
logger.debug(`Document ${path} was renamed locally (from ${remoteRelPath}), scheduling sync`);
|
||||||
}
|
}
|
||||||
|
|
@ -243,5 +243,5 @@ async function handleNewFile(
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.debug(`Document ${relativePath} not found in database, scheduling sync to create it`);
|
logger.debug(`Document ${relativePath} not found in database, scheduling sync to create it`);
|
||||||
return { instruction: { type: SyncEventType.Create, relativePath } };
|
return { instruction: { type: SyncEventType.LocalCreate, relativePath } };
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -38,13 +38,13 @@ describe("SyncEventQueue", () => {
|
||||||
remoteHash: "hash-a"
|
remoteHash: "hash-a"
|
||||||
});
|
});
|
||||||
|
|
||||||
queue.enqueue({ type: SyncEventType.SyncLocal, path: "a.md" });
|
queue.enqueue({ type: SyncEventType.LocalUpdate, path: "a.md" });
|
||||||
queue.enqueue({ type: SyncEventType.SyncLocal, path: "a.md" });
|
queue.enqueue({ type: SyncEventType.LocalUpdate, path: "a.md" });
|
||||||
queue.enqueue({ type: SyncEventType.Delete, path: "a.md" });
|
queue.enqueue({ type: SyncEventType.LocalDelete, path: "a.md" });
|
||||||
|
|
||||||
const event = await queue.next();
|
const event = await queue.next();
|
||||||
assert.strictEqual(event?.type, SyncEventType.Delete);
|
assert.strictEqual(event?.type, SyncEventType.LocalDelete);
|
||||||
if (event?.type === SyncEventType.Delete) {
|
if (event?.type === SyncEventType.LocalDelete) {
|
||||||
assert.strictEqual(event.documentId, "A");
|
assert.strictEqual(event.documentId, "A");
|
||||||
}
|
}
|
||||||
assert.strictEqual(await queue.next(), undefined);
|
assert.strictEqual(await queue.next(), undefined);
|
||||||
|
|
@ -58,34 +58,34 @@ describe("SyncEventQueue", () => {
|
||||||
remoteHash: "hash-a"
|
remoteHash: "hash-a"
|
||||||
});
|
});
|
||||||
|
|
||||||
queue.enqueue({ type: SyncEventType.SyncLocal, path: "a.md" });
|
queue.enqueue({ type: SyncEventType.LocalUpdate, path: "a.md" });
|
||||||
queue.enqueue({ type: SyncEventType.SyncLocal, path: "a.md" });
|
queue.enqueue({ type: SyncEventType.LocalUpdate, path: "a.md" });
|
||||||
queue.enqueue({ type: SyncEventType.SyncLocal, path: "a.md" });
|
queue.enqueue({ type: SyncEventType.LocalUpdate, path: "a.md" });
|
||||||
|
|
||||||
const event = await queue.next();
|
const event = await queue.next();
|
||||||
assert.strictEqual(event?.type, SyncEventType.SyncLocal);
|
assert.strictEqual(event?.type, SyncEventType.LocalUpdate);
|
||||||
assert.strictEqual(await queue.next(), undefined);
|
assert.strictEqual(await queue.next(), undefined);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("sync-remote events for the same documentId coalesce to the last one", async () => {
|
it("sync-remote-content events for the same documentId coalesce to the last one", async () => {
|
||||||
const queue = createQueue();
|
const queue = createQueue();
|
||||||
|
|
||||||
queue.enqueue({
|
queue.enqueue({
|
||||||
type: SyncEventType.SyncRemote,
|
type: SyncEventType.RemoteUpdate,
|
||||||
remoteVersion: fakeRemoteVersion("A", { vaultUpdateId: 1 })
|
remoteVersion: fakeRemoteVersion("A", { vaultUpdateId: 1 })
|
||||||
});
|
});
|
||||||
queue.enqueue({
|
queue.enqueue({
|
||||||
type: SyncEventType.SyncRemote,
|
type: SyncEventType.RemoteUpdate,
|
||||||
remoteVersion: fakeRemoteVersion("A", { vaultUpdateId: 2 })
|
remoteVersion: fakeRemoteVersion("A", { vaultUpdateId: 2 })
|
||||||
});
|
});
|
||||||
queue.enqueue({
|
queue.enqueue({
|
||||||
type: SyncEventType.SyncRemote,
|
type: SyncEventType.RemoteUpdate,
|
||||||
remoteVersion: fakeRemoteVersion("A", { vaultUpdateId: 3 })
|
remoteVersion: fakeRemoteVersion("A", { vaultUpdateId: 3 })
|
||||||
});
|
});
|
||||||
|
|
||||||
const event = await queue.next();
|
const event = await queue.next();
|
||||||
assert.strictEqual(event?.type, SyncEventType.SyncRemote);
|
assert.strictEqual(event?.type, SyncEventType.RemoteUpdate);
|
||||||
if (event?.type === SyncEventType.SyncRemote) {
|
if (event?.type === SyncEventType.RemoteUpdate) {
|
||||||
assert.strictEqual(event.remoteVersion.vaultUpdateId, 3);
|
assert.strictEqual(event.remoteVersion.vaultUpdateId, 3);
|
||||||
}
|
}
|
||||||
assert.strictEqual(await queue.next(), undefined);
|
assert.strictEqual(await queue.next(), undefined);
|
||||||
|
|
@ -93,18 +93,18 @@ describe("SyncEventQueue", () => {
|
||||||
|
|
||||||
it("create events are returned FIFO", async () => {
|
it("create events are returned FIFO", async () => {
|
||||||
const queue = createQueue();
|
const queue = createQueue();
|
||||||
queue.enqueue({ type: SyncEventType.Create, path: "a.md" });
|
queue.enqueue({ type: SyncEventType.LocalCreate, path: "a.md" });
|
||||||
queue.enqueue({ type: SyncEventType.Create, path: "b.md" });
|
queue.enqueue({ type: SyncEventType.LocalCreate, path: "b.md" });
|
||||||
|
|
||||||
const first = await queue.next();
|
const first = await queue.next();
|
||||||
assert.strictEqual(first?.type, SyncEventType.Create);
|
assert.strictEqual(first?.type, SyncEventType.LocalCreate);
|
||||||
if (first?.type === SyncEventType.Create) {
|
if (first?.type === SyncEventType.LocalCreate) {
|
||||||
assert.strictEqual(first.path, "a.md");
|
assert.strictEqual(first.path, "a.md");
|
||||||
}
|
}
|
||||||
|
|
||||||
const second = await queue.next();
|
const second = await queue.next();
|
||||||
assert.strictEqual(second?.type, SyncEventType.Create);
|
assert.strictEqual(second?.type, SyncEventType.LocalCreate);
|
||||||
if (second?.type === SyncEventType.Create) {
|
if (second?.type === SyncEventType.LocalCreate) {
|
||||||
assert.strictEqual(second.path, "b.md");
|
assert.strictEqual(second.path, "b.md");
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
@ -117,33 +117,33 @@ describe("SyncEventQueue", () => {
|
||||||
remoteHash: "hash-a"
|
remoteHash: "hash-a"
|
||||||
});
|
});
|
||||||
|
|
||||||
queue.enqueue({ type: SyncEventType.Delete, path: "a.md" });
|
queue.enqueue({ type: SyncEventType.LocalDelete, path: "a.md" });
|
||||||
|
|
||||||
const event = await queue.next();
|
const event = await queue.next();
|
||||||
assert.strictEqual(event?.type, SyncEventType.Delete);
|
assert.strictEqual(event?.type, SyncEventType.LocalDelete);
|
||||||
if (event?.type === SyncEventType.Delete) {
|
if (event?.type === SyncEventType.LocalDelete) {
|
||||||
assert.strictEqual(event.documentId, "A");
|
assert.strictEqual(event.documentId, "A");
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
it("delete for unknown path is silently ignored", () => {
|
it("delete for unknown path is silently ignored", () => {
|
||||||
const queue = createQueue();
|
const queue = createQueue();
|
||||||
queue.enqueue({ type: SyncEventType.Delete, path: "unknown.md" });
|
queue.enqueue({ type: SyncEventType.LocalDelete, path: "unknown.md" });
|
||||||
assert.strictEqual(queue.size, 0);
|
assert.strictEqual(queue.pendingUpdateCount, 0);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("document store CRUD operations work correctly", () => {
|
it("document store CRUD operations work correctly", () => {
|
||||||
const queue = createQueue();
|
const queue = createQueue();
|
||||||
|
|
||||||
assert.strictEqual(queue.getSettledDocumentByPath("a.md"), undefined);
|
assert.strictEqual(queue.getSettledDocumentByPath("a.md"), undefined);
|
||||||
assert.strictEqual(queue.documentCount, 0);
|
assert.strictEqual(queue.syncedDocumentCount, 0);
|
||||||
|
|
||||||
queue.setDocument("a.md", {
|
queue.setDocument("a.md", {
|
||||||
documentId: "A",
|
documentId: "A",
|
||||||
parentVersionId: 1,
|
parentVersionId: 1,
|
||||||
remoteHash: "hash-a"
|
remoteHash: "hash-a"
|
||||||
});
|
});
|
||||||
assert.strictEqual(queue.documentCount, 1);
|
assert.strictEqual(queue.syncedDocumentCount, 1);
|
||||||
assert.deepStrictEqual(queue.getSettledDocumentByPath("a.md"), {
|
assert.deepStrictEqual(queue.getSettledDocumentByPath("a.md"), {
|
||||||
documentId: "A",
|
documentId: "A",
|
||||||
parentVersionId: 1,
|
parentVersionId: 1,
|
||||||
|
|
@ -155,7 +155,7 @@ describe("SyncEventQueue", () => {
|
||||||
assert.strictEqual(found?.record.documentId, "A");
|
assert.strictEqual(found?.record.documentId, "A");
|
||||||
|
|
||||||
queue.removeDocument("a.md");
|
queue.removeDocument("a.md");
|
||||||
assert.strictEqual(queue.documentCount, 0);
|
assert.strictEqual(queue.syncedDocumentCount, 0);
|
||||||
assert.strictEqual(queue.getSettledDocumentByPath("a.md"), undefined);
|
assert.strictEqual(queue.getSettledDocumentByPath("a.md"), undefined);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -167,7 +167,7 @@ describe("SyncEventQueue", () => {
|
||||||
remoteHash: "hash-a"
|
remoteHash: "hash-a"
|
||||||
});
|
});
|
||||||
|
|
||||||
queue.enqueue({ type: SyncEventType.SyncLocal, path: "b.md", oldPath: "a.md" });
|
queue.enqueue({ type: SyncEventType.LocalUpdate, path: "b.md", oldPath: "a.md" });
|
||||||
assert.strictEqual(queue.getSettledDocumentByPath("a.md"), undefined);
|
assert.strictEqual(queue.getSettledDocumentByPath("a.md"), undefined);
|
||||||
assert.strictEqual(queue.getSettledDocumentByPath("b.md")?.documentId, "A");
|
assert.strictEqual(queue.getSettledDocumentByPath("b.md")?.documentId, "A");
|
||||||
});
|
});
|
||||||
|
|
@ -185,29 +185,29 @@ describe("SyncEventQueue", () => {
|
||||||
remoteHash: "hash-b"
|
remoteHash: "hash-b"
|
||||||
});
|
});
|
||||||
|
|
||||||
queue.enqueue({ type: SyncEventType.SyncLocal, path: "a.md" });
|
queue.enqueue({ type: SyncEventType.LocalUpdate, path: "a.md" });
|
||||||
queue.enqueue({ type: SyncEventType.SyncLocal, path: "b.md" });
|
queue.enqueue({ type: SyncEventType.LocalUpdate, path: "b.md" });
|
||||||
queue.enqueue({ type: SyncEventType.Delete, path: "a.md" });
|
queue.enqueue({ type: SyncEventType.LocalDelete, path: "a.md" });
|
||||||
queue.enqueue({ type: SyncEventType.SyncLocal, path: "b.md" });
|
queue.enqueue({ type: SyncEventType.LocalUpdate, path: "b.md" });
|
||||||
|
|
||||||
// First next() should see the delete for A (coalescing sync-local + delete)
|
// First next() should see the delete for A (coalescing sync-local + delete)
|
||||||
const first = await queue.next();
|
const first = await queue.next();
|
||||||
assert.strictEqual(first?.type, SyncEventType.Delete);
|
assert.strictEqual(first?.type, SyncEventType.LocalDelete);
|
||||||
if (first?.type === SyncEventType.Delete) {
|
if (first?.type === SyncEventType.LocalDelete) {
|
||||||
assert.strictEqual(first.documentId, "A");
|
assert.strictEqual(first.documentId, "A");
|
||||||
}
|
}
|
||||||
|
|
||||||
// Remaining should be the coalesced sync-local for B
|
// Remaining should be the coalesced sync-local for B
|
||||||
const second = await queue.next();
|
const second = await queue.next();
|
||||||
assert.strictEqual(second?.type, SyncEventType.SyncLocal);
|
assert.strictEqual(second?.type, SyncEventType.LocalUpdate);
|
||||||
if (second?.type === SyncEventType.SyncLocal) {
|
if (second?.type === SyncEventType.LocalUpdate) {
|
||||||
assert.strictEqual(second.documentId, "B");
|
assert.strictEqual(second.documentId, "B");
|
||||||
}
|
}
|
||||||
|
|
||||||
assert.strictEqual(await queue.next(), undefined);
|
assert.strictEqual(await queue.next(), undefined);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("delete discards subsequent sync-remote events for the same document", async () => {
|
it("delete discards subsequent sync-remote-content events for the same document", async () => {
|
||||||
const queue = createQueue();
|
const queue = createQueue();
|
||||||
queue.setDocument("a.md", {
|
queue.setDocument("a.md", {
|
||||||
documentId: "A",
|
documentId: "A",
|
||||||
|
|
@ -215,18 +215,18 @@ describe("SyncEventQueue", () => {
|
||||||
remoteHash: "hash-a"
|
remoteHash: "hash-a"
|
||||||
});
|
});
|
||||||
|
|
||||||
queue.enqueue({ type: SyncEventType.Delete, path: "a.md" });
|
queue.enqueue({ type: SyncEventType.LocalDelete, path: "a.md" });
|
||||||
queue.enqueue({
|
queue.enqueue({
|
||||||
type: SyncEventType.SyncRemote,
|
type: SyncEventType.RemoteUpdate,
|
||||||
remoteVersion: fakeRemoteVersion("A", { vaultUpdateId: 5 })
|
remoteVersion: fakeRemoteVersion("A", { vaultUpdateId: 5 })
|
||||||
});
|
});
|
||||||
|
|
||||||
const event = await queue.next();
|
const event = await queue.next();
|
||||||
assert.strictEqual(event?.type, SyncEventType.Delete);
|
assert.strictEqual(event?.type, SyncEventType.LocalDelete);
|
||||||
assert.strictEqual(await queue.next(), undefined);
|
assert.strictEqual(await queue.next(), undefined);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("delete discards subsequent sync-local and sync-remote for the same document", async () => {
|
it("delete discards subsequent sync-local and sync-remote-content for the same document", async () => {
|
||||||
const queue = createQueue();
|
const queue = createQueue();
|
||||||
queue.setDocument("a.md", {
|
queue.setDocument("a.md", {
|
||||||
documentId: "A",
|
documentId: "A",
|
||||||
|
|
@ -234,20 +234,20 @@ describe("SyncEventQueue", () => {
|
||||||
remoteHash: "hash-a"
|
remoteHash: "hash-a"
|
||||||
});
|
});
|
||||||
|
|
||||||
queue.enqueue({ type: SyncEventType.Delete, path: "a.md" });
|
queue.enqueue({ type: SyncEventType.LocalDelete, path: "a.md" });
|
||||||
queue.enqueue({ type: SyncEventType.SyncLocal, path: "a.md" });
|
queue.enqueue({ type: SyncEventType.LocalUpdate, path: "a.md" });
|
||||||
queue.enqueue({ type: SyncEventType.Create, path: "b.md" });
|
queue.enqueue({ type: SyncEventType.LocalCreate, path: "b.md" });
|
||||||
queue.enqueue({
|
queue.enqueue({
|
||||||
type: SyncEventType.SyncRemote,
|
type: SyncEventType.RemoteUpdate,
|
||||||
remoteVersion: fakeRemoteVersion("A", { vaultUpdateId: 5 })
|
remoteVersion: fakeRemoteVersion("A", { vaultUpdateId: 5 })
|
||||||
});
|
});
|
||||||
|
|
||||||
const first = await queue.next();
|
const first = await queue.next();
|
||||||
assert.strictEqual(first?.type, SyncEventType.Delete);
|
assert.strictEqual(first?.type, SyncEventType.LocalDelete);
|
||||||
|
|
||||||
// Only the unrelated create should remain
|
// Only the unrelated create should remain
|
||||||
const second = await queue.next();
|
const second = await queue.next();
|
||||||
assert.strictEqual(second?.type, SyncEventType.Create);
|
assert.strictEqual(second?.type, SyncEventType.LocalCreate);
|
||||||
assert.strictEqual(await queue.next(), undefined);
|
assert.strictEqual(await queue.next(), undefined);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -260,30 +260,30 @@ describe("SyncEventQueue", () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
// Create is pending — Delete for same path gets a promise documentId
|
// Create is pending — Delete for same path gets a promise documentId
|
||||||
queue.enqueue({ type: SyncEventType.Create, path: "unknown.md" });
|
queue.enqueue({ type: SyncEventType.LocalCreate, path: "unknown.md" });
|
||||||
queue.enqueue({ type: SyncEventType.Delete, path: "unknown.md" });
|
queue.enqueue({ type: SyncEventType.LocalDelete, path: "unknown.md" });
|
||||||
queue.enqueue({ type: SyncEventType.SyncLocal, path: "a.md" });
|
queue.enqueue({ type: SyncEventType.LocalUpdate, path: "a.md" });
|
||||||
|
|
||||||
// Dequeue and resolve the Create
|
// Dequeue and resolve the Create
|
||||||
const event = await queue.next();
|
const event = await queue.next();
|
||||||
assert.ok(event?.type === SyncEventType.Create);
|
assert.ok(event?.type === SyncEventType.LocalCreate);
|
||||||
event.resolvers!.resolve("NEW");
|
event.resolvers!.resolve("NEW");
|
||||||
|
|
||||||
await queue.next(); // delete
|
await queue.next(); // delete
|
||||||
const second = await queue.next();
|
const second = await queue.next();
|
||||||
assert.strictEqual(second?.type, SyncEventType.SyncLocal);
|
assert.strictEqual(second?.type, SyncEventType.LocalUpdate);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("getCreatePromise returns a promise resolved by the event's resolvers", async () => {
|
it("getCreatePromise returns a promise resolved by the event's resolvers", async () => {
|
||||||
const queue = createQueue();
|
const queue = createQueue();
|
||||||
queue.enqueue({ type: SyncEventType.Create, path: "a.md" });
|
queue.enqueue({ type: SyncEventType.LocalCreate, path: "a.md" });
|
||||||
|
|
||||||
const promise = queue.getCreatePromise("a.md");
|
const promise = queue.getCreatePromise("a.md");
|
||||||
assert.ok(promise !== undefined);
|
assert.ok(promise !== undefined);
|
||||||
|
|
||||||
// The syncer resolves via event.resolvers after dequeuing
|
// The syncer resolves via event.resolvers after dequeuing
|
||||||
const event = await queue.next();
|
const event = await queue.next();
|
||||||
assert.ok(event?.type === SyncEventType.Create);
|
assert.ok(event?.type === SyncEventType.LocalCreate);
|
||||||
assert.ok(event.resolvers !== undefined);
|
assert.ok(event.resolvers !== undefined);
|
||||||
event.resolvers.resolve("resolved-id");
|
event.resolvers.resolve("resolved-id");
|
||||||
|
|
||||||
|
|
@ -292,13 +292,13 @@ describe("SyncEventQueue", () => {
|
||||||
|
|
||||||
it("rejecting the event's resolvers rejects the create promise", async () => {
|
it("rejecting the event's resolvers rejects the create promise", async () => {
|
||||||
const queue = createQueue();
|
const queue = createQueue();
|
||||||
queue.enqueue({ type: SyncEventType.Create, path: "a.md" });
|
queue.enqueue({ type: SyncEventType.LocalCreate, path: "a.md" });
|
||||||
|
|
||||||
const promise = queue.getCreatePromise("a.md");
|
const promise = queue.getCreatePromise("a.md");
|
||||||
assert.ok(promise !== undefined);
|
assert.ok(promise !== undefined);
|
||||||
|
|
||||||
const event = await queue.next();
|
const event = await queue.next();
|
||||||
assert.ok(event?.type === SyncEventType.Create);
|
assert.ok(event?.type === SyncEventType.LocalCreate);
|
||||||
assert.ok(event.resolvers !== undefined);
|
assert.ok(event.resolvers !== undefined);
|
||||||
event.resolvers.promise.catch(() => { });
|
event.resolvers.promise.catch(() => { });
|
||||||
event.resolvers.reject(new Error("cancelled"));
|
event.resolvers.reject(new Error("cancelled"));
|
||||||
|
|
@ -308,8 +308,8 @@ describe("SyncEventQueue", () => {
|
||||||
|
|
||||||
it("clear rejects all pending create promises", async () => {
|
it("clear rejects all pending create promises", async () => {
|
||||||
const queue = createQueue();
|
const queue = createQueue();
|
||||||
queue.enqueue({ type: SyncEventType.Create, path: "a.md" });
|
queue.enqueue({ type: SyncEventType.LocalCreate, path: "a.md" });
|
||||||
queue.enqueue({ type: SyncEventType.Create, path: "b.md" });
|
queue.enqueue({ type: SyncEventType.LocalCreate, path: "b.md" });
|
||||||
|
|
||||||
const promiseA = queue.getCreatePromise("a.md");
|
const promiseA = queue.getCreatePromise("a.md");
|
||||||
const promiseB = queue.getCreatePromise("b.md");
|
const promiseB = queue.getCreatePromise("b.md");
|
||||||
|
|
@ -324,28 +324,28 @@ describe("SyncEventQueue", () => {
|
||||||
|
|
||||||
it("create can be re-enqueued after being dequeued", async () => {
|
it("create can be re-enqueued after being dequeued", async () => {
|
||||||
const queue = createQueue();
|
const queue = createQueue();
|
||||||
queue.enqueue({ type: SyncEventType.Create, path: "a.md" });
|
queue.enqueue({ type: SyncEventType.LocalCreate, path: "a.md" });
|
||||||
await queue.next();
|
await queue.next();
|
||||||
|
|
||||||
queue.enqueue({ type: SyncEventType.Create, path: "a.md" });
|
queue.enqueue({ type: SyncEventType.LocalCreate, path: "a.md" });
|
||||||
assert.strictEqual(queue.size, 1);
|
assert.strictEqual(queue.pendingUpdateCount, 1);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("silently ignores create events matching ignore patterns", () => {
|
it("silently ignores create events matching ignore patterns", () => {
|
||||||
const queue = createQueue(["*.tmp", ".hidden/**"]);
|
const queue = createQueue(["*.tmp", ".hidden/**"]);
|
||||||
|
|
||||||
queue.enqueue({ type: SyncEventType.Create, path: "scratch.tmp" });
|
queue.enqueue({ type: SyncEventType.LocalCreate, path: "scratch.tmp" });
|
||||||
queue.enqueue({ type: SyncEventType.Create, path: ".hidden/secret.md" });
|
queue.enqueue({ type: SyncEventType.LocalCreate, path: ".hidden/secret.md" });
|
||||||
assert.strictEqual(queue.size, 0);
|
assert.strictEqual(queue.pendingUpdateCount, 0);
|
||||||
|
|
||||||
queue.enqueue({ type: SyncEventType.Create, path: "notes-new.md" });
|
queue.enqueue({ type: SyncEventType.LocalCreate, path: "notes-new.md" });
|
||||||
assert.strictEqual(queue.size, 1);
|
assert.strictEqual(queue.pendingUpdateCount, 1);
|
||||||
|
|
||||||
queue.enqueue({
|
queue.enqueue({
|
||||||
type: SyncEventType.SyncRemote,
|
type: SyncEventType.RemoteUpdate,
|
||||||
remoteVersion: fakeRemoteVersion("N")
|
remoteVersion: fakeRemoteVersion("N")
|
||||||
});
|
});
|
||||||
assert.strictEqual(queue.size, 2);
|
assert.strictEqual(queue.pendingUpdateCount, 2);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("clear removes events but keeps documents", () => {
|
it("clear removes events but keeps documents", () => {
|
||||||
|
|
@ -355,15 +355,15 @@ describe("SyncEventQueue", () => {
|
||||||
parentVersionId: 1,
|
parentVersionId: 1,
|
||||||
remoteHash: "hash-a"
|
remoteHash: "hash-a"
|
||||||
});
|
});
|
||||||
queue.enqueue({ type: SyncEventType.Create, path: "b.md" });
|
queue.enqueue({ type: SyncEventType.LocalCreate, path: "b.md" });
|
||||||
queue.enqueue({ type: SyncEventType.SyncLocal, path: "a.md" });
|
queue.enqueue({ type: SyncEventType.LocalUpdate, path: "a.md" });
|
||||||
|
|
||||||
assert.strictEqual(queue.size, 2);
|
assert.strictEqual(queue.pendingUpdateCount, 2);
|
||||||
|
|
||||||
queue.clear();
|
queue.clear();
|
||||||
|
|
||||||
assert.strictEqual(queue.size, 0);
|
assert.strictEqual(queue.pendingUpdateCount, 0);
|
||||||
assert.strictEqual(queue.documentCount, 1);
|
assert.strictEqual(queue.syncedDocumentCount, 1);
|
||||||
assert.strictEqual(queue.getSettledDocumentByPath("a.md")?.documentId, "A");
|
assert.strictEqual(queue.getSettledDocumentByPath("a.md")?.documentId, "A");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -407,7 +407,7 @@ describe("SyncEventQueue", () => {
|
||||||
lastSeenUpdateId: 4
|
lastSeenUpdateId: 4
|
||||||
}, async () => { });
|
}, async () => { });
|
||||||
|
|
||||||
assert.strictEqual(queue.documentCount, 2);
|
assert.strictEqual(queue.syncedDocumentCount, 2);
|
||||||
assert.strictEqual(queue.getSettledDocumentByPath("a.md")?.documentId, "A");
|
assert.strictEqual(queue.getSettledDocumentByPath("a.md")?.documentId, "A");
|
||||||
assert.strictEqual(queue.getSettledDocumentByPath("b.md")?.documentId, "B");
|
assert.strictEqual(queue.getSettledDocumentByPath("b.md")?.documentId, "B");
|
||||||
assert.strictEqual(queue.lastSeenUpdateId, 5);
|
assert.strictEqual(queue.lastSeenUpdateId, 5);
|
||||||
|
|
@ -427,9 +427,9 @@ describe("SyncEventQueue", () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
// Pending create adds a path
|
// Pending create adds a path
|
||||||
queue.enqueue({ type: SyncEventType.Create, path: "c.md" });
|
queue.enqueue({ type: SyncEventType.LocalCreate, path: "c.md" });
|
||||||
// Pending delete removes a path
|
// Pending delete removes a path
|
||||||
queue.enqueue({ type: SyncEventType.Delete, path: "a.md" });
|
queue.enqueue({ type: SyncEventType.LocalDelete, path: "a.md" });
|
||||||
|
|
||||||
const paths = queue.trackedPaths();
|
const paths = queue.trackedPaths();
|
||||||
assert.deepStrictEqual(
|
assert.deepStrictEqual(
|
||||||
|
|
@ -441,10 +441,10 @@ describe("SyncEventQueue", () => {
|
||||||
it("trackedPaths handles create-delete-create for the same path", () => {
|
it("trackedPaths handles create-delete-create for the same path", () => {
|
||||||
const queue = createQueue();
|
const queue = createQueue();
|
||||||
|
|
||||||
queue.enqueue({ type: SyncEventType.Create, path: "a.md" });
|
queue.enqueue({ type: SyncEventType.LocalCreate, path: "a.md" });
|
||||||
// Delete gets promise documentId from pending Create
|
// Delete gets promise documentId from pending Create
|
||||||
queue.enqueue({ type: SyncEventType.Delete, path: "a.md" });
|
queue.enqueue({ type: SyncEventType.LocalDelete, path: "a.md" });
|
||||||
queue.enqueue({ type: SyncEventType.Create, path: "a.md" });
|
queue.enqueue({ type: SyncEventType.LocalCreate, path: "a.md" });
|
||||||
|
|
||||||
const paths = queue.trackedPaths();
|
const paths = queue.trackedPaths();
|
||||||
assert.ok(paths.has("a.md"));
|
assert.ok(paths.has("a.md"));
|
||||||
|
|
@ -453,10 +453,10 @@ describe("SyncEventQueue", () => {
|
||||||
it("trackedPaths applies moves for pending SyncLocal events", () => {
|
it("trackedPaths applies moves for pending SyncLocal events", () => {
|
||||||
const queue = createQueue();
|
const queue = createQueue();
|
||||||
|
|
||||||
queue.enqueue({ type: SyncEventType.Create, path: "a.md" });
|
queue.enqueue({ type: SyncEventType.LocalCreate, path: "a.md" });
|
||||||
|
|
||||||
// File was renamed from a.md to b.md
|
// File was renamed from a.md to b.md
|
||||||
queue.enqueue({ type: SyncEventType.SyncLocal, path: "b.md", oldPath: "a.md" });
|
queue.enqueue({ type: SyncEventType.LocalUpdate, path: "b.md", oldPath: "a.md" });
|
||||||
|
|
||||||
const paths = queue.trackedPaths();
|
const paths = queue.trackedPaths();
|
||||||
assert.ok(!paths.has("a.md"));
|
assert.ok(!paths.has("a.md"));
|
||||||
|
|
@ -466,10 +466,10 @@ describe("SyncEventQueue", () => {
|
||||||
it("trackedPaths tracks multiple moves for the same pending create", () => {
|
it("trackedPaths tracks multiple moves for the same pending create", () => {
|
||||||
const queue = createQueue();
|
const queue = createQueue();
|
||||||
|
|
||||||
queue.enqueue({ type: SyncEventType.Create, path: "a.md" });
|
queue.enqueue({ type: SyncEventType.LocalCreate, path: "a.md" });
|
||||||
|
|
||||||
queue.enqueue({ type: SyncEventType.SyncLocal, path: "b.md", oldPath: "a.md" });
|
queue.enqueue({ type: SyncEventType.LocalUpdate, path: "b.md", oldPath: "a.md" });
|
||||||
queue.enqueue({ type: SyncEventType.SyncLocal, path: "c.md", oldPath: "b.md" });
|
queue.enqueue({ type: SyncEventType.LocalUpdate, path: "c.md", oldPath: "b.md" });
|
||||||
|
|
||||||
const paths = queue.trackedPaths();
|
const paths = queue.trackedPaths();
|
||||||
assert.ok(!paths.has("a.md"));
|
assert.ok(!paths.has("a.md"));
|
||||||
|
|
@ -480,15 +480,15 @@ describe("SyncEventQueue", () => {
|
||||||
it("resolveCreate settles the document and replaces promise documentIds in the queue", async () => {
|
it("resolveCreate settles the document and replaces promise documentIds in the queue", async () => {
|
||||||
const queue = createQueue();
|
const queue = createQueue();
|
||||||
|
|
||||||
queue.enqueue({ type: SyncEventType.Create, path: "a.md" });
|
queue.enqueue({ type: SyncEventType.LocalCreate, path: "a.md" });
|
||||||
const createPromise = queue.getCreatePromise("a.md")!;
|
const createPromise = queue.getCreatePromise("a.md")!;
|
||||||
|
|
||||||
// Dependent events enqueued while create is still pending
|
// Dependent events enqueued while create is still pending
|
||||||
queue.enqueue({ type: SyncEventType.SyncLocal, path: "a.md" });
|
queue.enqueue({ type: SyncEventType.LocalUpdate, path: "a.md" });
|
||||||
queue.enqueue({ type: SyncEventType.Delete, path: "a.md" });
|
queue.enqueue({ type: SyncEventType.LocalDelete, path: "a.md" });
|
||||||
|
|
||||||
const event = await queue.next(); // dequeue the create
|
const event = await queue.next(); // dequeue the create
|
||||||
assert.ok(event?.type === SyncEventType.Create);
|
assert.ok(event?.type === SyncEventType.LocalCreate);
|
||||||
|
|
||||||
queue.resolveCreate(event, {
|
queue.resolveCreate(event, {
|
||||||
documentId: "DOC-1",
|
documentId: "DOC-1",
|
||||||
|
|
@ -506,7 +506,7 @@ describe("SyncEventQueue", () => {
|
||||||
// The SyncLocal + Delete for "DOC-1" coalesce: sync-local is
|
// The SyncLocal + Delete for "DOC-1" coalesce: sync-local is
|
||||||
// discarded and the delete is returned (standard coalescing).
|
// discarded and the delete is returned (standard coalescing).
|
||||||
const deleteEvt = await queue.next();
|
const deleteEvt = await queue.next();
|
||||||
assert.ok(deleteEvt?.type === SyncEventType.Delete);
|
assert.ok(deleteEvt?.type === SyncEventType.LocalDelete);
|
||||||
assert.strictEqual(deleteEvt.documentId, "DOC-1");
|
assert.strictEqual(deleteEvt.documentId, "DOC-1");
|
||||||
|
|
||||||
assert.strictEqual(await queue.next(), undefined);
|
assert.strictEqual(await queue.next(), undefined);
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
import type { Settings } from "../persistence/settings";
|
import type { Settings } from "../persistence/settings";
|
||||||
import type { Logger } from "../tracing/logger";
|
import type { Logger } from "../tracing/logger";
|
||||||
import { globsToRegexes } from "../utils/globs-to-regexes";
|
import { globsToRegexes } from "../utils/globs-to-regexes";
|
||||||
import { isConflictPath } from "../utils/conflict-path";
|
import { isConflictPath } from "./conflict-path";
|
||||||
import { removeFromArray } from "../utils/remove-from-array";
|
import { removeFromArray } from "../utils/remove-from-array";
|
||||||
import {
|
import {
|
||||||
SyncEventType,
|
SyncEventType,
|
||||||
|
|
@ -13,6 +13,10 @@ import {
|
||||||
type SyncEvent,
|
type SyncEvent,
|
||||||
type VaultUpdateId,
|
type VaultUpdateId,
|
||||||
} from "./types";
|
} from "./types";
|
||||||
|
import { sleep } from "../utils/sleep";
|
||||||
|
|
||||||
|
export const SAVE_RETRY_BASE_DELAY_MS = 50;
|
||||||
|
export const SAVE_RETRY_MAX_ATTEMPTS = 3;
|
||||||
|
|
||||||
export class SyncEventQueue {
|
export class SyncEventQueue {
|
||||||
// Latest state of the filesystem as we know it, excluding
|
// Latest state of the filesystem as we know it, excluding
|
||||||
|
|
@ -27,7 +31,7 @@ export class SyncEventQueue {
|
||||||
// can include multiple generations of the same document,
|
// can include multiple generations of the same document,
|
||||||
// e.g.: a create, delete, create sequence for the same path.
|
// e.g.: a create, delete, create sequence for the same path.
|
||||||
//
|
//
|
||||||
// The paths for the events must always correspond to the latest
|
// The paths within the events must always correspond to the latest
|
||||||
// path on disk, so the path of each event may be updated multiple
|
// path on disk, so the path of each event may be updated multiple
|
||||||
// times.
|
// times.
|
||||||
//
|
//
|
||||||
|
|
@ -37,6 +41,11 @@ export class SyncEventQueue {
|
||||||
// file creations for paths matching any of these patterns will be ignored
|
// file creations for paths matching any of these patterns will be ignored
|
||||||
private ignorePatterns: RegExp[];
|
private ignorePatterns: RegExp[];
|
||||||
|
|
||||||
|
private savePending = false;
|
||||||
|
|
||||||
|
|
||||||
|
private lastSeenUpdateId: VaultUpdateId;
|
||||||
|
|
||||||
public constructor(
|
public constructor(
|
||||||
private readonly settings: Settings,
|
private readonly settings: Settings,
|
||||||
private readonly logger: Logger,
|
private readonly logger: Logger,
|
||||||
|
|
@ -62,29 +71,19 @@ export class SyncEventQueue {
|
||||||
this.documents.set(relativePath, record);
|
this.documents.set(relativePath, record);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
this.lastSeenUpdateId = initialState.lastSeenUpdateId ?? -1;
|
||||||
|
|
||||||
this.logger.debug(`Loaded ${this.documents.size} documents`);
|
this.logger.debug(`Loaded ${this.documents.size} documents and lastSeenUpdateId=${this.lastSeenUpdateId} from storage`);
|
||||||
}
|
}
|
||||||
|
|
||||||
public get size(): number {
|
public get pendingUpdateCount(): number {
|
||||||
return this.events.length;
|
return this.events.length;
|
||||||
}
|
}
|
||||||
|
|
||||||
public get documentCount(): number {
|
public get syncedDocumentCount(): number {
|
||||||
return this.documents.size;
|
return this.documents.size;
|
||||||
}
|
}
|
||||||
|
|
||||||
public get lastSeenUpdateId(): VaultUpdateId {
|
|
||||||
let max = 0;
|
|
||||||
for (const record of this.documents.values()) {
|
|
||||||
if (record.parentVersionId > max) {
|
|
||||||
max = record.parentVersionId;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return max;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
// todo: let's remove
|
// todo: let's remove
|
||||||
public getSettledDocumentByPath(path: RelativePath): DocumentRecord | undefined {
|
public getSettledDocumentByPath(path: RelativePath): DocumentRecord | undefined {
|
||||||
return this.documents.get(path);
|
return this.documents.get(path);
|
||||||
|
|
@ -149,7 +148,7 @@ export class SyncEventQueue {
|
||||||
this.documents.set(newPath, record);
|
this.documents.set(newPath, record);
|
||||||
for (const e of this.events) {
|
for (const e of this.events) {
|
||||||
if (
|
if (
|
||||||
e.type === SyncEventType.SyncLocal &&
|
e.type === SyncEventType.LocalUpdate &&
|
||||||
e.documentId === record.documentId
|
e.documentId === record.documentId
|
||||||
) {
|
) {
|
||||||
e.path = newPath;
|
e.path = newPath;
|
||||||
|
|
@ -168,7 +167,7 @@ export class SyncEventQueue {
|
||||||
* Call once a create has been acknowledged by the server.
|
* Call once a create has been acknowledged by the server.
|
||||||
*/
|
*/
|
||||||
public resolveCreate(
|
public resolveCreate(
|
||||||
event: Extract<SyncEvent, { type: SyncEventType.Create }>,
|
event: Extract<SyncEvent, { type: SyncEventType.LocalCreate }>,
|
||||||
record: DocumentRecord
|
record: DocumentRecord
|
||||||
): void {
|
): void {
|
||||||
const promise = event.resolvers?.promise;
|
const promise = event.resolvers?.promise;
|
||||||
|
|
@ -179,7 +178,7 @@ export class SyncEventQueue {
|
||||||
if (promise !== undefined) {
|
if (promise !== undefined) {
|
||||||
for (const e of this.events) {
|
for (const e of this.events) {
|
||||||
if (
|
if (
|
||||||
(e.type === SyncEventType.SyncLocal || e.type === SyncEventType.Delete) &&
|
(e.type === SyncEventType.LocalUpdate || e.type === SyncEventType.LocalDelete) &&
|
||||||
e.documentId === promise
|
e.documentId === promise
|
||||||
) {
|
) {
|
||||||
(e as { documentId: DocumentId | Promise<DocumentId> }).documentId = record.documentId;
|
(e as { documentId: DocumentId | Promise<DocumentId> }).documentId = record.documentId;
|
||||||
|
|
@ -211,12 +210,12 @@ export class SyncEventQueue {
|
||||||
const pendingPaths = new Map<Promise<DocumentId>, RelativePath>();
|
const pendingPaths = new Map<Promise<DocumentId>, RelativePath>();
|
||||||
|
|
||||||
for (const event of this.events) {
|
for (const event of this.events) {
|
||||||
if (event.type === SyncEventType.Create) {
|
if (event.type === SyncEventType.LocalCreate) {
|
||||||
paths.add(event.path);
|
paths.add(event.path);
|
||||||
if (event.resolvers !== undefined) {
|
if (event.resolvers !== undefined) {
|
||||||
pendingPaths.set(event.resolvers.promise, event.path);
|
pendingPaths.set(event.resolvers.promise, event.path);
|
||||||
}
|
}
|
||||||
} else if (event.type === SyncEventType.Delete) {
|
} else if (event.type === SyncEventType.LocalDelete) {
|
||||||
if (typeof event.documentId === "string") {
|
if (typeof event.documentId === "string") {
|
||||||
const path = this.getDocumentByDocumentId(event.documentId)?.path;
|
const path = this.getDocumentByDocumentId(event.documentId)?.path;
|
||||||
if (path) {
|
if (path) {
|
||||||
|
|
@ -244,14 +243,16 @@ export class SyncEventQueue {
|
||||||
const docId = record.documentId;
|
const docId = record.documentId;
|
||||||
return this.events.some(
|
return this.events.some(
|
||||||
(e) =>
|
(e) =>
|
||||||
(e.type === SyncEventType.Create && e.path === path) ||
|
(e.type === SyncEventType.LocalCreate && e.path === path) ||
|
||||||
(e.type === SyncEventType.SyncLocal &&
|
(e.type === SyncEventType.LocalUpdate &&
|
||||||
e.documentId === docId) ||
|
e.documentId === docId) ||
|
||||||
(e.type === SyncEventType.Delete &&
|
(e.type === SyncEventType.LocalDelete &&
|
||||||
e.documentId === docId) ||
|
e.documentId === docId) ||
|
||||||
(e.type === SyncEventType.SyncRemote &&
|
(e.type === SyncEventType.RemoteUpdate &&
|
||||||
// we care about the local path not the remote
|
// we care about the local path not the remote
|
||||||
this.getDocumentByDocumentId(e.remoteVersion.documentId)?.path === path)
|
this.getDocumentByDocumentId(e.remoteVersion.documentId)?.path === path) ||
|
||||||
|
(e.type === SyncEventType.RemotePathChange &&
|
||||||
|
this.getDocumentByDocumentId(e.pathChange.documentId)?.path === path)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -279,7 +280,10 @@ export class SyncEventQueue {
|
||||||
}
|
}
|
||||||
|
|
||||||
public enqueue(input: FileSyncEvent): void {
|
public enqueue(input: FileSyncEvent): void {
|
||||||
if (input.type === SyncEventType.SyncRemote) {
|
if (
|
||||||
|
input.type === SyncEventType.RemoteUpdate ||
|
||||||
|
input.type === SyncEventType.RemotePathChange
|
||||||
|
) {
|
||||||
this.events.push(input);
|
this.events.push(input);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
@ -303,19 +307,19 @@ export class SyncEventQueue {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (input.type === SyncEventType.Create) {
|
if (input.type === SyncEventType.LocalCreate) {
|
||||||
this.events.push({ type: SyncEventType.Create, path, originalPath: path });
|
this.events.push({ type: SyncEventType.LocalCreate, path, originalPath: path });
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const lookupPath = (input.type === SyncEventType.SyncLocal && input.oldPath) ? input.oldPath : path;
|
const lookupPath = (input.type === SyncEventType.LocalUpdate && input.oldPath) ? input.oldPath : path;
|
||||||
const record = this.documents.get(lookupPath);
|
const record = this.documents.get(lookupPath);
|
||||||
const documentId: DocumentId | Promise<DocumentId> | undefined =
|
const documentId: DocumentId | Promise<DocumentId> | undefined =
|
||||||
record?.documentId ?? this.getCreatePromise(lookupPath);
|
record?.documentId ?? this.getCreatePromise(lookupPath);
|
||||||
if (documentId === undefined) return;
|
if (documentId === undefined) return;
|
||||||
|
|
||||||
if (input.type === SyncEventType.Delete) {
|
if (input.type === SyncEventType.LocalDelete) {
|
||||||
this.events.push({ type: SyncEventType.Delete, documentId });
|
this.events.push({ type: SyncEventType.LocalDelete, documentId });
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -324,7 +328,7 @@ export class SyncEventQueue {
|
||||||
this.documents.delete(input.oldPath);
|
this.documents.delete(input.oldPath);
|
||||||
this.documents.set(path, record!);
|
this.documents.set(path, record!);
|
||||||
for (const e of this.events) {
|
for (const e of this.events) {
|
||||||
if (e.type === SyncEventType.SyncLocal && e.documentId === documentId) {
|
if (e.type === SyncEventType.LocalUpdate && e.documentId === documentId) {
|
||||||
e.path = path;
|
e.path = path;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -333,7 +337,7 @@ export class SyncEventQueue {
|
||||||
this.updatePendingCreatePath(input.oldPath, path);
|
this.updatePendingCreatePath(input.oldPath, path);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
this.events.push({ type: SyncEventType.SyncLocal, documentId, path, originalPath: path });
|
this.events.push({ type: SyncEventType.LocalUpdate, documentId, path, originalPath: path });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -344,7 +348,7 @@ export class SyncEventQueue {
|
||||||
const [first] = this.events;
|
const [first] = this.events;
|
||||||
|
|
||||||
// Creates are always returned immediately (FIFO)
|
// Creates are always returned immediately (FIFO)
|
||||||
if (first.type === SyncEventType.Create) {
|
if (first.type === SyncEventType.LocalCreate) {
|
||||||
this.events.shift();
|
this.events.shift();
|
||||||
return first;
|
return first;
|
||||||
}
|
}
|
||||||
|
|
@ -355,7 +359,7 @@ export class SyncEventQueue {
|
||||||
// `Promise<DocumentId>` (the originating Create hasn't landed
|
// `Promise<DocumentId>` (the originating Create hasn't landed
|
||||||
// yet), awaiting it may reject — handle that: the Create was
|
// yet), awaiting it may reject — handle that: the Create was
|
||||||
// cancelled, so the Delete has nothing to delete, just drop it.
|
// cancelled, so the Delete has nothing to delete, just drop it.
|
||||||
if (first.type === SyncEventType.Delete) {
|
if (first.type === SyncEventType.LocalDelete) {
|
||||||
this.events.shift();
|
this.events.shift();
|
||||||
const { documentId } = first;
|
const { documentId } = first;
|
||||||
let resolvedId: DocumentId;
|
let resolvedId: DocumentId;
|
||||||
|
|
@ -371,14 +375,14 @@ export class SyncEventQueue {
|
||||||
return first;
|
return first;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (first.type === SyncEventType.SyncLocal) {
|
if (first.type === SyncEventType.LocalUpdate) {
|
||||||
const { documentId } = first;
|
const { documentId } = first;
|
||||||
|
|
||||||
// If there's a later delete for the same documentId, discard
|
// If there's a later delete for the same documentId, discard
|
||||||
// all sync-locals for that document and return the delete
|
// all sync-locals for that document and return the delete
|
||||||
const deleteEvent = this.events.find(
|
const deleteEvent = this.events.find(
|
||||||
(e) =>
|
(e) =>
|
||||||
e.type === SyncEventType.Delete &&
|
e.type === SyncEventType.LocalDelete &&
|
||||||
e.documentId === documentId
|
e.documentId === documentId
|
||||||
);
|
);
|
||||||
if (deleteEvent !== undefined) {
|
if (deleteEvent !== undefined) {
|
||||||
|
|
@ -399,7 +403,7 @@ export class SyncEventQueue {
|
||||||
// original path to the last one
|
// original path to the last one
|
||||||
const matching = this.events.filter(
|
const matching = this.events.filter(
|
||||||
(e) =>
|
(e) =>
|
||||||
e.type === SyncEventType.SyncLocal &&
|
e.type === SyncEventType.LocalUpdate &&
|
||||||
e.documentId === documentId &&
|
e.documentId === documentId &&
|
||||||
e.originalPath === first.originalPath // can't coalesce moves as they can depend on each other so we have to sync them in the same order, could do topological sort but let's keep it simple for now
|
e.originalPath === first.originalPath // can't coalesce moves as they can depend on each other so we have to sync them in the same order, could do topological sort but let's keep it simple for now
|
||||||
);
|
);
|
||||||
|
|
@ -410,12 +414,31 @@ export class SyncEventQueue {
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
// SyncRemote: coalesce multiple events for the same documentId to the last one
|
// Coalesce multiple events of the same remote kind for the same
|
||||||
const { documentId } = first.remoteVersion;
|
// documentId to the last one. Kinds are coalesced independently so
|
||||||
|
// that an interleaved content+path stream (e.g. VaultUpdate →
|
||||||
|
// PathChange) still preserves the VaultUpdate-before-PathChange
|
||||||
|
// ordering invariant the syncer relies on.
|
||||||
|
if (first.type === SyncEventType.RemoteUpdate) {
|
||||||
|
const { documentId } = first.remoteVersion;
|
||||||
|
const matching = this.events.filter(
|
||||||
|
(e) =>
|
||||||
|
e.type === SyncEventType.RemoteUpdate &&
|
||||||
|
e.remoteVersion.documentId === documentId
|
||||||
|
);
|
||||||
|
const result = matching[matching.length - 1];
|
||||||
|
for (const item of matching) {
|
||||||
|
removeFromArray(this.events, item);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
// SyncRemotePath
|
||||||
|
const { documentId } = first.pathChange;
|
||||||
const matching = this.events.filter(
|
const matching = this.events.filter(
|
||||||
(e) =>
|
(e) =>
|
||||||
e.type === SyncEventType.SyncRemote &&
|
e.type === SyncEventType.RemotePathChange &&
|
||||||
e.remoteVersion.documentId === documentId
|
e.pathChange.documentId === documentId
|
||||||
);
|
);
|
||||||
const result = matching[matching.length - 1];
|
const result = matching[matching.length - 1];
|
||||||
for (const item of matching) {
|
for (const item of matching) {
|
||||||
|
|
@ -436,11 +459,13 @@ export class SyncEventQueue {
|
||||||
for (let i = this.events.length - 1; i >= 0; i--) {
|
for (let i = this.events.length - 1; i >= 0; i--) {
|
||||||
const e = this.events[i];
|
const e = this.events[i];
|
||||||
if (
|
if (
|
||||||
(e.type === SyncEventType.SyncLocal &&
|
(e.type === SyncEventType.LocalUpdate &&
|
||||||
e.documentId === documentId) ||
|
e.documentId === documentId) ||
|
||||||
(e.type === SyncEventType.SyncRemote &&
|
(e.type === SyncEventType.RemoteUpdate &&
|
||||||
e.remoteVersion.documentId === documentId) ||
|
e.remoteVersion.documentId === documentId) ||
|
||||||
(e.type === SyncEventType.Delete &&
|
(e.type === SyncEventType.RemotePathChange &&
|
||||||
|
e.pathChange.documentId === documentId) ||
|
||||||
|
(e.type === SyncEventType.LocalDelete &&
|
||||||
e.documentId === documentId)
|
e.documentId === documentId)
|
||||||
) {
|
) {
|
||||||
// eslint-disable-next-line no-restricted-syntax -- Bulk removal by predicate, not single-item removal
|
// eslint-disable-next-line no-restricted-syntax -- Bulk removal by predicate, not single-item removal
|
||||||
|
|
@ -462,7 +487,7 @@ export class SyncEventQueue {
|
||||||
if (promise !== undefined) {
|
if (promise !== undefined) {
|
||||||
for (const e of this.events) {
|
for (const e of this.events) {
|
||||||
if (
|
if (
|
||||||
e.type === SyncEventType.SyncLocal &&
|
e.type === SyncEventType.LocalUpdate &&
|
||||||
e.documentId === promise
|
e.documentId === promise
|
||||||
) {
|
) {
|
||||||
e.path = newPath;
|
e.path = newPath;
|
||||||
|
|
@ -477,7 +502,7 @@ export class SyncEventQueue {
|
||||||
for (let i = this.events.length - 1; i >= 0; i--) {
|
for (let i = this.events.length - 1; i >= 0; i--) {
|
||||||
const e = this.events[i];
|
const e = this.events[i];
|
||||||
if (
|
if (
|
||||||
e.type === SyncEventType.Create &&
|
e.type === SyncEventType.LocalCreate &&
|
||||||
e.resolvers?.promise === promise
|
e.resolvers?.promise === promise
|
||||||
) {
|
) {
|
||||||
return e.path;
|
return e.path;
|
||||||
|
|
@ -488,10 +513,10 @@ export class SyncEventQueue {
|
||||||
|
|
||||||
private findLastCreate(
|
private findLastCreate(
|
||||||
path: RelativePath
|
path: RelativePath
|
||||||
): Extract<SyncEvent, { type: SyncEventType.Create }> | undefined {
|
): Extract<SyncEvent, { type: SyncEventType.LocalCreate }> | undefined {
|
||||||
for (let i = this.events.length - 1; i >= 0; i--) {
|
for (let i = this.events.length - 1; i >= 0; i--) {
|
||||||
const e = this.events[i];
|
const e = this.events[i];
|
||||||
if (e.type === SyncEventType.Create && e.path === path) {
|
if (e.type === SyncEventType.LocalCreate && e.path === path) {
|
||||||
return e;
|
return e;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -535,52 +560,23 @@ export class SyncEventQueue {
|
||||||
|
|
||||||
private rejectAllPendingCreates(): void {
|
private rejectAllPendingCreates(): void {
|
||||||
for (const event of this.events) {
|
for (const event of this.events) {
|
||||||
if (event.type === SyncEventType.Create && event.resolvers !== undefined) {
|
if (event.type === SyncEventType.LocalCreate && event.resolvers !== undefined) {
|
||||||
event.resolvers.promise.catch(() => { /* suppressed — consumer may not be listening */ });
|
event.resolvers.promise.catch(() => { /* suppressed — consumer may not be listening */ });
|
||||||
event.resolvers.reject(new Error("Create was cancelled"));
|
event.resolvers.reject(new Error("Create was cancelled"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private savePending = false;
|
|
||||||
|
|
||||||
// Coalesce bursts of mutations into one persist per microtask. A drain
|
// Coalesce bursts of mutations into one persist per microtask. A drain
|
||||||
// iteration can easily produce 10+ mutations; without this, we'd fire
|
// iteration can easily produce 10+ mutations; without this, we'd fire
|
||||||
// 10 overlapping `save()` calls racing on the persistence backend.
|
// 10 overlapping `save()` calls racing on the persistence backend.
|
||||||
//
|
|
||||||
// On failure, retry with bounded exponential backoff instead of
|
|
||||||
// silently dropping the write — otherwise a transient IDB/fs error
|
|
||||||
// leaves the in-memory state permanently diverged from persisted state
|
|
||||||
// and the user loses queue progress on restart.
|
|
||||||
private saveInTheBackground(): void {
|
private saveInTheBackground(): void {
|
||||||
if (this.savePending) return;
|
if (this.savePending) return;
|
||||||
this.savePending = true;
|
this.savePending = true;
|
||||||
queueMicrotask(() => {
|
queueMicrotask(() => {
|
||||||
this.savePending = false;
|
this.savePending = false;
|
||||||
void this.saveWithRetry();
|
this.save();
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private async saveWithRetry(): Promise<void> {
|
|
||||||
const maxAttempts = 3;
|
|
||||||
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
||||||
try {
|
|
||||||
await this.save();
|
|
||||||
return;
|
|
||||||
} catch (error) {
|
|
||||||
if (attempt === maxAttempts) {
|
|
||||||
this.logger.error(
|
|
||||||
`Error saving sync state after ${maxAttempts} attempts: ${error}`
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
this.logger.warn(
|
|
||||||
`Error saving sync state (attempt ${attempt}/${maxAttempts}): ${error}; retrying`
|
|
||||||
);
|
|
||||||
await new Promise((resolve) =>
|
|
||||||
setTimeout(resolve, 50 * attempt)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -67,27 +67,6 @@ export class Syncer {
|
||||||
this.webSocketManager.onWebSocketStatusChanged.add((isConnected) => {
|
this.webSocketManager.onWebSocketStatusChanged.add((isConnected) => {
|
||||||
if (isConnected) {
|
if (isConnected) {
|
||||||
this.sendHandshakeMessage();
|
this.sendHandshakeMessage();
|
||||||
} else {
|
|
||||||
// Don't null the reference synchronously — if the scan is
|
|
||||||
// still in flight, the next reconnect would spawn a second
|
|
||||||
// concurrent scan racing on the same queue. Defer the
|
|
||||||
// clear until the in-flight task actually resolves, so a
|
|
||||||
// fresh scan can only start once the prior one is done.
|
|
||||||
const current = this.runningScheduleSyncForOfflineChanges;
|
|
||||||
if (current === undefined) return;
|
|
||||||
current
|
|
||||||
.catch(() => {
|
|
||||||
/* swallow — internal error already logged */
|
|
||||||
})
|
|
||||||
.finally(() => {
|
|
||||||
if (
|
|
||||||
this.runningScheduleSyncForOfflineChanges ===
|
|
||||||
current
|
|
||||||
) {
|
|
||||||
this.runningScheduleSyncForOfflineChanges =
|
|
||||||
undefined;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
this.webSocketManager.onRemoteVaultUpdateReceived.add(
|
this.webSocketManager.onRemoteVaultUpdateReceived.add(
|
||||||
|
|
@ -102,20 +81,8 @@ export class Syncer {
|
||||||
return this._isFirstSyncComplete;
|
return this._isFirstSyncComplete;
|
||||||
}
|
}
|
||||||
|
|
||||||
public hasPendingOperationsForDocument(relativePath: string): boolean {
|
|
||||||
return this.queue.hasPendingEventsForPath(relativePath);
|
|
||||||
}
|
|
||||||
|
|
||||||
public syncLocallyCreatedFile(relativePath: RelativePath): void {
|
public syncLocallyCreatedFile(relativePath: RelativePath): void {
|
||||||
this.queue.enqueue({ type: SyncEventType.Create, path: relativePath });
|
this.queue.enqueue({ type: SyncEventType.LocalCreate, path: relativePath });
|
||||||
this.ensureDraining();
|
|
||||||
}
|
|
||||||
|
|
||||||
public syncLocallyDeletedFile(relativePath: RelativePath): void {
|
|
||||||
this.queue.enqueue({
|
|
||||||
type: SyncEventType.Delete,
|
|
||||||
path: relativePath,
|
|
||||||
});
|
|
||||||
this.ensureDraining();
|
this.ensureDraining();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -126,10 +93,78 @@ export class Syncer {
|
||||||
oldPath?: RelativePath;
|
oldPath?: RelativePath;
|
||||||
relativePath: RelativePath;
|
relativePath: RelativePath;
|
||||||
}): void {
|
}): void {
|
||||||
this.queue.enqueue({ type: SyncEventType.SyncLocal, path: relativePath, oldPath });
|
this.queue.enqueue({ type: SyncEventType.LocalUpdate, path: relativePath, oldPath });
|
||||||
this.ensureDraining();
|
this.ensureDraining();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public syncLocallyDeletedFile(relativePath: RelativePath): void {
|
||||||
|
this.queue.enqueue({
|
||||||
|
type: SyncEventType.LocalDelete,
|
||||||
|
path: relativePath,
|
||||||
|
});
|
||||||
|
this.ensureDraining();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public async syncRemotelyUpdatedFile(
|
||||||
|
message: WebSocketVaultUpdate
|
||||||
|
): Promise<void> {
|
||||||
|
await this.scheduleSyncForOfflineChanges();
|
||||||
|
|
||||||
|
for (const remoteVersion of message.documents) {
|
||||||
|
this.queue.enqueue({
|
||||||
|
type: SyncEventType.RemoteUpdate,
|
||||||
|
remoteVersion
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (message.isInitialSync) {
|
||||||
|
this._isFirstSyncComplete = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.ensureDraining();
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// A PathChange notifies us that a document now lives at a new server-
|
||||||
|
// canonical path. It's delivered to every client (origin included)
|
||||||
|
// because the create/update HTTP response no longer carries the path,
|
||||||
|
// so the only way the origin learns about dedupe or first-rename-wins
|
||||||
|
// is via this event.
|
||||||
|
//
|
||||||
|
// Algorithmic assumptions:
|
||||||
|
// (1) Per-vault broadcast ordering is preserved by the server, so if
|
||||||
|
// the same write produced a `VaultUpdate` (content change) and a
|
||||||
|
// `PathChange` (path change), the `VaultUpdate` is handled first
|
||||||
|
// — that's what lets us skip advancing `parentVersionId` here
|
||||||
|
// without risking a stuck "already up-to-date" check later.
|
||||||
|
// (2) On a lag-induced disconnect (`broadcast::error::Lagged`) the
|
||||||
|
// server disconnects the client for a full resync, so out-of-
|
||||||
|
// order delivery across a reconnect boundary can't leave us with
|
||||||
|
// a stale PathChange overwriting a newer one.
|
||||||
|
public async syncRemotelyChangedPath(
|
||||||
|
pathChange: WebSocketVaultPathChange
|
||||||
|
): Promise<void> {
|
||||||
|
try {
|
||||||
|
await this.scheduleSyncForOfflineChanges();
|
||||||
|
|
||||||
|
this.queue.enqueue({
|
||||||
|
type: SyncEventType.RemotePathChange,
|
||||||
|
pathChange
|
||||||
|
});
|
||||||
|
|
||||||
|
await this.scheduleDrain();
|
||||||
|
} catch (e) {
|
||||||
|
if (e instanceof SyncResetError) {
|
||||||
|
this.logger.info(
|
||||||
|
"Failed to apply remote path change due to a reset"
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.logger.error(`Failed to apply remote path change: ${e}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public async scheduleSyncForOfflineChanges(): Promise<void> {
|
public async scheduleSyncForOfflineChanges(): Promise<void> {
|
||||||
if (this.runningScheduleSyncForOfflineChanges !== undefined) {
|
if (this.runningScheduleSyncForOfflineChanges !== undefined) {
|
||||||
this.logger.debug("Uploading local changes is already in progress");
|
this.logger.debug("Uploading local changes is already in progress");
|
||||||
|
|
@ -167,114 +202,27 @@ export class Syncer {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public async syncRemotelyUpdatedFile(
|
|
||||||
message: WebSocketVaultUpdate
|
|
||||||
): Promise<void> {
|
|
||||||
try {
|
|
||||||
await this.scheduleSyncForOfflineChanges();
|
|
||||||
|
|
||||||
for (const remoteVersion of message.documents) {
|
|
||||||
this.queue.enqueue({
|
|
||||||
type: SyncEventType.SyncRemote,
|
|
||||||
remoteVersion
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
if (message.isInitialSync) {
|
|
||||||
this._isFirstSyncComplete = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
await this.scheduleDrain();
|
|
||||||
} catch (e) {
|
|
||||||
if (e instanceof SyncResetError) {
|
|
||||||
this.logger.info(
|
|
||||||
"Failed to sync remotely updated file due to a reset"
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
this.logger.error(`Failed to sync remotely updated file: ${e}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A PathChange notifies us that a document now lives at a new server-
|
|
||||||
// canonical path. It's delivered to every client (origin included)
|
|
||||||
// because the create/update HTTP response no longer carries the path,
|
|
||||||
// so the only way the origin learns about dedupe or first-rename-wins
|
|
||||||
// is via this event.
|
|
||||||
//
|
|
||||||
// Algorithmic assumptions:
|
|
||||||
// (1) Per-vault broadcast ordering is preserved by the server, so if
|
|
||||||
// the same write produced a `VaultUpdate` (content change) and a
|
|
||||||
// `PathChange` (path change), the `VaultUpdate` is handled first
|
|
||||||
// — that's what lets us skip advancing `parentVersionId` here
|
|
||||||
// without risking a stuck "already up-to-date" check later.
|
|
||||||
// (2) On a lag-induced disconnect (`broadcast::error::Lagged`) the
|
|
||||||
// server disconnects the client for a full resync, so out-of-
|
|
||||||
// order delivery across a reconnect boundary can't leave us with
|
|
||||||
// a stale PathChange overwriting a newer one.
|
|
||||||
public async syncRemotelyChangedPath(
|
|
||||||
pathChange: WebSocketVaultPathChange
|
|
||||||
): Promise<void> {
|
|
||||||
// Serialize onto the drain chain so this handler can't race against
|
|
||||||
// an in-flight `processSyncRemote` / `processSyncLocal` etc. that
|
|
||||||
// captured the old path before our move.
|
|
||||||
try {
|
|
||||||
await this.chainOntoDrain(async () => {
|
|
||||||
const existing = this.queue.getDocumentByDocumentId(
|
|
||||||
pathChange.documentId
|
|
||||||
);
|
|
||||||
if (existing === undefined) {
|
|
||||||
throw new Error(
|
|
||||||
`Received path change for unknown document ${pathChange.documentId}`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const { path: currentPath, record } = existing;
|
|
||||||
const newPath = pathChange.relativePath;
|
|
||||||
|
|
||||||
if (currentPath !== newPath) {
|
|
||||||
await this.operations.move(currentPath, newPath);
|
|
||||||
|
|
||||||
this.history.addHistoryEntry({
|
|
||||||
status: SyncStatus.SUCCESS,
|
|
||||||
details: {
|
|
||||||
type: SyncType.MOVE,
|
|
||||||
relativePath: newPath,
|
|
||||||
movedFrom: currentPath
|
|
||||||
},
|
|
||||||
message: "Applied remote path change",
|
|
||||||
author: pathChange.userId,
|
|
||||||
timestamp: new Date(pathChange.updatedDate)
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// `operations.move` updates the queue's path index, but
|
|
||||||
// doesn't touch `remoteRelativePath`. Refresh it so offline
|
|
||||||
// change detection compares against the server's path.
|
|
||||||
// parentVersionId intentionally stays at its prior value:
|
|
||||||
// if the write also changed content, the corresponding
|
|
||||||
// VaultUpdate handles that; advancing it here would make us
|
|
||||||
// skip fetching content we don't yet have.
|
|
||||||
this.queue.setDocument(newPath, {
|
|
||||||
...record,
|
|
||||||
remoteRelativePath: newPath
|
|
||||||
});
|
|
||||||
});
|
|
||||||
} catch (e) {
|
|
||||||
if (e instanceof SyncResetError) {
|
|
||||||
this.logger.info(
|
|
||||||
"Failed to apply remote path change due to a reset"
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
this.logger.error(`Failed to apply remote path change: ${e}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public reset(): void {
|
public reset(): void {
|
||||||
this._isFirstSyncComplete = false;
|
this._isFirstSyncComplete = false;
|
||||||
this.queue.clear();
|
this.queue.clear();
|
||||||
this.runningScheduleSyncForOfflineChanges = undefined;
|
// Don't null the reference synchronously — if the scan is
|
||||||
|
// still in flight, the next reconnect would spawn a second
|
||||||
|
// concurrent scan racing on the same queue. Defer the
|
||||||
|
// clear until the in-flight task actually resolves, so a
|
||||||
|
// fresh scan can only start once the prior one is done.
|
||||||
|
const current = this.runningScheduleSyncForOfflineChanges;
|
||||||
|
if (current !== undefined) {
|
||||||
|
current.finally(() => {
|
||||||
|
if (
|
||||||
|
this.runningScheduleSyncForOfflineChanges ===
|
||||||
|
current
|
||||||
|
) {
|
||||||
|
this.runningScheduleSyncForOfflineChanges =
|
||||||
|
undefined;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
// Do not set this.draining = undefined — the in-flight drain will
|
// Do not set this.draining = undefined — the in-flight drain will
|
||||||
// exit naturally (SyncResetError or empty queue) and the promise
|
// exit naturally (SyncResetError or empty queue) and the promise
|
||||||
// chain stays intact, preventing concurrent drain invocations
|
// chain stays intact, preventing concurrent drain invocations
|
||||||
|
|
@ -372,17 +320,20 @@ export class Syncer {
|
||||||
|
|
||||||
try {
|
try {
|
||||||
switch (event.type) {
|
switch (event.type) {
|
||||||
case SyncEventType.Create:
|
case SyncEventType.LocalCreate:
|
||||||
await this.processCreate(event);
|
await this.processCreate(event);
|
||||||
break;
|
break;
|
||||||
case SyncEventType.Delete:
|
case SyncEventType.LocalDelete:
|
||||||
await this.processDelete(event);
|
await this.processDelete(event);
|
||||||
break;
|
break;
|
||||||
case SyncEventType.SyncLocal:
|
case SyncEventType.LocalUpdate:
|
||||||
await this.processSyncLocal(event);
|
await this.processSyncLocal(event);
|
||||||
break;
|
break;
|
||||||
case SyncEventType.SyncRemote:
|
case SyncEventType.RemoteUpdate:
|
||||||
await this.processSyncRemote(event);
|
await this.processSyncRemoteContent(event);
|
||||||
|
break;
|
||||||
|
case SyncEventType.RemotePathChange:
|
||||||
|
await this.processSyncRemotePath(event);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
|
|
@ -390,7 +341,7 @@ export class Syncer {
|
||||||
this.logger.info(
|
this.logger.info(
|
||||||
`Skipping sync event '${event.type}' because the file no longer exists`
|
`Skipping sync event '${event.type}' because the file no longer exists`
|
||||||
);
|
);
|
||||||
if (event.type === SyncEventType.Create) {
|
if (event.type === SyncEventType.LocalCreate) {
|
||||||
event.resolvers?.promise.catch(() => { });
|
event.resolvers?.promise.catch(() => { });
|
||||||
event.resolvers?.reject(new Error("Create was cancelled"));
|
event.resolvers?.reject(new Error("Create was cancelled"));
|
||||||
}
|
}
|
||||||
|
|
@ -404,7 +355,7 @@ export class Syncer {
|
||||||
// `processEvent` ran; if it was a Create, its resolver
|
// `processEvent` ran; if it was a Create, its resolver
|
||||||
// promise would otherwise hang forever, blocking any
|
// promise would otherwise hang forever, blocking any
|
||||||
// queued Delete / SyncLocal that `await`s it.
|
// queued Delete / SyncLocal that `await`s it.
|
||||||
if (event.type === SyncEventType.Create) {
|
if (event.type === SyncEventType.LocalCreate) {
|
||||||
event.resolvers?.promise.catch(() => {
|
event.resolvers?.promise.catch(() => {
|
||||||
/* suppressed */
|
/* suppressed */
|
||||||
});
|
});
|
||||||
|
|
@ -423,7 +374,7 @@ export class Syncer {
|
||||||
|
|
||||||
|
|
||||||
private async processCreate(
|
private async processCreate(
|
||||||
event: Extract<SyncEvent, { type: SyncEventType.Create }>
|
event: Extract<SyncEvent, { type: SyncEventType.LocalCreate }>
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const effectivePath = event.path;
|
const effectivePath = event.path;
|
||||||
const contentBytes = await this.operations.read(effectivePath);
|
const contentBytes = await this.operations.read(effectivePath);
|
||||||
|
|
@ -487,7 +438,7 @@ export class Syncer {
|
||||||
}
|
}
|
||||||
|
|
||||||
private async processDelete(
|
private async processDelete(
|
||||||
event: Extract<SyncEvent, { type: SyncEventType.Delete }>
|
event: Extract<SyncEvent, { type: SyncEventType.LocalDelete }>
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
let documentId: DocumentId;
|
let documentId: DocumentId;
|
||||||
if (typeof event.documentId === "string") {
|
if (typeof event.documentId === "string") {
|
||||||
|
|
@ -531,7 +482,7 @@ export class Syncer {
|
||||||
}
|
}
|
||||||
|
|
||||||
private async processSyncLocal(
|
private async processSyncLocal(
|
||||||
event: Extract<SyncEvent, { type: SyncEventType.SyncLocal }>
|
event: Extract<SyncEvent, { type: SyncEventType.LocalUpdate }>
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
let documentId: DocumentId;
|
let documentId: DocumentId;
|
||||||
if (typeof event.documentId === "string") {
|
if (typeof event.documentId === "string") {
|
||||||
|
|
@ -606,8 +557,8 @@ export class Syncer {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private async processSyncRemote(
|
private async processSyncRemoteContent(
|
||||||
event: Extract<SyncEvent, { type: SyncEventType.SyncRemote }>
|
event: Extract<SyncEvent, { type: SyncEventType.RemoteUpdate }>
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const { remoteVersion } = event;
|
const { remoteVersion } = event;
|
||||||
const existingDoc = this.queue.getDocumentByDocumentId(
|
const existingDoc = this.queue.getDocumentByDocumentId(
|
||||||
|
|
@ -643,6 +594,51 @@ export class Syncer {
|
||||||
await this.processRemoteUpdateForNewDocument(remoteVersion);
|
await this.processRemoteUpdateForNewDocument(remoteVersion);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async processSyncRemotePath(
|
||||||
|
event: Extract<SyncEvent, { type: SyncEventType.RemotePathChange }>
|
||||||
|
): Promise<void> {
|
||||||
|
const { pathChange } = event;
|
||||||
|
const existing = this.queue.getDocumentByDocumentId(
|
||||||
|
pathChange.documentId
|
||||||
|
);
|
||||||
|
if (existing === undefined) {
|
||||||
|
throw new Error(
|
||||||
|
`Received path change for unknown document ${pathChange.documentId}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const { path: currentPath, record } = existing;
|
||||||
|
const newPath = pathChange.relativePath;
|
||||||
|
|
||||||
|
if (currentPath !== newPath) {
|
||||||
|
await this.operations.move(currentPath, newPath);
|
||||||
|
|
||||||
|
this.history.addHistoryEntry({
|
||||||
|
status: SyncStatus.SUCCESS,
|
||||||
|
details: {
|
||||||
|
type: SyncType.MOVE,
|
||||||
|
relativePath: newPath,
|
||||||
|
movedFrom: currentPath
|
||||||
|
},
|
||||||
|
message: "Applied remote path change",
|
||||||
|
author: pathChange.userId,
|
||||||
|
timestamp: new Date(pathChange.updatedDate)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// `operations.move` updates the queue's path index, but doesn't
|
||||||
|
// touch `remoteRelativePath`. Refresh it so offline change
|
||||||
|
// detection compares against the server's path. parentVersionId
|
||||||
|
// intentionally stays at its prior value: if the write also
|
||||||
|
// changed content, the corresponding VaultUpdate handles that;
|
||||||
|
// advancing it here would make us skip fetching content we don't
|
||||||
|
// yet have.
|
||||||
|
this.queue.setDocument(newPath, {
|
||||||
|
...record,
|
||||||
|
remoteRelativePath: newPath
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
private async processRemoteUpdateForExistingDocument(
|
private async processRemoteUpdateForExistingDocument(
|
||||||
currentPath: RelativePath,
|
currentPath: RelativePath,
|
||||||
record: DocumentRecord,
|
record: DocumentRecord,
|
||||||
|
|
@ -793,14 +789,14 @@ export class Syncer {
|
||||||
details:
|
details:
|
||||||
targetPath !== currentPath
|
targetPath !== currentPath
|
||||||
? {
|
? {
|
||||||
type: SyncType.MOVE,
|
type: SyncType.MOVE,
|
||||||
relativePath: targetPath,
|
relativePath: targetPath,
|
||||||
movedFrom: currentPath
|
movedFrom: currentPath
|
||||||
}
|
}
|
||||||
: {
|
: {
|
||||||
type: SyncType.UPDATE,
|
type: SyncType.UPDATE,
|
||||||
relativePath: targetPath
|
relativePath: targetPath
|
||||||
},
|
},
|
||||||
message:
|
message:
|
||||||
"Successfully downloaded remotely updated file from the server",
|
"Successfully downloaded remotely updated file from the server",
|
||||||
author: fullVersion.userId,
|
author: fullVersion.userId,
|
||||||
|
|
@ -1063,7 +1059,7 @@ export class Syncer {
|
||||||
// `resolvers` promise can be fulfilled (or rejected, on a deleted
|
// `resolvers` promise can be fulfilled (or rejected, on a deleted
|
||||||
// response). Dependent SyncLocal/Delete events are chained through
|
// response). Dependent SyncLocal/Delete events are chained through
|
||||||
// that promise and would otherwise `await` forever.
|
// that promise and would otherwise `await` forever.
|
||||||
createEvent?: Extract<SyncEvent, { type: SyncEventType.Create }>;
|
createEvent?: Extract<SyncEvent, { type: SyncEventType.LocalCreate }>;
|
||||||
}): Promise<void> {
|
}): Promise<void> {
|
||||||
if (response.isDeleted) {
|
if (response.isDeleted) {
|
||||||
// A Create that the server returned as already-deleted means
|
// A Create that the server returned as already-deleted means
|
||||||
|
|
@ -1222,7 +1218,7 @@ export class Syncer {
|
||||||
}
|
}
|
||||||
|
|
||||||
private notifyRemainingOperationsChanged(): void {
|
private notifyRemainingOperationsChanged(): void {
|
||||||
const currentCount = this.queue.size;
|
const currentCount = this.queue.pendingUpdateCount;
|
||||||
if (this.previousRemainingOperationsCount !== currentCount) {
|
if (this.previousRemainingOperationsCount !== currentCount) {
|
||||||
this.previousRemainingOperationsCount = currentCount;
|
this.previousRemainingOperationsCount = currentCount;
|
||||||
this.onRemainingOperationsCountChanged.trigger(currentCount);
|
this.onRemainingOperationsCountChanged.trigger(currentCount);
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent";
|
import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent";
|
||||||
|
import type { WebSocketVaultPathChange } from "../services/types/WebSocketVaultPathChange";
|
||||||
|
|
||||||
export type VaultUpdateId = number;
|
export type VaultUpdateId = number;
|
||||||
export type DocumentId = string;
|
export type DocumentId = string;
|
||||||
|
|
@ -21,36 +22,43 @@ export interface StoredSyncState {
|
||||||
}
|
}
|
||||||
|
|
||||||
export enum SyncEventType {
|
export enum SyncEventType {
|
||||||
Create = "create",
|
LocalCreate = "local-create",
|
||||||
SyncLocal = "sync-local",
|
LocalUpdate = "local-update", // includes both content and path changes
|
||||||
Delete = "delete",
|
LocalDelete = "local-delete",
|
||||||
SyncRemote = "sync-remote",
|
RemoteUpdate = "remote-update",
|
||||||
|
RemotePathChange = "remote-path-change",
|
||||||
}
|
}
|
||||||
|
|
||||||
export type FileSyncEvent =
|
export type FileSyncEvent =
|
||||||
| { type: SyncEventType.Create; path: RelativePath }
|
| { type: SyncEventType.LocalCreate; path: RelativePath }
|
||||||
| { type: SyncEventType.SyncLocal; path: RelativePath; oldPath?: RelativePath }
|
| { type: SyncEventType.LocalUpdate; path: RelativePath; oldPath?: RelativePath }
|
||||||
| { type: SyncEventType.Delete; path: RelativePath }
|
| { type: SyncEventType.LocalDelete; path: RelativePath }
|
||||||
| { type: SyncEventType.SyncRemote; remoteVersion: DocumentVersionWithoutContent };
|
| { type: SyncEventType.RemoteUpdate; remoteVersion: DocumentVersionWithoutContent }
|
||||||
|
| { type: SyncEventType.RemotePathChange; pathChange: WebSocketVaultPathChange };
|
||||||
|
|
||||||
export type SyncEvent =
|
export type SyncEvent =
|
||||||
| {
|
| {
|
||||||
type: SyncEventType.Create;
|
type: SyncEventType.LocalCreate;
|
||||||
path: RelativePath; // current path on disk
|
path: RelativePath; // current path on disk
|
||||||
originalPath: RelativePath; // original path on disk when the event was created
|
originalPath: RelativePath; // original path on disk when the event was queued
|
||||||
resolvers?: PromiseWithResolvers<DocumentId>
|
resolvers?: PromiseWithResolvers<DocumentId>
|
||||||
}
|
}
|
||||||
| {
|
| {
|
||||||
type: SyncEventType.SyncLocal;
|
type: SyncEventType.LocalUpdate;
|
||||||
documentId: DocumentId | Promise<DocumentId>; // if it's a promise, the promise is fulfilled once the document's create event is processed
|
documentId: DocumentId | Promise<DocumentId>; // if it's a promise, the promise is fulfilled once the document's create event is processed
|
||||||
path: RelativePath; // current path on disk
|
path: RelativePath; // current path on disk
|
||||||
originalPath: RelativePath; // original path on disk when the event was created
|
originalPath: RelativePath; // original path on disk when the event was queued
|
||||||
|
// no need to store the old path in case of a rename; the server will figure it out from the parent's path
|
||||||
}
|
}
|
||||||
| {
|
| {
|
||||||
type: SyncEventType.Delete;
|
type: SyncEventType.LocalDelete;
|
||||||
documentId: DocumentId | Promise<DocumentId>; // if it's a promise, the promise is fulfilled once the document's create event is processed
|
documentId: DocumentId | Promise<DocumentId>; // if it's a promise, the promise is fulfilled once the document's create event is processed
|
||||||
}
|
}
|
||||||
| {
|
| {
|
||||||
type: SyncEventType.SyncRemote;
|
type: SyncEventType.RemoteUpdate;
|
||||||
remoteVersion: DocumentVersionWithoutContent;
|
remoteVersion: DocumentVersionWithoutContent;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
type: SyncEventType.RemotePathChange;
|
||||||
|
pathChange: WebSocketVaultPathChange;
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -58,6 +58,7 @@ pub struct CursorPositionFromServer {
|
||||||
pub clients: Vec<ClientCursors>,
|
pub clients: Vec<ClientCursors>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Clients only get notified of other clients' updates through WebSocketVaultUpdate.
|
||||||
#[derive(TS, Serialize, Clone, Debug)]
|
#[derive(TS, Serialize, Clone, Debug)]
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
pub struct WebSocketVaultUpdate {
|
pub struct WebSocketVaultUpdate {
|
||||||
|
|
@ -65,6 +66,8 @@ pub struct WebSocketVaultUpdate {
|
||||||
pub is_initial_sync: bool,
|
pub is_initial_sync: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Clients get notified of both their own and other clients' path changes through WebSocketVaultPathChange.
|
||||||
|
// This is becuase we must absolutely order path updates as they may all depend on all previous updates.
|
||||||
#[derive(TS, Serialize, Clone, Debug)]
|
#[derive(TS, Serialize, Clone, Debug)]
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
pub struct WebSocketVaultPathChange {
|
pub struct WebSocketVaultPathChange {
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue