andras/vault-link
andras/vault-link
1
0
Fork 0
Code Issues Pull requests 19 Projects Releases 55 Packages Wiki Activity Actions 26

Working setup

Browse source
This commit is contained in:
Andras Schmelczer 2026-01-12 21:24:05 +00:00
commit 2dfb8b71e5
16 changed files with 460 additions and 319 deletions
Download patch file Download diff file Expand all files Collapse all files

134
frontend/sync-client/src/sync-operations/syncer.ts
View file

@ -42,7 +42,7 @@ export class Syncer {
private readonly settings: Settings,
private readonly webSocketManager: WebSocketManager,
private readonly operations: FileOperations,
private readonly internalSyncer: UnrestrictedSyncer
private readonly unrestrictedSyncer: UnrestrictedSyncer
) {
this.syncQueue = new PQueue({
concurrency: settings.getSettings().syncConcurrency
@ -81,12 +81,15 @@ export class Syncer {
}
public async syncLocallyCreatedFile(
relativePath: RelativePath
relativePath: RelativePath,
{ forceMerge }: { forceMerge: boolean }
): Promise<void> {
if (
this.database.getLatestDocumentByRelativePath(relativePath)
?.isDeleted === false
) {
// This is likely a consequence of us creating a file because of a remote update
// which triggered a local create, so we don't need to do anything here.
this.logger.debug(
`Document ${relativePath} already exists in the database, skipping`
);
@ -94,6 +97,7 @@ export class Syncer {
}
const [promise, resolve, reject] = createPromise();
this.logger.warn(`creating ${relativePath} locally`);
const document = this.database.createNewPendingDocument(
relativePath,
@ -102,8 +106,13 @@ export class Syncer {
try {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncLocallyCreatedFile(document)
);
this.unrestrictedSyncer.unrestrictedSyncLocallyCreatedOrUpdatedFile(
{ document, forceMerge }
)
)
this.logger.warn(`done creating ${relativePath} locally`);
resolve();
} catch (e) {
@ -123,7 +132,7 @@ export class Syncer {
// This is must be a consequence of us deleting a file because of a remote update
// which triggered a local delete, so we don't need to do anything here.
this.logger.debug(
`Document ${relativePath} has already been markes as deleted, skipping`
`Document ${relativePath} has already been marked as deleted, skipping`
);
return;
}
@ -141,7 +150,7 @@ export class Syncer {
try {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncLocallyDeletedFile(document)
this.unrestrictedSyncer.unrestrictedSyncLocallyDeletedFile(document)
);
resolve();
@ -166,7 +175,7 @@ export class Syncer {
// in that case, we mustn't move it again.
if (
this.database.getLatestDocumentByRelativePath(relativePath) ===
undefined ||
undefined ||
this.database.getLatestDocumentByRelativePath(relativePath)
?.isDeleted === true
) {
@ -183,6 +192,8 @@ export class Syncer {
let document =
this.database.getLatestDocumentByRelativePath(relativePath);
this.logger.warn(`sync doc ${JSON.stringify(document)} for path ${relativePath} (old path: ${oldPath}), len docs: ${document?.updates.length}`);
if (
oldPath !== undefined &&
document?.metadata?.remoteRelativePath === relativePath
@ -193,6 +204,7 @@ export class Syncer {
return;
}
// must have been removed after a successful delete
if (document === undefined) {
this.logger.debug(
`Cannot find document ${relativePath} in the database, skipping`
@ -213,12 +225,13 @@ export class Syncer {
relativePath,
promise
);
this.logger.warn(`updating ${document.relativePath} locally`);
try {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncLocallyUpdatedFile({
this.unrestrictedSyncer.unrestrictedSyncLocallyCreatedOrUpdatedFile({
oldPath,
document
document: document!
})
);
@ -252,8 +265,6 @@ export class Syncer {
`Not all local changes have been applied remotely: ${e}`
);
throw e;
} finally {
this.runningScheduleSyncForOfflineChanges = undefined;
}
}
@ -266,6 +277,8 @@ export class Syncer {
message: WebSocketVaultUpdate
): Promise<void> {
try {
await this.scheduleSyncForOfflineChanges();
const handlerPromise = awaitAll(
message.documents.map(async (document) =>
this.internalSyncRemotelyUpdatedFile(document)
@ -312,25 +325,45 @@ export class Syncer {
remoteVersion.documentId
);
this.logger.warn(`${remoteVersion.documentId} got remote update ${JSON.stringify(remoteVersion)}`);
if (document === undefined) {
// Let's avoid the same documents getting created in parallel multiple times.
// There might be multiple tasks waiting for the lock
this.logger.warn(`${remoteVersion.documentId} but document doesn't exist`)
return this.remoteDocumentsLock.withLock(
// Avoid the same documents getting created in parallel multiple times through fetching multiple updates of the same
// new remote document concurrently.
// There might be multiple tasks waiting for the lock
remoteVersion.documentId,
async () => {
// We have to wait for any ongoing creates sent for this file to finish,
// This is to avoid fetching one's own creates before the corresponding local create has finished syncing. This is a concern because
// documents being created don't yet have a document id in the local database and we could be notified of the remote create
// before the local create has finished syncing, so we can't just ignore the update based on the local DB content as we
// can't find the corresponding document yet.
if (document?.metadata === undefined) {
await this.unrestrictedSyncer.fileCreationLock.waitForLockWithoutAcquiringLock(remoteVersion.relativePath);
}
document = this.database.getDocumentByDocumentId(
remoteVersion.documentId
);
// We're either the first one to get the lock, so we have to create the document in `unrestrictedSyncRemotelyUpdatedFile`
this.logger.warn(`${remoteVersion.documentId} rechecking, document is now ${JSON.stringify(document)}`)
// We're the first one to get the lock, so we have to create the document in `unrestrictedSyncRemotelyUpdatedFile`
if (document === undefined) {
this.logger.warn(`${remoteVersion.documentId} document is undefined, creating new document`)
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
this.unrestrictedSyncer.unrestrictedSyncRemotelyUpdatedFile(
remoteVersion
)
);
} else {
const [promise, resolve, reject] = createPromise();
const [promise, resolve, reject] =
createPromise();
document =
await this.database.getResolvedDocumentByRelativePath(
@ -340,7 +373,7 @@ export class Syncer {
try {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
this.unrestrictedSyncer.unrestrictedSyncRemotelyUpdatedFile(
remoteVersion,
document
)
@ -350,13 +383,19 @@ export class Syncer {
} catch (e) {
reject(e);
} finally {
this.database.removeDocumentPromise(promise);
this.database.removeDocumentPromise(
promise
);
}
}
this.database.addSeenUpdateId(remoteVersion.vaultUpdateId);
this.database.addSeenUpdateId(
remoteVersion.vaultUpdateId
);
}
);
)
} else {
this.logger.warn(`${remoteVersion.documentId} and document exists (path: ${JSON.stringify(document)})`);
}
// We're either the first one to get the lock, so we have to create the document in `unrestrictedSyncRemotelyUpdatedFile`
@ -369,7 +408,7 @@ export class Syncer {
try {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
this.unrestrictedSyncer.unrestrictedSyncRemotelyUpdatedFile(
remoteVersion,
document
)
@ -402,7 +441,8 @@ export class Syncer {
}
}
await awaitAll(
type Instruction = { "type": "update" | "create", relativePath: string, oldPath?: string };
const instructions: (Instruction | undefined)[] = await awaitAll(
allLocalFiles.map(async (relativePath) => {
if (
this.database.getLatestDocumentByRelativePath(relativePath)
@ -412,9 +452,7 @@ export class Syncer {
`Document ${relativePath} might have been updated locally, scheduling sync to validate and update it`
);
return this.syncLocallyUpdatedFile({
relativePath
});
return { type: "update", relativePath } as Instruction;
}
// Perhaps the file has been moved; let's check by looking at the deleted files
@ -457,21 +495,26 @@ export class Syncer {
`Document '${originalFile.relativePath}' was not found under its current path in the database but was found under a different path (${relativePath}), scheduling sync to move it`
);
// We're outside of the pqueue, so we need to call the public wrapper
return this.syncLocallyUpdatedFile({
return {
type: "update",
oldPath: originalFile.relativePath,
relativePath
});
} as Instruction;
}
this.logger.debug(
`Document ${relativePath} not found in database, scheduling sync to create it`
);
// We're outside of the pqueue, so we need to call the public wrapper
return this.syncLocallyCreatedFile(relativePath);
return {
type: "create",
relativePath
} as Instruction;
})
);
// this has to happen strictly after the previous awaitAll, as that one
// might have removed some of the documents from the list
await awaitAll(
@ -484,5 +527,36 @@ export class Syncer {
return this.syncLocallyDeletedFile(relativePath);
})
);
await awaitAll(instructions.map(async (instruction) => {
if (instruction === undefined) {
return;
}
if (instruction.type === "update") {
// We're outside of the pqueue, so we need to call the public wrapper
return await this.syncLocallyUpdatedFile({
oldPath: instruction.oldPath,
relativePath: instruction.relativePath
});
}
}));
// we have to ensure the deletes & updates have finished before starting creates,
// otherwise the server might return an existing document (that we're about to delete)
// instead of actually creating a new one
await awaitAll(instructions.map(async (instruction) => {
if (instruction === undefined) {
return;
}
if (instruction.type === "create") {
// We're outside of the pqueue, so we need to call the public wrapper
return await this.syncLocallyCreatedFile(instruction.relativePath, { forceMerge: true });
}
}));
}
}