andras/vault-link
1
0
Fork 0
Code Issues Pull requests 18 Projects Releases 55 Packages Wiki Activity Actions 1

Working setup

Browse source
This commit is contained in:
Andras Schmelczer 2026-01-12 21:24:05 +00:00
parent e3a90833ff
commit 2dfb8b71e5
16 changed files with 459 additions and 318 deletions
Download patch file Download diff file Expand all files Collapse all files

134
frontend/sync-client/src/sync-operations/syncer.ts
View file

@ -42,7 +42,7 @@ export class Syncer {
private readonly settings: Settings,
private readonly webSocketManager: WebSocketManager,
private readonly operations: FileOperations,
private readonly internalSyncer: UnrestrictedSyncer
private readonly unrestrictedSyncer: UnrestrictedSyncer
) {
this.syncQueue = new PQueue({
concurrency: settings.getSettings().syncConcurrency
@ -81,12 +81,15 @@ export class Syncer {
}
public async syncLocallyCreatedFile(
relativePath: RelativePath
relativePath: RelativePath,
{ forceMerge }: { forceMerge: boolean }
): Promise<void> {
if (
this.database.getLatestDocumentByRelativePath(relativePath)
?.isDeleted === false
) {
// This is likely a consequence of us creating a file because of a remote update
// which triggered a local create, so we don't need to do anything here.
this.logger.debug(
`Document ${relativePath} already exists in the database, skipping`
);
@ -94,6 +97,7 @@ export class Syncer {
}
const [promise, resolve, reject] = createPromise();
this.logger.warn(`creating ${relativePath} locally`);
const document = this.database.createNewPendingDocument(
relativePath,
@ -102,8 +106,13 @@ export class Syncer {
try {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncLocallyCreatedFile(document)
);
this.unrestrictedSyncer.unrestrictedSyncLocallyCreatedOrUpdatedFile(
{ document, forceMerge }
)
)
this.logger.warn(`done creating ${relativePath} locally`);
resolve();
} catch (e) {
@ -123,7 +132,7 @@ export class Syncer {
// This is must be a consequence of us deleting a file because of a remote update
// which triggered a local delete, so we don't need to do anything here.
this.logger.debug(
`Document ${relativePath} has already been markes as deleted, skipping`
`Document ${relativePath} has already been marked as deleted, skipping`
);
return;
}
@ -141,7 +150,7 @@ export class Syncer {
try {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncLocallyDeletedFile(document)
this.unrestrictedSyncer.unrestrictedSyncLocallyDeletedFile(document)
);
resolve();
@ -166,7 +175,7 @@ export class Syncer {
// in that case, we mustn't move it again.
if (
this.database.getLatestDocumentByRelativePath(relativePath) ===
undefined ||
undefined ||
this.database.getLatestDocumentByRelativePath(relativePath)
?.isDeleted === true
) {
@ -183,6 +192,8 @@ export class Syncer {
let document =
this.database.getLatestDocumentByRelativePath(relativePath);
this.logger.warn(`sync doc ${JSON.stringify(document)} for path ${relativePath} (old path: ${oldPath}), len docs: ${document?.updates.length}`);
if (
oldPath !== undefined &&
document?.metadata?.remoteRelativePath === relativePath
@ -193,6 +204,7 @@ export class Syncer {
return;
}
// must have been removed after a successful delete
if (document === undefined) {
this.logger.debug(
`Cannot find document ${relativePath} in the database, skipping`
@ -213,12 +225,13 @@ export class Syncer {
relativePath,
promise
);
this.logger.warn(`updating ${document.relativePath} locally`);
try {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncLocallyUpdatedFile({
this.unrestrictedSyncer.unrestrictedSyncLocallyCreatedOrUpdatedFile({
oldPath,
document
document: document!
})
);
@ -252,8 +265,6 @@ export class Syncer {
`Not all local changes have been applied remotely: ${e}`
);
throw e;
} finally {
this.runningScheduleSyncForOfflineChanges = undefined;
}
}
@ -266,6 +277,8 @@ export class Syncer {
message: WebSocketVaultUpdate
): Promise<void> {
try {
await this.scheduleSyncForOfflineChanges();
const handlerPromise = awaitAll(
message.documents.map(async (document) =>
this.internalSyncRemotelyUpdatedFile(document)
@ -312,25 +325,45 @@ export class Syncer {
remoteVersion.documentId
);
this.logger.warn(`${remoteVersion.documentId} got remote update ${JSON.stringify(remoteVersion)}`);
if (document === undefined) {
// Let's avoid the same documents getting created in parallel multiple times.
// There might be multiple tasks waiting for the lock
this.logger.warn(`${remoteVersion.documentId} but document doesn't exist`)
return this.remoteDocumentsLock.withLock(
// Avoid the same documents getting created in parallel multiple times through fetching multiple updates of the same
// new remote document concurrently.
// There might be multiple tasks waiting for the lock
remoteVersion.documentId,
async () => {
// We have to wait for any ongoing creates sent for this file to finish,
// This is to avoid fetching one's own creates before the corresponding local create has finished syncing. This is a concern because
// documents being created don't yet have a document id in the local database and we could be notified of the remote create
// before the local create has finished syncing, so we can't just ignore the update based on the local DB content as we
// can't find the corresponding document yet.
if (document?.metadata === undefined) {
await this.unrestrictedSyncer.fileCreationLock.waitForLockWithoutAcquiringLock(remoteVersion.relativePath);
}
document = this.database.getDocumentByDocumentId(
remoteVersion.documentId
);
// We're either the first one to get the lock, so we have to create the document in `unrestrictedSyncRemotelyUpdatedFile`
this.logger.warn(`${remoteVersion.documentId} rechecking, document is now ${JSON.stringify(document)}`)
// We're the first one to get the lock, so we have to create the document in `unrestrictedSyncRemotelyUpdatedFile`
if (document === undefined) {
this.logger.warn(`${remoteVersion.documentId} document is undefined, creating new document`)
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
this.unrestrictedSyncer.unrestrictedSyncRemotelyUpdatedFile(
remoteVersion
)
);
} else {
const [promise, resolve, reject] = createPromise();
const [promise, resolve, reject] =
createPromise();
document =
await this.database.getResolvedDocumentByRelativePath(
@ -340,7 +373,7 @@ export class Syncer {
try {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
this.unrestrictedSyncer.unrestrictedSyncRemotelyUpdatedFile(
remoteVersion,
document
)
@ -350,13 +383,19 @@ export class Syncer {
} catch (e) {
reject(e);
} finally {
this.database.removeDocumentPromise(promise);
this.database.removeDocumentPromise(
promise
);
}
}
this.database.addSeenUpdateId(remoteVersion.vaultUpdateId);
this.database.addSeenUpdateId(
remoteVersion.vaultUpdateId
);
}
);
)
} else {
this.logger.warn(`${remoteVersion.documentId} and document exists (path: ${JSON.stringify(document)})`);
}
// We're either the first one to get the lock, so we have to create the document in `unrestrictedSyncRemotelyUpdatedFile`
@ -369,7 +408,7 @@ export class Syncer {
try {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
this.unrestrictedSyncer.unrestrictedSyncRemotelyUpdatedFile(
remoteVersion,
document
)
@ -402,7 +441,8 @@ export class Syncer {
}
}
await awaitAll(
type Instruction = { "type": "update" | "create", relativePath: string, oldPath?: string };
const instructions: (Instruction | undefined)[] = await awaitAll(
allLocalFiles.map(async (relativePath) => {
if (
this.database.getLatestDocumentByRelativePath(relativePath)
@ -412,9 +452,7 @@ export class Syncer {
`Document ${relativePath} might have been updated locally, scheduling sync to validate and update it`
);
return this.syncLocallyUpdatedFile({
relativePath
});
return { type: "update", relativePath } as Instruction;
}
// Perhaps the file has been moved; let's check by looking at the deleted files
@ -457,21 +495,26 @@ export class Syncer {
`Document '${originalFile.relativePath}' was not found under its current path in the database but was found under a different path (${relativePath}), scheduling sync to move it`
);
// We're outside of the pqueue, so we need to call the public wrapper
return this.syncLocallyUpdatedFile({
return {
type: "update",
oldPath: originalFile.relativePath,
relativePath
});
} as Instruction;
}
this.logger.debug(
`Document ${relativePath} not found in database, scheduling sync to create it`
);
// We're outside of the pqueue, so we need to call the public wrapper
return this.syncLocallyCreatedFile(relativePath);
return {
type: "create",
relativePath
} as Instruction;
})
);
// this has to happen strictly after the previous awaitAll, as that one
// might have removed some of the documents from the list
await awaitAll(
@ -484,5 +527,36 @@ export class Syncer {
return this.syncLocallyDeletedFile(relativePath);
})
);
await awaitAll(instructions.map(async (instruction) => {
if (instruction === undefined) {
return;
}
if (instruction.type === "update") {
// We're outside of the pqueue, so we need to call the public wrapper
return await this.syncLocallyUpdatedFile({
oldPath: instruction.oldPath,
relativePath: instruction.relativePath
});
}
}));
// we have to ensure the deletes & updates have finished before starting creates,
// otherwise the server might return an existing document (that we're about to delete)
// instead of actually creating a new one
await awaitAll(instructions.map(async (instruction) => {
if (instruction === undefined) {
return;
}
if (instruction.type === "create") {
// We're outside of the pqueue, so we need to call the public wrapper
return await this.syncLocallyCreatedFile(instruction.relativePath, { forceMerge: true });
}
}));
}
}

349
frontend/sync-client/src/sync-operations/unrestricted-syncer.ts
View file

@ -33,9 +33,12 @@ import type { FixedSizeDocumentCache } from "../utils/data-structures/fix-sized-
import { isFileTypeMergable } from "../utils/is-file-type-mergable";
import { isBinary } from "../utils/is-binary";
import type { ServerConfig } from "../services/server-config";
import { Locks } from "../utils/data-structures/locks";
export class UnrestrictedSyncer {
private ignorePatterns: RegExp[];
public readonly fileCreationLock: Locks<RelativePath> = new Locks<RelativePath>();
public constructor(
private readonly logger: Logger,
@ -60,118 +63,50 @@ export class UnrestrictedSyncer {
});
}
public async unrestrictedSyncLocallyCreatedFile(
document: DocumentRecord
): Promise<void> {
const updateDetails: SyncCreateDetails = {
type: SyncType.CREATE,
relativePath: document.relativePath
};
return this.executeSync(updateDetails, async () => {
const originalRelativePath = document.relativePath;
if (document.isDeleted) {
this.logger.debug(
`Document ${originalRelativePath} has been already deleted, no need to create it`
);
return;
}
const contentBytes =
await this.operations.read(originalRelativePath); // this can throw FileNotFoundError
const contentHash = hash(contentBytes);
const response = await this.syncService.create({
relativePath: originalRelativePath,
contentBytes,
forceMerge: true
});
await this.handleMaybeMergingResponse({
document,
response,
contentHash,
originalRelativePath,
originalContentBytes: contentBytes
});
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
details: updateDetails,
message: `Successfully uploaded locally created file`
});
});
}
public async unrestrictedSyncLocallyDeletedFile(
document: DocumentRecord
): Promise<void> {
const updateDetails: SyncDeleteDetails = {
type: SyncType.DELETE,
relativePath: document.relativePath
};
await this.executeSync(updateDetails, async () => {
if (document.metadata === undefined) {
this.logger.debug(
`Document ${document.relativePath} has no metadata, so it has never got synced remotely; no need to delete it remotely`
);
return;
}
const response = await this.syncService.delete({
documentId: document.metadata.documentId,
relativePath: document.relativePath
});
this.database.updateDocumentMetadata(
{
...document.metadata,
parentVersionId: response.vaultUpdateId,
hash: EMPTY_HASH,
remoteRelativePath: document.relativePath
},
document
);
this.database.addSeenUpdateId(response.vaultUpdateId);
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
details: updateDetails,
message: `Successfully deleted locally deleted file on the server`,
author: response.userId
});
});
}
public async unrestrictedSyncLocallyUpdatedFile({
public async unrestrictedSyncLocallyCreatedOrUpdatedFile({
oldPath,
document,
forceMerge,
// We use the same code path for both local and remote updates. We need to force the update
// if there are no local changes but we know that the remote version is newer.
force = false
}: {
oldPath?: RelativePath;
force?: boolean;
forceMerge?: boolean
document: DocumentRecord;
}): Promise<void> {
const updateDetails: SyncUpdateDetails | SyncMovedDetails =
oldPath !== undefined
? {
type: SyncType.MOVE,
relativePath: document.relativePath,
movedFrom: oldPath
}
: {
type: SyncType.UPDATE,
relativePath: document.relativePath
};
// this.history.addHistoryEntry({
// status: SyncStatus.SUCCESS,
// details: updateDetails,
// message: `Successfully uploaded locally created file`
// });
let updateDetails: SyncCreateDetails | SyncUpdateDetails | SyncMovedDetails;
if (document.metadata === undefined) {
updateDetails = {
type: SyncType.CREATE,
relativePath: document.relativePath
};
}
else if (oldPath !== undefined) {
updateDetails = {
type: SyncType.MOVE,
relativePath: document.relativePath,
movedFrom: oldPath
};
} else {
updateDetails = {
type: SyncType.UPDATE,
relativePath: document.relativePath
};
}
await this.executeSync(updateDetails, async () => {
const originalRelativePath = document.relativePath;
if (document.isDeleted || document.metadata === undefined) {
if (document.isDeleted) {
this.logger.debug(
`Document ${document.relativePath} has been already deleted, no need to update it`
);
@ -183,64 +118,88 @@ export class UnrestrictedSyncer {
); // this can throw FileNotFoundError
const contentHash = hash(contentBytes);
const areThereLocalChanges = !(
document.metadata.hash === contentHash && oldPath === undefined
);
this.logger.warn(`updating ${document.relativePath} locally, inner`);
let response: DocumentVersion | DocumentUpdateResponse | undefined =
undefined;
if (areThereLocalChanges) {
const isText =
!isBinary(contentBytes) &&
isFileTypeMergable(
document.relativePath,
(await this.serverConfig.getConfig())
.mergeableFileExtensions
);
const cachedVersion = this.contentCache.get(
document.metadata.parentVersionId
);
if (document.metadata === undefined) {
response = await this.fileCreationLock.withLock(document.relativePath, async () => {
const response = await this.syncService.create({
relativePath: originalRelativePath,
contentBytes,
forceMerge
});
response =
isText && cachedVersion !== undefined
? await this.syncService.putText({
documentId: document.metadata.documentId,
parentVersionId:
document.metadata.parentVersionId,
relativePath: document.relativePath,
content: diff(
new TextDecoder().decode(cachedVersion),
new TextDecoder().decode(contentBytes)
)
})
: await this.syncService.putBinary({
documentId: document.metadata.documentId,
parentVersionId:
document.metadata.parentVersionId,
relativePath: document.relativePath,
contentBytes
});
await this.handleMaybeMergingResponse({
document,
response,
contentHash,
originalRelativePath,
originalContentBytes: contentBytes
});
return response;
});
} else {
if (!force) {
this.logger.debug(
`File hash of ${document.relativePath} matches with last synced version and the path hasn't changed; no need to sync`
const areThereLocalChanges =
document.metadata.hash !== contentHash || oldPath !== undefined;
if (areThereLocalChanges) {
const isText =
!isBinary(contentBytes) &&
isFileTypeMergable(
document.relativePath,
(await this.serverConfig.getConfig())
.mergeableFileExtensions
);
const cachedVersion = this.contentCache.get(
document.metadata.parentVersionId
);
return;
response =
isText && cachedVersion !== undefined
? await this.syncService.putText({
documentId: document.metadata.documentId,
parentVersionId:
document.metadata.parentVersionId,
relativePath: document.relativePath,
content: diff(
new TextDecoder().decode(cachedVersion),
new TextDecoder().decode(contentBytes)
)
})
: await this.syncService.putBinary({
documentId: document.metadata.documentId,
parentVersionId:
document.metadata.parentVersionId,
relativePath: document.relativePath,
contentBytes
});
} else {
if (!force) {
this.logger.debug(
`File hash of ${document.relativePath} matches with last synced version and the path hasn't changed; no need to sync`
);
return;
}
// we use this code path (force == true) to sync remotely updated files which have no local changes
response = await this.syncService.get({
documentId: document.metadata.documentId
});
}
response = await this.syncService.get({
documentId: document.metadata.documentId
await this.handleMaybeMergingResponse({
document,
response,
contentHash,
originalRelativePath,
originalContentBytes: contentBytes
});
}
await this.handleMaybeMergingResponse({
document,
response: response,
contentHash,
originalRelativePath,
originalContentBytes: contentBytes
});
if (!("type" in response) || response.type === "MergingUpdate") {
if (!force) {
@ -249,30 +208,33 @@ export class UnrestrictedSyncer {
details: updateDetails,
message: `The file we updated had been updated remotely, so we downloaded the merged version`
});
return;
}
}
const actualUpdateDetails: SyncUpdateDetails | SyncMovedDetails =
oldPath !== undefined ||
response.relativePath != originalRelativePath
response.relativePath != originalRelativePath
? {
type: SyncType.MOVE,
relativePath: response.relativePath,
movedFrom: oldPath ?? originalRelativePath
}
type: SyncType.MOVE,
relativePath: response.relativePath,
movedFrom: originalRelativePath
}
: {
type: SyncType.UPDATE,
relativePath: response.relativePath
};
type: SyncType.UPDATE,
relativePath: response.relativePath
};
if (areThereLocalChanges) {
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
details: actualUpdateDetails,
message: `Successfully uploaded locally updated file to the server`,
author: response.userId
});
} else if (!response.isDeleted) {
// if (areThereLocalChanges) {
// this.history.addHistoryEntry({
// status: SyncStatus.SUCCESS,
// details: actualUpdateDetails,
// message: `Successfully uploaded locally updated file to the server`,
// author: response.userId
// });
// } else
if (!response.isDeleted) {
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
details: actualUpdateDetails,
@ -296,6 +258,49 @@ export class UnrestrictedSyncer {
});
}
public async unrestrictedSyncLocallyDeletedFile(
document: DocumentRecord
): Promise<void> {
const updateDetails: SyncDeleteDetails = {
type: SyncType.DELETE,
relativePath: document.relativePath
};
await this.executeSync(updateDetails, async () => {
if (document.metadata === undefined) {
this.logger.debug(
`Document ${document.relativePath} has never been synced, no need to delete it remotely`
);
return;
}
const response = await this.syncService.delete({
documentId: document.metadata.documentId,
relativePath: document.relativePath
});
this.database.updateDocumentMetadata(
{
documentId: response.documentId,
parentVersionId: response.vaultUpdateId,
hash: EMPTY_HASH,
remoteRelativePath: document.relativePath
},
document
);
this.database.addSeenUpdateId(response.vaultUpdateId);
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
details: updateDetails,
message: `Successfully deleted locally deleted file on the server`,
author: response.userId
});
});
}
public async unrestrictedSyncRemotelyUpdatedFile(
remoteVersion: DocumentVersionWithoutContent,
document?: DocumentRecord
@ -305,6 +310,7 @@ export class UnrestrictedSyncer {
relativePath: remoteVersion.relativePath
};
await this.executeSync(updateDetails, async () => {
if (document?.metadata !== undefined) {
// If the file exists locally, let's pretend the user has updated it
@ -320,7 +326,7 @@ export class UnrestrictedSyncer {
return;
}
return this.unrestrictedSyncLocallyUpdatedFile({
return this.unrestrictedSyncLocallyCreatedOrUpdatedFile({
document,
force: true
});
@ -403,10 +409,21 @@ export class UnrestrictedSyncer {
});
}
public async executeSync<T>(
public reset(): void {
this.fileCreationLock.reset();
}
private async executeSync<T>(
details: SyncDetails,
fn: () => Promise<T>
): Promise<T | undefined> {
if (!this.settings.getSettings().isSyncEnabled) {
this.logger.info(
`Skipping sync operation for file '${details.relativePath}' because sync is disabled`
);
return;
}
for (const pattern of this.ignorePatterns) {
if (pattern.test(details.relativePath)) {
this.logger.debug(
@ -460,6 +477,8 @@ export class UnrestrictedSyncer {
}
}
private async handleMaybeMergingResponse({
document,
response,
@ -474,7 +493,6 @@ export class UnrestrictedSyncer {
originalContentBytes: Uint8Array;
}): Promise<void> {
// `document` is mutable and reflects the latest state in the local database
if (document.isDeleted) {
this.logger.info(
`Document ${document.relativePath} has been deleted before we could finish updating it`
@ -569,9 +587,8 @@ export class UnrestrictedSyncer {
type: SyncType.SKIPPED,
relativePath
},
message: `File size of ${sizeInMB} MB exceeds the maximum file size limit of ${
maxFileSizeMB
} MB`
message: `File size of ${sizeInMB} MB exceeds the maximum file size limit of ${maxFileSizeMB
} MB`
};
}
}