andras/vault-link
1
0
Fork 0
Code Issues Pull requests 18 Projects Releases 55 Packages Wiki Activity Actions 1

Simplify syncing logic

Browse source
This commit is contained in:
Andras Schmelczer 2026-03-28 11:55:37 +00:00
parent e8c57b3a37
commit 4493365076
48 changed files with 1054 additions and 918 deletions
Download patch file Download diff file Expand all files Collapse all files

8
frontend/sync-client/src/sync-operations/cursor-tracker.ts
View file

@ -10,6 +10,7 @@ import { hash } from "../utils/hash";
import type { FileChangeNotifier } from "./file-change-notifier";
import { Lock } from "../utils/data-structures/locks";
import { EventListeners } from "../utils/data-structures/event-listeners";
import { Logger } from "../tracing/logger";
// Cursor positions are updated separately from documents. However, a given cursor position is only
// valid within a certain version of the document it belongs to. This class tracks previous and the latest
@ -22,7 +23,7 @@ export class CursorTracker {
(cursors: MaybeOutdatedClientCursors[]) => unknown
>();
private readonly updateLock = new Lock();
private readonly updateLock: Lock;
private knownRemoteCursors: (ClientCursors & {
upToDateness: DocumentUpToDateness;
@ -33,11 +34,14 @@ export class CursorTracker {
[];
public constructor(
private readonly logger: Logger,
private readonly database: Database,
private readonly webSocketManager: WebSocketManager,
private readonly fileOperations: FileOperations,
private readonly fileChangeNotifier: FileChangeNotifier
) {
this.updateLock = new Lock(CursorTracker.name, logger);
this.webSocketManager.onRemoteCursorsUpdateReceived.add(
async (clientCursors) => {
await this.updateLock.withLock(async () => {
@ -113,7 +117,7 @@ export class CursorTracker {
documentsWithCursors.push({
relative_path: relativePath,
document_id: record.documentId,
document_id: record.metadata.documentId,
vault_update_id: record.metadata.parentVersionId,
cursors: cursors.map(({ start, end }) => ({
start: Math.min(start, end),

375
frontend/sync-client/src/sync-operations/syncer.ts
View file

@ -4,17 +4,14 @@ import type {
DocumentRecord,
RelativePath
} from "../persistence/database";
import type { SyncService } from "../services/sync-service";
import type { Logger } from "../tracing/logger";
import PQueue from "p-queue";
import { hash } from "../utils/hash";
import { v4 as uuidv4 } from "uuid";
import type { Settings } from "../persistence/settings";
import type { FileOperations } from "../file-operations/file-operations";
import { findMatchingFile } from "../utils/find-matching-file";
import type { UnrestrictedSyncer } from "./unrestricted-syncer";
import { createPromise } from "../utils/create-promise";
import { SyncResetError } from "../services/sync-reset-error";
import { SyncResetError } from "../errors/sync-reset-error";
import { Locks } from "../utils/data-structures/locks";
import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent";
import type { WebSocketVaultUpdate } from "../services/types/WebSocketVaultUpdate";
@ -28,7 +25,7 @@ export class Syncer {
(remainingOperations: number) => unknown
>();
private readonly remoteDocumentsLock: Locks<DocumentId>;
public readonly updatedDocumentsByPathAndKeysLocks: Locks<string>; // can be DocumentId or RelativePath
// FIFO to limit the number of concurrent sync operations
private readonly syncQueue: PQueue;
@ -42,16 +39,18 @@ export class Syncer {
private readonly logger: Logger,
private readonly database: Database,
private readonly settings: Settings,
private readonly syncService: SyncService,
private readonly webSocketManager: WebSocketManager,
private readonly operations: FileOperations,
private readonly internalSyncer: UnrestrictedSyncer
private readonly unrestrictedSyncer: UnrestrictedSyncer
) {
this.syncQueue = new PQueue({
concurrency: settings.getSettings().syncConcurrency
});
this.remoteDocumentsLock = new Locks<DocumentId>(this.logger);
this.updatedDocumentsByPathAndKeysLocks = new Locks<DocumentId>(
Syncer.name,
this.logger
);
settings.onSettingsChanged.add((newSettings, oldSettings) => {
if (newSettings.syncConcurrency !== oldSettings.syncConcurrency) {
@ -83,52 +82,50 @@ export class Syncer {
return this._isFirstSyncComplete;
}
public hasPendingOperationsForDocument(relativePath: string): boolean {
return this.updatedDocumentsByPathAndKeysLocks.isLocked(relativePath);
}
public async syncLocallyCreatedFile(
relativePath: RelativePath
): Promise<void> {
// check whether someone else has already created the document in the database
if (
this.database.getLatestDocumentByRelativePath(relativePath)
?.isDeleted === false
) {
// This is likely a consequence of us creating a file because of a remote update
// which triggered a local create, so we don't need to do anything here.
this.logger.debug(
`Document ${relativePath} already exists in the database, skipping`
);
return;
}
const [promise, resolve, reject] = createPromise();
const document = this.database.createNewPendingDocument(relativePath);
const id = uuidv4();
const document = this.database.createNewPendingDocument(
id,
relativePath,
promise
await this.enqueueSyncOperation(
async () =>
this.unrestrictedSyncer.unrestrictedSyncLocallyCreatedOrUpdatedFile(
{
document
}
),
[relativePath]
);
try {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncLocallyCreatedFile(document)
);
resolve();
} catch (e) {
reject(e);
} finally {
this.database.removeDocumentPromise(promise);
}
}
public async syncLocallyDeletedFile(
relativePath: RelativePath
): Promise<void> {
if (
this.database.getLatestDocumentByRelativePath(relativePath)
?.isDeleted === true
) {
let document =
this.database.getLatestDocumentByRelativePath(relativePath);
if (document == null || document.isDeleted === true) {
// This is must be a consequence of us deleting a file because of a remote update
// which triggered a local delete, so we don't need to do anything here.
this.logger.debug(
`Document ${relativePath} has already been markes as deleted, skipping`
`Document ${relativePath} has already been marked as deleted, skipping`
);
return;
}
@ -137,26 +134,13 @@ export class Syncer {
// document which finishes after the delete has succeeded and would introduce a phantom metadata record.
this.database.delete(relativePath);
const [promise, resolve, reject] = createPromise();
const document = await this.database.getResolvedDocumentByRelativePath(
relativePath,
promise
);
try {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncLocallyDeletedFile(document)
await this.enqueueSyncOperation(async () => {
await this.unrestrictedSyncer.unrestrictedSyncLocallyDeletedFile(
document
);
resolve();
this.database.removeDocument(document);
} catch (e) {
reject(e);
} finally {
this.database.removeDocumentPromise(promise);
}
}, [document?.metadata?.documentId, relativePath]);
}
public async syncLocallyUpdatedFile({
@ -166,38 +150,10 @@ export class Syncer {
oldPath?: RelativePath;
relativePath: RelativePath;
}): Promise<void> {
if (oldPath !== undefined) {
// We might have moved the document in the database before calling this method,
// in that case, we mustn't move it again.
if (
this.database.getLatestDocumentByRelativePath(relativePath) ===
undefined ||
this.database.getLatestDocumentByRelativePath(relativePath)
?.isDeleted === true
) {
if (oldPath === relativePath) {
throw new Error(
`Old path and new path are the same: ${oldPath}`
);
}
this.database.move(oldPath, relativePath);
}
}
let document =
this.database.getLatestDocumentByRelativePath(relativePath);
if (
oldPath !== undefined &&
document?.metadata?.remoteRelativePath === relativePath
) {
this.logger.debug(
`Document ${relativePath} has been moved as a result of a remote update, skipping sync`
);
return;
}
const document =
this.database.getLatestDocumentByRelativePath(oldPath ?? relativePath);
// must have been removed after a successful delete
if (document === undefined) {
this.logger.debug(
`Cannot find document ${relativePath} in the database, skipping`
@ -212,27 +168,47 @@ export class Syncer {
return;
}
const [promise, resolve, reject] = createPromise();
const documentAtNewPath =
this.database.getLatestDocumentByRelativePath(relativePath);
document = await this.database.getResolvedDocumentByRelativePath(
relativePath,
promise
);
if (oldPath !== undefined) {
// We might have moved the document in the database before calling this method,
// in that case, we mustn't move it again.
if (
documentAtNewPath === undefined ||
documentAtNewPath.isDeleted
) {
if (oldPath === relativePath) {
throw new Error(
`Old path and new path are the same: ${oldPath}`
);
}
try {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncLocallyUpdatedFile({
oldPath,
document
})
);
resolve();
} catch (e) {
reject(e);
} finally {
this.database.removeDocumentPromise(promise);
this.database.move(oldPath, relativePath);
}
}
if (
oldPath !== undefined &&
document?.metadata?.remoteRelativePath === relativePath
) {
this.logger.debug(
`Document ${relativePath} has been moved as a result of a remote update, skipping sync`
);
return;
}
await this.enqueueSyncOperation(
async () =>
this.unrestrictedSyncer.unrestrictedSyncLocallyCreatedOrUpdatedFile(
{
oldPath,
document
}
),
[document.metadata?.documentId, relativePath, oldPath]
);
}
public async scheduleSyncForOfflineChanges(): Promise<void> {
@ -257,8 +233,6 @@ export class Syncer {
`Not all local changes have been applied remotely: ${e}`
);
throw e;
} finally {
this.runningScheduleSyncForOfflineChanges = undefined;
}
}
@ -271,6 +245,8 @@ export class Syncer {
message: WebSocketVaultUpdate
): Promise<void> {
try {
await this.scheduleSyncForOfflineChanges();
const handlerPromise = awaitAll(
message.documents.map(async (document) =>
this.internalSyncRemotelyUpdatedFile(document)
@ -296,7 +272,7 @@ export class Syncer {
public reset(): void {
this._isFirstSyncComplete = false;
this.syncQueue.clear();
this.remoteDocumentsLock.reset();
this.updatedDocumentsByPathAndKeysLocks.reset();
this.runningScheduleSyncForOfflineChanges = undefined;
}
@ -313,86 +289,26 @@ export class Syncer {
private async internalSyncRemotelyUpdatedFile(
remoteVersion: DocumentVersionWithoutContent
): Promise<void> {
let document = this.database.getDocumentByDocumentId(
const document = this.database.getDocumentByDocumentId(
remoteVersion.documentId
);
if (document === undefined) {
// Let's avoid the same documents getting created in parallel multiple times.
// There might be multiple tasks waiting for the lock
return this.remoteDocumentsLock.withLock(
remoteVersion.documentId,
async () => {
document = this.database.getDocumentByDocumentId(
remoteVersion.documentId
);
// We're either the first one to get the lock, so we have to create the document in `unrestrictedSyncRemotelyUpdatedFile`
if (document === undefined) {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
remoteVersion
)
);
} else {
const [promise, resolve, reject] = createPromise();
document =
await this.database.getResolvedDocumentByRelativePath(
document.relativePath,
promise
);
try {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
remoteVersion,
document
)
);
resolve();
} catch (e) {
reject(e);
} finally {
this.database.removeDocumentPromise(promise);
}
}
this.database.addSeenUpdateId(remoteVersion.vaultUpdateId);
}
);
}
// We're either the first one to get the lock, so we have to create the document in `unrestrictedSyncRemotelyUpdatedFile`
const [promise, resolve, reject] = createPromise();
document = await this.database.getResolvedDocumentByRelativePath(
document.relativePath,
promise
);
try {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
await this.enqueueSyncOperation(
async () =>
this.unrestrictedSyncer.unrestrictedSyncRemotelyUpdatedFile(
remoteVersion,
document
)
);
resolve();
} catch (e) {
reject(e);
} finally {
this.database.removeDocumentPromise(promise);
}
),
[
document?.relativePath,
remoteVersion.relativePath,
remoteVersion.documentId
]
);
this.database.addSeenUpdateId(remoteVersion.vaultUpdateId);
}
private async internalScheduleSyncForOfflineChanges(): Promise<void> {
await this.createFakeDocumentsFromRemoteState();
const allLocalFiles = await this.operations.listFilesRecursively();
this.logger.info(
`Scheduling sync for ${allLocalFiles.length} local files`
@ -409,7 +325,12 @@ export class Syncer {
}
}
await awaitAll(
interface Instruction {
type: "update" | "create";
relativePath: string;
oldPath?: string;
}
const instructions: (Instruction | undefined)[] = await awaitAll(
allLocalFiles.map(async (relativePath) => {
if (
this.database.getLatestDocumentByRelativePath(relativePath)
@ -419,16 +340,24 @@ export class Syncer {
`Document ${relativePath} might have been updated locally, scheduling sync to validate and update it`
);
return this.syncLocallyUpdatedFile({
relativePath
});
return { type: "update", relativePath } as Instruction;
}
// Perhaps the file has been moved; let's check by looking at the deleted files
const contentHash = await this.syncQueue.add(async () => {
const contentBytes =
await this.operations.read(relativePath); // this can throw FileNotFoundError
return hash(contentBytes);
try {
const contentBytes =
await this.operations.read(relativePath); // this can throw FileNotFoundError
return hash(contentBytes);
} catch (e) {
if (
e instanceof Error &&
e.name === "FileNotFoundError"
) {
return undefined;
}
throw e;
}
});
if (contentHash == undefined) {
@ -454,18 +383,21 @@ export class Syncer {
`Document '${originalFile.relativePath}' was not found under its current path in the database but was found under a different path (${relativePath}), scheduling sync to move it`
);
// We're outside of the pqueue, so we need to call the public wrapper
return this.syncLocallyUpdatedFile({
return {
type: "update",
oldPath: originalFile.relativePath,
relativePath
});
} as Instruction;
}
this.logger.debug(
`Document ${relativePath} not found in database, scheduling sync to create it`
);
// We're outside of the pqueue, so we need to call the public wrapper
return this.syncLocallyCreatedFile(relativePath);
return {
type: "create",
relativePath
} as Instruction;
})
);
@ -481,42 +413,49 @@ export class Syncer {
return this.syncLocallyDeletedFile(relativePath);
})
);
await awaitAll(
instructions.map(async (instruction) => {
if (instruction === undefined) {
return;
}
if (instruction.type === "update") {
// We're outside of the pqueue, so we need to call the public wrapper
await this.syncLocallyUpdatedFile({
oldPath: instruction.oldPath,
relativePath: instruction.relativePath
});
return;
}
})
);
// we have to ensure the deletes & updates have finished before starting creates,
// otherwise the server might return an existing document (that we're about to delete)
// instead of actually creating a new one
await awaitAll(
instructions.map(async (instruction) => {
if (instruction === undefined) {
return;
}
if (instruction.type === "create") {
// We're outside of the pqueue, so we need to call the public wrapper
await this.syncLocallyCreatedFile(instruction.relativePath);
return;
}
})
);
}
/**
* Create fake documents in the database for all files that are present locally
* and also exist remotely. This will stop the subequent syncs from duplicating
* the documents by creating the same documents from multiple clients.
*/
private async createFakeDocumentsFromRemoteState(): Promise<void> {
if (this.database.getHasInitialSyncCompleted()) {
return;
}
const [allLocalFiles, remote] = await awaitAll([
this.operations.listFilesRecursively(),
this.syncQueue.add(async () => this.syncService.getAll())
]);
if (remote !== undefined) {
remote.latestDocuments
.filter(
(remoteDocument) =>
allLocalFiles.includes(remoteDocument.relativePath) &&
!remoteDocument.isDeleted &&
this.database.getDocumentByDocumentId(
remoteDocument.documentId
) === undefined
)
.forEach((remoteDocument) => {
this.database.createNewEmptyDocument(
remoteDocument.documentId,
remoteDocument.vaultUpdateId,
remoteDocument.relativePath
);
});
}
this.database.setHasInitialSyncCompleted(true);
private async enqueueSyncOperation<T>(
operation: () => Promise<T>,
keys: (string | undefined | null)[]
): Promise<T> {
return this.updatedDocumentsByPathAndKeysLocks.withLock(
keys.filter((k) => k !== undefined && k !== null),
async () => this.syncQueue.add(operation)
);
}
}

580
frontend/sync-client/src/sync-operations/unrestricted-syncer.ts
View file

@ -3,7 +3,6 @@ import type {
DocumentRecord,
RelativePath
} from "../persistence/database";
import { diff } from "reconcile-text";
import type { SyncService } from "../services/sync-service";
import type { Logger } from "../tracing/logger";
@ -18,13 +17,11 @@ import type {
} from "../tracing/sync-history";
import { SyncStatus, SyncType } from "../tracing/sync-history";
import { EMPTY_HASH, hash } from "../utils/hash";
import { base64ToBytes } from "byte-base64";
import type { Settings } from "../persistence/settings";
import type { FileOperations } from "../file-operations/file-operations";
import { createPromise } from "../utils/create-promise";
import { FileNotFoundError } from "../file-operations/file-not-found-error";
import { SyncResetError } from "../services/sync-reset-error";
import { FileNotFoundError } from "../errors/file-not-found-error";
import { SyncResetError } from "../errors/sync-reset-error";
import { globsToRegexes } from "../utils/globs-to-regexes";
import type { DocumentVersion } from "../services/types/DocumentVersion";
import type { DocumentUpdateResponse } from "../services/types/DocumentUpdateResponse";
@ -60,65 +57,172 @@ export class UnrestrictedSyncer {
});
}
public async unrestrictedSyncLocallyCreatedFile(
document: DocumentRecord
): Promise<void> {
const updateDetails: SyncCreateDetails = {
type: SyncType.CREATE,
relativePath: document.relativePath
};
public async unrestrictedSyncLocallyCreatedOrUpdatedFile({
oldPath,
// We use the same code path for both local and remote updates. We need to force the update
// if there are no local changes but we know that the remote version is newer.
force = false,
document
}: {
oldPath?: RelativePath;
force?: boolean;
document: DocumentRecord;
}): Promise<void> {
const updateDetails:
| SyncCreateDetails
| SyncUpdateDetails
| SyncMovedDetails =
document.metadata === undefined
? {
type: SyncType.CREATE,
relativePath: document.relativePath
}
: oldPath !== undefined
? {
type: SyncType.MOVE,
relativePath: document.relativePath,
movedFrom: oldPath
}
: {
type: SyncType.UPDATE,
relativePath: document.relativePath
};
return this.executeSync(updateDetails, async () => {
await this.executeSync(updateDetails, async () => {
const originalRelativePath = document.relativePath;
if (document.isDeleted) {
this.logger.debug(
`Document ${originalRelativePath} has been already deleted, no need to create it`
`Document ${document.relativePath} has been already deleted, no need to update it`
);
return;
}
const contentBytes =
await this.operations.read(originalRelativePath); // this can throw FileNotFoundError
const contentBytes = await this.operations.read(
document.relativePath
); // this can throw FileNotFoundError
const contentHash = hash(contentBytes);
const response = await this.syncService.create({
documentId: document.documentId,
relativePath: originalRelativePath,
contentBytes
});
let response: DocumentVersion | DocumentUpdateResponse | undefined =
undefined;
if (document.metadata === undefined) {
response = await this.syncService.create({
relativePath: originalRelativePath,
contentBytes
});
// In case a document with the same name (but different ID) had existed remotely that we haven't known about
if (response.relativePath != originalRelativePath) {
this.logger.debug(
`Document ${originalRelativePath} has been created remotely at a different path: ${response.relativePath}, moving it locally`
);
await this.operations.move(
document.relativePath,
response.relativePath
); // this can throw FileNotFoundError
await this.handleMaybeMergingResponse({
document,
response,
contentHash,
originalRelativePath,
originalContentBytes: contentBytes,
isCreate: true
});
} else {
const areThereLocalChanges =
document.metadata.hash !== contentHash ||
oldPath !== undefined;
if (areThereLocalChanges) {
const isText =
!isBinary(contentBytes) &&
isFileTypeMergable(
document.relativePath,
(await this.serverConfig.getConfig())
.mergeableFileExtensions
);
const cachedVersion = this.contentCache.get(
document.metadata.parentVersionId
);
response =
isText && cachedVersion !== undefined
? await this.syncService.putText({
documentId: document.metadata.documentId,
parentVersionId:
document.metadata.parentVersionId,
relativePath: document.relativePath,
content: diff(
new TextDecoder().decode(cachedVersion),
new TextDecoder().decode(contentBytes)
)
})
: await this.syncService.putBinary({
documentId: document.metadata.documentId,
parentVersionId:
document.metadata.parentVersionId,
relativePath: document.relativePath,
contentBytes
});
} else {
if (!force) {
this.logger.debug(
`File hash of ${document.relativePath} matches with last synced version and the path hasn't changed; no need to sync`
);
return;
}
// we use this code path (force == true) to sync remotely updated files which have no local changes
response = await this.syncService.get({
documentId: document.metadata.documentId
});
}
await this.handleMaybeMergingResponse({
document,
response,
contentHash,
originalRelativePath,
originalContentBytes: contentBytes
});
}
this.database.updateDocumentMetadata(
{
parentVersionId: response.vaultUpdateId,
hash: contentHash,
remoteRelativePath: response.relativePath
},
document
);
if (!("type" in response) || response.type === "MergingUpdate") {
if (!force) {
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
details: updateDetails,
message: `The file we updated had been updated remotely, so we downloaded the merged version`
});
return;
}
}
this.database.addSeenUpdateId(response.vaultUpdateId);
await this.updateCache(
response.vaultUpdateId,
contentBytes,
response.relativePath
);
const actualUpdateDetails: SyncUpdateDetails | SyncMovedDetails =
oldPath !== undefined ||
response.relativePath != originalRelativePath
? {
type: SyncType.MOVE,
relativePath: response.relativePath,
movedFrom: originalRelativePath
}
: {
type: SyncType.UPDATE,
relativePath: response.relativePath
};
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
details: updateDetails,
message: `Successfully uploaded locally created file`
});
if (!response.isDeleted) {
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
details: actualUpdateDetails,
message: `Successfully downloaded remotely updated file from the server`,
author: response.userId,
timestamp: new Date(response.updatedDate)
});
} else {
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
details: {
type: SyncType.DELETE,
relativePath: document.relativePath
},
message:
"Successfully deleted file which had been deleted remotely",
author: response.userId,
timestamp: new Date(response.updatedDate)
});
}
});
}
@ -131,13 +235,21 @@ export class UnrestrictedSyncer {
};
await this.executeSync(updateDetails, async () => {
if (document.metadata === undefined) {
this.logger.debug(
`Document ${document.relativePath} has never been synced, no need to delete it remotely`
);
return;
}
const response = await this.syncService.delete({
documentId: document.documentId,
documentId: document.metadata.documentId,
relativePath: document.relativePath
});
this.database.updateDocumentMetadata(
{
documentId: response.documentId,
parentVersionId: response.vaultUpdateId,
hash: EMPTY_HASH,
remoteRelativePath: document.relativePath
@ -156,214 +268,6 @@ export class UnrestrictedSyncer {
});
}
public async unrestrictedSyncLocallyUpdatedFile({
oldPath,
document,
// We use the same code path for both local and remote updates. We need to force the update
// if there are no local changes but we know that the remote version is newer.
force = false
}: {
oldPath?: RelativePath;
force?: boolean;
document: DocumentRecord;
}): Promise<void> {
const updateDetails: SyncUpdateDetails | SyncMovedDetails =
oldPath !== undefined
? {
type: SyncType.MOVE,
relativePath: document.relativePath,
movedFrom: oldPath
}
: {
type: SyncType.UPDATE,
relativePath: document.relativePath
};
await this.executeSync(updateDetails, async () => {
const originalRelativePath = document.relativePath;
if (document.isDeleted || document.metadata === undefined) {
this.logger.debug(
`Document ${document.relativePath} has been already deleted, no need to update it`
);
return;
}
const contentBytes = await this.operations.read(
document.relativePath
); // this can throw FileNotFoundError
let contentHash = hash(contentBytes);
const areThereLocalChanges = !(
document.metadata.hash === contentHash && oldPath === undefined
);
let response: DocumentVersion | DocumentUpdateResponse | undefined =
undefined;
if (areThereLocalChanges) {
const isText =
!isBinary(contentBytes) &&
isFileTypeMergable(
document.relativePath,
(await this.serverConfig.getConfig())
.mergeableFileExtensions
);
const cachedVersion = this.contentCache.get(
document.metadata.parentVersionId
);
response =
isText && cachedVersion !== undefined
? await this.syncService.putText({
documentId: document.documentId,
parentVersionId:
document.metadata.parentVersionId,
relativePath: document.relativePath,
content: diff(
new TextDecoder().decode(cachedVersion),
new TextDecoder().decode(contentBytes)
)
})
: await this.syncService.putBinary({
documentId: document.documentId,
parentVersionId:
document.metadata.parentVersionId,
relativePath: document.relativePath,
contentBytes
});
} else {
if (!force) {
this.logger.debug(
`File hash of ${document.relativePath} matches with last synced version and the path hasn't changed; no need to sync`
);
return;
}
response = await this.syncService.get({
documentId: document.documentId
});
}
// `document` is mutable and reflects the latest state in the local database
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
if (document.isDeleted) {
this.logger.info(
`Document ${document.relativePath} has been deleted before we could finish updating it`
);
this.database.addSeenUpdateId(response.vaultUpdateId);
return;
}
if (
// `Syncer` creates fake local document metadata for all remote docs with invalid hashes. The parent IDs will likely match
// the latest versions so we still need to update the local versions to turn the fakes into real metadata.
document.metadata.parentVersionId > response.vaultUpdateId
) {
this.logger.debug(
`Document ${document.relativePath} is already more up to date than the fetched version`
);
this.database.addSeenUpdateId(response.vaultUpdateId); // in case the previous `vaultUpdateId` update hasn't made it through
return;
}
if (response.isDeleted) {
return this.applyRemoteDeleteLocally(document, response);
}
let actualPath = document.relativePath;
if (response.relativePath != originalRelativePath) {
actualPath = response.relativePath;
// Make sure to update the remote relative path to avoid uploading
// the file as a result of this filesystem event.
document.metadata.remoteRelativePath = response.relativePath;
await this.operations.move(
document.relativePath,
response.relativePath
); // this can throw FileNotFoundError
}
if (!("type" in response) || response.type === "MergingUpdate") {
const responseBytes = base64ToBytes(response.contentBase64);
contentHash = hash(responseBytes);
this.database.updateDocumentMetadata(
{
parentVersionId: response.vaultUpdateId,
hash: contentHash,
remoteRelativePath: response.relativePath
},
document
);
await this.operations.write(
actualPath,
contentBytes,
responseBytes
);
await this.updateCache(
response.vaultUpdateId,
responseBytes,
actualPath
);
if (!force) {
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
details: updateDetails,
message: `The file we updated had been updated remotely, so we downloaded the merged version`
});
}
} else {
this.database.updateDocumentMetadata(
{
parentVersionId: response.vaultUpdateId,
hash: contentHash,
remoteRelativePath: response.relativePath
},
document
);
await this.updateCache(
response.vaultUpdateId,
contentBytes,
actualPath
);
}
this.database.addSeenUpdateId(response.vaultUpdateId);
const actualUpdateDetails: SyncUpdateDetails | SyncMovedDetails =
oldPath !== undefined ||
response.relativePath != originalRelativePath
? {
type: SyncType.MOVE,
relativePath: response.relativePath,
movedFrom: originalRelativePath
}
: {
type: SyncType.UPDATE,
relativePath: response.relativePath
};
if (areThereLocalChanges) {
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
details: actualUpdateDetails,
message: `Successfully uploaded locally updated file to the server`,
author: response.userId
});
} else {
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
details: actualUpdateDetails,
message: `Successfully downloaded remotely updated file from the server`,
author: response.userId,
timestamp: new Date(response.updatedDate)
});
}
});
}
public async unrestrictedSyncRemotelyUpdatedFile(
remoteVersion: DocumentVersionWithoutContent,
document?: DocumentRecord
@ -382,13 +286,13 @@ export class UnrestrictedSyncer {
remoteVersion.vaultUpdateId
) {
this.logger.debug(
`Document ${remoteVersion.relativePath} is already at least as up to date as the fetched version`
`Document ${document.relativePath} is already at least as up-to-date as the fetched version`
);
return;
}
return this.unrestrictedSyncLocallyUpdatedFile({
return this.unrestrictedSyncLocallyCreatedOrUpdatedFile({
document,
force: true
});
@ -434,17 +338,15 @@ export class UnrestrictedSyncer {
await this.operations.ensureClearPath(remoteVersion.relativePath);
const [promise, resolve] = createPromise();
this.database.updateDocumentMetadata(
{
documentId: remoteVersion.documentId,
parentVersionId: remoteVersion.vaultUpdateId,
hash: hash(contentBytes),
remoteRelativePath: remoteVersion.relativePath
},
this.database.createNewPendingDocument(
remoteVersion.documentId,
remoteVersion.relativePath,
promise
remoteVersion.relativePath
)
);
@ -458,9 +360,6 @@ export class UnrestrictedSyncer {
remoteVersion.relativePath
);
resolve();
this.database.removeDocumentPromise(promise);
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
details: updateDetails,
@ -471,10 +370,17 @@ export class UnrestrictedSyncer {
});
}
public async executeSync<T>(
private async executeSync<T>(
details: SyncDetails,
fn: () => Promise<T>
): Promise<T | undefined> {
if (!this.settings.getSettings().isSyncEnabled) {
this.logger.info(
`Skipping sync operation for file '${details.relativePath}' because sync is disabled`
);
return;
}
for (const pattern of this.ignorePatterns) {
if (pattern.test(details.relativePath)) {
this.logger.debug(
@ -528,6 +434,127 @@ export class UnrestrictedSyncer {
}
}
private async handleMaybeMergingResponse({
document,
response,
contentHash,
originalRelativePath,
originalContentBytes,
isCreate
}: {
document: DocumentRecord;
response: DocumentVersion | DocumentUpdateResponse;
contentHash: string;
originalRelativePath: string;
originalContentBytes: Uint8Array;
isCreate?: boolean;
}): Promise<void> {
// `document` is mutable and reflects the latest state in the local database
if (document.isDeleted) {
this.logger.info(
`Document ${document.relativePath} has been deleted before we could finish updating it`
);
this.database.addSeenUpdateId(response.vaultUpdateId);
return;
}
if (
(document.metadata?.parentVersionId ?? 0) > response.vaultUpdateId
) {
this.logger.debug(
`Document ${document.relativePath} is already more up to date than the fetched version`
);
this.database.addSeenUpdateId(response.vaultUpdateId); // in case the previous `vaultUpdateId` update hasn't made it through
return;
}
if (response.isDeleted) {
return this.applyRemoteDeleteLocally(document, response);
}
let actualPath = document.relativePath;
if (isCreate) {
// We have a file locally that got moved by another client to the same path as the one we're trying to create.
// The server returns a merging update for the document ID that already exists locally (but at another path).
// We have to merge these two documents by extending the provenance of the existing document and deleting
// the old document that the new document already contains the content for.
const existingDocument = this.database.getDocumentByDocumentId(
response.documentId
);
if (existingDocument !== undefined) {
this.logger.info(
`Merging existing document ${existingDocument.relativePath} into ${document.relativePath
} after concurrent move & creation`
);
if (!existingDocument.isDeleted) {
this.database.delete(existingDocument.relativePath); // make sure syncLocallyDeletedFile doesn't actually schedule deleting the new file
this.database.removeDocument(existingDocument);
await this.operations.move(existingDocument.relativePath, document.relativePath);
} else {
this.database.removeDocument(existingDocument);
}
}
}
// this can't happen on the creation path as we can only get a merging response if a document already exists remotely on the same path
if (response.relativePath != originalRelativePath) {
actualPath = response.relativePath;
// Make sure to update the remote relative path to avoid uploading
// the file as a result of this filesystem event.
if (document.metadata !== undefined) {
document.metadata.remoteRelativePath = response.relativePath;
}
await this.operations.move(
document.relativePath,
response.relativePath
); // this can throw FileNotFoundError
}
if (!("type" in response) || response.type === "MergingUpdate") {
const responseBytes = base64ToBytes(response.contentBase64);
contentHash = hash(responseBytes);
this.database.updateDocumentMetadata(
{
documentId: response.documentId,
parentVersionId: response.vaultUpdateId,
hash: contentHash,
remoteRelativePath: response.relativePath
},
document
);
await this.operations.write(
actualPath,
originalContentBytes,
responseBytes
);
await this.updateCache(
response.vaultUpdateId,
responseBytes,
actualPath
);
} else {
this.database.updateDocumentMetadata(
{
documentId: response.documentId,
parentVersionId: response.vaultUpdateId,
hash: contentHash,
remoteRelativePath: response.relativePath
},
document
);
await this.updateCache(
response.vaultUpdateId,
originalContentBytes,
actualPath
);
}
this.database.addSeenUpdateId(response.vaultUpdateId);
}
private getHistoryEntryForSkippedOversizedFile(
sizeInBytes: number,
relativePath: RelativePath
@ -541,9 +568,8 @@ export class UnrestrictedSyncer {
type: SyncType.SKIPPED,
relativePath
},
message: `File size of ${sizeInMB} MB exceeds the maximum file size limit of ${
maxFileSizeMB
} MB`
message: `File size of ${sizeInMB} MB exceeds the maximum file size limit of ${maxFileSizeMB
} MB`
};
}
}
@ -568,20 +594,10 @@ export class UnrestrictedSyncer {
document: DocumentRecord,
response: DocumentVersion | DocumentUpdateResponse
): Promise<void> {
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
details: {
type: SyncType.DELETE,
relativePath: document.relativePath
},
message: "File has been deleted remotely, so we deleted it locally",
author: response.userId,
timestamp: new Date(response.updatedDate)
});
this.database.delete(document.relativePath);
this.database.updateDocumentMetadata(
{
documentId: response.documentId,
parentVersionId: response.vaultUpdateId,
hash: EMPTY_HASH,
remoteRelativePath: response.relativePath