andras/vault-link
1
0
Fork 0
Code Issues Pull requests 18 Projects Releases 55 Packages Wiki Activity Actions 1

Start fixing tests

Browse source
This commit is contained in:
Andras Schmelczer 2026-01-24 11:00:55 +00:00
parent 727b6b7ed5
commit 7fcd0f0bfa
19 changed files with 210 additions and 218 deletions
Download patch file Download diff file Expand all files Collapse all files

106
frontend/sync-client/src/sync-operations/syncer.ts
View file

@ -11,7 +11,6 @@ import type { Settings } from "../persistence/settings";
import type { FileOperations } from "../file-operations/file-operations";
import { findMatchingFile } from "../utils/find-matching-file";
import type { UnrestrictedSyncer } from "./unrestricted-syncer";
import { createPromise } from "../utils/create-promise";
import { SyncResetError } from "../errors/sync-reset-error";
import { Locks } from "../utils/data-structures/locks";
import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent";
@ -21,14 +20,12 @@ import type { WebSocketClientMessage } from "../services/types/WebSocketClientMe
import { awaitAll } from "../utils/await-all";
import { EventListeners } from "../utils/data-structures/event-listeners";
export const __debug_locks: Locks<any>[] = []; // Used only for debugging timeouts
export class Syncer {
public readonly onRemainingOperationsCountChanged = new EventListeners<
(remainingOperations: number) => unknown
>();
public readonly updatedDocumentsByPathAndKeysLock: Locks<DocumentId | RelativePath>;
public readonly updatedDocumentsByPathAndKeysLocks: Locks<string>; // can be DocumentId or RelativePath
// FIFO to limit the number of concurrent sync operations
private readonly syncQueue: PQueue;
@ -50,8 +47,9 @@ export class Syncer {
concurrency: settings.getSettings().syncConcurrency
});
this.updatedDocumentsByPathAndKeysLock = new Locks<DocumentId>(this.logger);
__debug_locks.push(this.updatedDocumentsByPathAndKeysLock); // Used only for debugging timeouts
this.updatedDocumentsByPathAndKeysLocks = new Locks<DocumentId>(
this.logger
);
settings.onSettingsChanged.add((newSettings, oldSettings) => {
if (newSettings.syncConcurrency !== oldSettings.syncConcurrency) {
@ -84,7 +82,7 @@ export class Syncer {
}
public hasPendingOperationsForDocument(relativePath: string): boolean {
return this.updatedDocumentsByPathAndKeysLock.isLocked(relativePath);
return this.updatedDocumentsByPathAndKeysLocks.isLocked(relativePath);
}
public async syncLocallyCreatedFile(
@ -102,29 +100,26 @@ export class Syncer {
return;
}
const document = this.database.createNewPendingDocument(
relativePath
);
const document = this.database.createNewPendingDocument(relativePath);
await this.enqueueSyncOperation(async () =>
this.unrestrictedSyncer.unrestrictedSyncLocallyCreatedOrUpdatedFile(
{
document
}
), [relativePath]
await this.enqueueSyncOperation(
async () =>
this.unrestrictedSyncer.unrestrictedSyncLocallyCreatedOrUpdatedFile(
{
document
}
),
[relativePath]
);
}
public async syncLocallyDeletedFile(
relativePath: RelativePath
): Promise<void> {
const document = this.database.getLatestDocumentByRelativePath(relativePath);
let document =
this.database.getLatestDocumentByRelativePath(relativePath);
if (
document
?.isDeleted === true
) {
if (document == null || document.isDeleted === true) {
// This is must be a consequence of us deleting a file because of a remote update
// which triggered a local delete, so we don't need to do anything here.
this.logger.debug(
@ -137,25 +132,13 @@ export class Syncer {
// document which finishes after the delete has succeeded and would introduce a phantom metadata record.
this.database.delete(relativePath);
await this.enqueueSyncOperation(async () => {
const document = this.database.getLatestDocumentByRelativePath(relativePath);
if (document === undefined) {
this.logger.debug(
`Cannot find document ${relativePath} in the database, must have been deleted already, skipping`
);
return;
}
await this.unrestrictedSyncer.unrestrictedSyncLocallyDeletedFile(
document
);
this.database.removeDocument(document);
}, [document?.metadata?.documentId, relativePath]
);
}, [document?.metadata?.documentId, relativePath]);
}
public async syncLocallyUpdatedFile({
@ -165,18 +148,15 @@ export class Syncer {
oldPath?: RelativePath;
relativePath: RelativePath;
}): Promise<void> {
const documentAtNewPath = this.database.getLatestDocumentByRelativePath(
relativePath
);
const documentAtNewPath =
this.database.getLatestDocumentByRelativePath(relativePath);
if (oldPath !== undefined) {
// We might have moved the document in the database before calling this method,
// in that case, we mustn't move it again.
if (
documentAtNewPath ===
undefined ||
documentAtNewPath
?.isDeleted === true
documentAtNewPath === undefined ||
documentAtNewPath.isDeleted
) {
if (oldPath === relativePath) {
throw new Error(
@ -188,7 +168,7 @@ export class Syncer {
}
}
let document =
const document =
this.database.getLatestDocumentByRelativePath(relativePath);
if (
@ -216,17 +196,16 @@ export class Syncer {
return;
}
await this.enqueueSyncOperation(async () =>
this.unrestrictedSyncer.unrestrictedSyncLocallyCreatedOrUpdatedFile(
{
oldPath,
document
}
), [document.metadata?.documentId, relativePath, oldPath]
await this.enqueueSyncOperation(
async () =>
this.unrestrictedSyncer.unrestrictedSyncLocallyCreatedOrUpdatedFile(
{
oldPath,
document
}
),
[document.metadata?.documentId, relativePath, oldPath]
);
}
public async scheduleSyncForOfflineChanges(): Promise<void> {
@ -290,7 +269,7 @@ export class Syncer {
public reset(): void {
this._isFirstSyncComplete = false;
this.syncQueue.clear();
this.updatedDocumentsByPathAndKeysLock.reset();
this.updatedDocumentsByPathAndKeysLocks.reset();
this.runningScheduleSyncForOfflineChanges = undefined;
}
@ -310,13 +289,17 @@ export class Syncer {
const document = this.database.getDocumentByDocumentId(
remoteVersion.documentId
);
this.enqueueSyncOperation(async () =>
await this.syncQueue.add(async () =>
await this.enqueueSyncOperation(
async () =>
this.unrestrictedSyncer.unrestrictedSyncRemotelyUpdatedFile(
remoteVersion,
document
)
), [document?.relativePath, remoteVersion.relativePath, remoteVersion.documentId]
),
[
document?.relativePath,
remoteVersion.relativePath,
remoteVersion.documentId
]
);
this.database.addSeenUpdateId(remoteVersion.vaultUpdateId);
@ -465,10 +448,11 @@ export class Syncer {
private async enqueueSyncOperation<T>(
operation: () => Promise<T>,
keys: Array<DocumentId | RelativePath | undefined | null>
keys: (DocumentId | undefined | null)[]
): Promise<T> {
return this.updatedDocumentsByPathAndKeysLock.withLock(keys.filter(k => k !== undefined && k !== null), async () =>
this.syncQueue.add(operation)
return this.updatedDocumentsByPathAndKeysLocks.withLock(
keys.filter((k) => k !== undefined && k !== null),
async () => this.syncQueue.add(operation)
);
}
}

71
frontend/sync-client/src/sync-operations/unrestricted-syncer.ts
View file

@ -3,7 +3,6 @@ import type {
DocumentRecord,
RelativePath
} from "../persistence/database";
import { diff } from "reconcile-text";
import type { SyncService } from "../services/sync-service";
import type { Logger } from "../tracing/logger";
@ -18,11 +17,9 @@ import type {
} from "../tracing/sync-history";
import { SyncStatus, SyncType } from "../tracing/sync-history";
import { EMPTY_HASH, hash } from "../utils/hash";
import { base64ToBytes } from "byte-base64";
import type { Settings } from "../persistence/settings";
import type { FileOperations } from "../file-operations/file-operations";
import { createPromise } from "../utils/create-promise";
import { FileNotFoundError } from "../errors/file-not-found-error";
import { SyncResetError } from "../errors/sync-reset-error";
import { globsToRegexes } from "../utils/globs-to-regexes";
@ -33,7 +30,6 @@ import type { FixedSizeDocumentCache } from "../utils/data-structures/fix-sized-
import { isFileTypeMergable } from "../utils/is-file-type-mergable";
import { isBinary } from "../utils/is-binary";
import type { ServerConfig } from "../services/server-config";
import { Locks } from "../utils/data-structures/locks";
export class UnrestrictedSyncer {
private ignorePatterns: RegExp[];
@ -66,7 +62,7 @@ export class UnrestrictedSyncer {
// We use the same code path for both local and remote updates. We need to force the update
// if there are no local changes but we know that the remote version is newer.
force = false,
document,
document
}: {
oldPath?: RelativePath;
force?: boolean;
@ -123,7 +119,6 @@ export class UnrestrictedSyncer {
originalContentBytes: contentBytes,
isCreate: true
});
} else {
const areThereLocalChanges =
document.metadata.hash !== contentHash ||
@ -351,7 +346,7 @@ export class UnrestrictedSyncer {
remoteRelativePath: remoteVersion.relativePath
},
this.database.createNewPendingDocument(
remoteVersion.relativePath,
remoteVersion.relativePath
)
);
@ -365,7 +360,6 @@ export class UnrestrictedSyncer {
remoteVersion.relativePath
);
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
details: updateDetails,
@ -376,8 +370,6 @@ export class UnrestrictedSyncer {
});
}
private async executeSync<T>(
details: SyncDetails,
fn: () => Promise<T>
@ -481,9 +473,9 @@ export class UnrestrictedSyncer {
}
let actualPath = document.relativePath;
let mustCreate = false;
if (isCreate === true) {
if (isCreate) {
// We have a file locally that got moved by another client to the same path as the one we're trying to create.
// The server returns a merging update for the document ID that already exists locally (but at another path).
// We have to merge these two documents by extending the provenance of the existing document and deleting
@ -492,14 +484,18 @@ export class UnrestrictedSyncer {
response.documentId
);
if (existingDocument !== undefined) {
this.logger.info(`Merging document ${existingDocument.relativePath} into existing document ${document.relativePath} after concurrent move & creation`);
this.logger.info(
`Merging document ${existingDocument.relativePath} into existing document ${document.relativePath
} after concurrent move & creation`
);
this.database.removeDocument(document); // this was a (fake) pending document
if (!existingDocument.isDeleted) {
this.operations.delete(document.relativePath);
this.database.delete(existingDocument.relativePath); // make sure syncLocallyDeletedFile doesn't actually schedule deleting the new file
await this.operations.delete(existingDocument.relativePath);
}
mustCreate = true;
document = existingDocument;
}
}
// this can't happen on the creation path as we can only get a merging response if a document already exists remotely on the same path
@ -516,26 +512,41 @@ export class UnrestrictedSyncer {
); // this can throw FileNotFoundError
}
if (!("type" in response) || response.type === "MergingUpdate") {
const responseBytes = base64ToBytes(response.contentBase64);
contentHash = hash(responseBytes);
this.database.updateDocumentMetadata(
{
documentId: response.documentId,
parentVersionId: response.vaultUpdateId,
hash: contentHash,
remoteRelativePath: response.relativePath
},
document
);
await this.operations.write(
actualPath,
originalContentBytes,
responseBytes
);
if (mustCreate) {
this.database.createNewPendingDocument(actualPath);
this.database.updateDocumentMetadata(
{
documentId: response.documentId,
parentVersionId: response.vaultUpdateId,
hash: contentHash,
remoteRelativePath: response.relativePath
},
document
);
await this.operations.create(actualPath, responseBytes);
} else {
this.database.updateDocumentMetadata(
{
documentId: response.documentId,
parentVersionId: response.vaultUpdateId,
hash: contentHash,
remoteRelativePath: response.relativePath
},
document
);
await this.operations.write(
actualPath,
originalContentBytes,
responseBytes
);
}
await this.updateCache(
response.vaultUpdateId,
responseBytes,