andras/vault-link
andras/vault-link
1
0
Fork 0
Code Issues Pull requests 19 Projects Releases 55 Packages Wiki Activity Actions 28

Start fixing tests

Browse source
This commit is contained in:
Andras Schmelczer 2026-01-24 11:00:55 +00:00
commit 7fcd0f0bfa
19 changed files with 209 additions and 217 deletions
Download patch file Download diff file Expand all files Collapse all files

1
README.md
View file

@ -80,5 +80,4 @@ And to clean up the logs & database files, run `task clean`
- [Sync server](./sync-server/README.md) - [Sync server](./sync-server/README.md)
remove force merge everywhere remove force merge everywhere

2
frontend/deterministic-tests/package.json
View file

@ -19,4 +19,4 @@
"webpack": "^5.103.0", "webpack": "^5.103.0",
"webpack-cli": "^6.0.1" "webpack-cli": "^6.0.1"
} }
} }

4
frontend/deterministic-tests/src/cli.ts
View file

@ -80,10 +80,10 @@ async function main(): Promise<void> {
if (!result.success) { if (!result.success) {
allPassed = false; allPassed = false;
logger.error(`\n✗ FAILED: ${test.name}`); logger.error(`✗ FAILED: ${test.name}`);
logger.error(`Error: ${result.error}`); logger.error(`Error: ${result.error}`);
} else { } else {
logger.info(`\n✓ PASSED: ${test.name} (${result.duration}ms)`); logger.info(`✓ PASSED: ${test.name} (${result.duration}ms)`);
} }
} }
} finally { } finally {

3
frontend/deterministic-tests/src/test-runner.ts
View file

@ -46,12 +46,11 @@ export class TestRunner {
for (let i = 0; i < test.steps.length; i++) { for (let i = 0; i < test.steps.length; i++) {
const step = test.steps[i]; const step = test.steps[i];
this.logger.info( this.logger.info(
`\nStep ${i + 1}/${test.steps.length}: ${JSON.stringify(step)}` `Step ${i + 1}/${test.steps.length}: ${JSON.stringify(step)}`
); );
await this.executeStep(step); await this.executeStep(step);
} }
// Cleanup
await this.cleanup(); await this.cleanup();
const duration = Date.now() - startTime; const duration = Date.now() - startTime;

2
frontend/local-client-cli/src/cli.ts
View file

@ -66,7 +66,7 @@ async function main(): Promise<void> {
console.log( console.log(
styleText("VaultLink Local CLI", "bold", "cyan") + styleText("VaultLink Local CLI", "bold", "cyan") +
colorize(` v${packageJson.version}`, "dim") colorize(` v${packageJson.version}`, "dim")
); );
console.log(colorize("=".repeat(50), "dim")); console.log(colorize("=".repeat(50), "dim"));
console.log( console.log(

2
frontend/package.json
View file

@ -41,4 +41,4 @@
"prettier": "^3.7.4", "prettier": "^3.7.4",
"typescript-eslint": "8.49.0" "typescript-eslint": "8.49.0"
} }
} }

3
frontend/sync-client/src/index.ts
View file

@ -33,14 +33,13 @@ export type { AuthenticationError } from "./errors/authentication-error";
export type { MaybeOutdatedClientCursors } from "./types/maybe-outdated-client-cursors"; export type { MaybeOutdatedClientCursors } from "./types/maybe-outdated-client-cursors";
export { DocumentSyncStatus } from "./types/document-sync-status"; export { DocumentSyncStatus } from "./types/document-sync-status";
export { SyncClient } from "./sync-client"; export { SyncClient } from "./sync-client";
export { __debug_locks } from "./sync-operations/syncer";
export type { TextWithCursors, CursorPosition } from "reconcile-text"; export type { TextWithCursors, CursorPosition } from "reconcile-text";
export const debugging = { export const debugging = {
slowFetchFactory, slowFetchFactory,
slowWebSocketFactory, slowWebSocketFactory,
logToConsole, logToConsole,
InMemoryFileSystem, InMemoryFileSystem
}; };
export const utils = { export const utils = {

15
frontend/sync-client/src/persistence/database.ts
View file

@ -101,7 +101,7 @@ export class Database {
i === 0 i === 0
? false ? false
: records[i - 1].parallelVersion === : records[i - 1].parallelVersion ===
current.parallelVersion current.parallelVersion
) )
) { ) {
throw new Error( throw new Error(
@ -130,11 +130,7 @@ export class Database {
target.metadata, target.metadata,
null, null,
2 2
)} to ${JSON.stringify( )} to ${JSON.stringify(metadata, null, 2)}`
metadata,
null,
2
)}`
); );
target.metadata = metadata; target.metadata = metadata;
@ -142,7 +138,6 @@ export class Database {
this.saveInTheBackground(); this.saveInTheBackground();
} }
public getLatestDocumentByRelativePath( public getLatestDocumentByRelativePath(
find: RelativePath find: RelativePath
): DocumentRecord | undefined { ): DocumentRecord | undefined {
@ -153,9 +148,8 @@ export class Database {
return candidates[0]; return candidates[0];
} }
public createNewPendingDocument( public createNewPendingDocument(
relativePath: RelativePath, relativePath: RelativePath
): DocumentRecord { ): DocumentRecord {
this.logger.debug(`Creating new pending document: ${relativePath}`); this.logger.debug(`Creating new pending document: ${relativePath}`);
const previousEntry = const previousEntry =
@ -223,7 +217,6 @@ export class Database {
candidate.isDeleted = true; candidate.isDeleted = true;
} }
public removeDocument(find: DocumentRecord): void { public removeDocument(find: DocumentRecord): void {
removeFromArray(this.documents, find); removeFromArray(this.documents, find);
this.saveInTheBackground(); this.saveInTheBackground();
@ -294,7 +287,7 @@ export class Database {
if (duplicates.length > 0) { if (duplicates.length > 0) {
throw new Error( throw new Error(
"Document IDs are not unique, found duplicates: " + "Document IDs are not unique, found duplicates: " +
duplicates.join("; ") duplicates.join("; ")
); );
} }
} }

5
frontend/sync-client/src/sync-client.ts
View file

@ -40,7 +40,6 @@ export class SyncClient {
private readonly history: SyncHistory, private readonly history: SyncHistory,
private readonly settings: Settings, private readonly settings: Settings,
private readonly database: Database, private readonly database: Database,
private readonly unrestrictedSyncer: UnrestrictedSyncer,
private readonly syncer: Syncer, private readonly syncer: Syncer,
private readonly webSocketManager: WebSocketManager, private readonly webSocketManager: WebSocketManager,
public readonly logger: Logger, public readonly logger: Logger,
@ -56,7 +55,7 @@ export class SyncClient {
database: Partial<StoredDatabase>; database: Partial<StoredDatabase>;
}> }>
> >
) { } ) {}
public get documentCount(): number { public get documentCount(): number {
return this.database.length; return this.database.length;
@ -221,7 +220,6 @@ export class SyncClient {
history, history,
settings, settings,
database, database,
unrestrictedSyncer,
syncer, syncer,
webSocketManager, webSocketManager,
logger, logger,
@ -410,7 +408,6 @@ export class SyncClient {
return DocumentSyncStatus.SYNCING; return DocumentSyncStatus.SYNCING;
} }
return this.syncer.hasPendingOperationsForDocument(relativePath) return this.syncer.hasPendingOperationsForDocument(relativePath)
? DocumentSyncStatus.SYNCING ? DocumentSyncStatus.SYNCING
: DocumentSyncStatus.UP_TO_DATE; : DocumentSyncStatus.UP_TO_DATE;

106
frontend/sync-client/src/sync-operations/syncer.ts
View file

@ -11,7 +11,6 @@ import type { Settings } from "../persistence/settings";
import type { FileOperations } from "../file-operations/file-operations"; import type { FileOperations } from "../file-operations/file-operations";
import { findMatchingFile } from "../utils/find-matching-file"; import { findMatchingFile } from "../utils/find-matching-file";
import type { UnrestrictedSyncer } from "./unrestricted-syncer"; import type { UnrestrictedSyncer } from "./unrestricted-syncer";
import { createPromise } from "../utils/create-promise";
import { SyncResetError } from "../errors/sync-reset-error"; import { SyncResetError } from "../errors/sync-reset-error";
import { Locks } from "../utils/data-structures/locks"; import { Locks } from "../utils/data-structures/locks";
import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent"; import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent";
@ -21,14 +20,12 @@ import type { WebSocketClientMessage } from "../services/types/WebSocketClientMe
import { awaitAll } from "../utils/await-all"; import { awaitAll } from "../utils/await-all";
import { EventListeners } from "../utils/data-structures/event-listeners"; import { EventListeners } from "../utils/data-structures/event-listeners";
export const __debug_locks: Locks<any>[] = []; // Used only for debugging timeouts
export class Syncer { export class Syncer {
public readonly onRemainingOperationsCountChanged = new EventListeners< public readonly onRemainingOperationsCountChanged = new EventListeners<
(remainingOperations: number) => unknown (remainingOperations: number) => unknown
>(); >();
public readonly updatedDocumentsByPathAndKeysLock: Locks<DocumentId | RelativePath>; public readonly updatedDocumentsByPathAndKeysLocks: Locks<string>; // can be DocumentId or RelativePath
// FIFO to limit the number of concurrent sync operations // FIFO to limit the number of concurrent sync operations
private readonly syncQueue: PQueue; private readonly syncQueue: PQueue;
@ -50,8 +47,9 @@ export class Syncer {
concurrency: settings.getSettings().syncConcurrency concurrency: settings.getSettings().syncConcurrency
}); });
this.updatedDocumentsByPathAndKeysLock = new Locks<DocumentId>(this.logger); this.updatedDocumentsByPathAndKeysLocks = new Locks<DocumentId>(
__debug_locks.push(this.updatedDocumentsByPathAndKeysLock); // Used only for debugging timeouts this.logger
);
settings.onSettingsChanged.add((newSettings, oldSettings) => { settings.onSettingsChanged.add((newSettings, oldSettings) => {
if (newSettings.syncConcurrency !== oldSettings.syncConcurrency) { if (newSettings.syncConcurrency !== oldSettings.syncConcurrency) {
@ -84,7 +82,7 @@ export class Syncer {
} }
public hasPendingOperationsForDocument(relativePath: string): boolean { public hasPendingOperationsForDocument(relativePath: string): boolean {
return this.updatedDocumentsByPathAndKeysLock.isLocked(relativePath); return this.updatedDocumentsByPathAndKeysLocks.isLocked(relativePath);
} }
public async syncLocallyCreatedFile( public async syncLocallyCreatedFile(
@ -102,29 +100,26 @@ export class Syncer {
return; return;
} }
const document = this.database.createNewPendingDocument( const document = this.database.createNewPendingDocument(relativePath);
relativePath
);
await this.enqueueSyncOperation(async () => await this.enqueueSyncOperation(
this.unrestrictedSyncer.unrestrictedSyncLocallyCreatedOrUpdatedFile( async () =>
{ this.unrestrictedSyncer.unrestrictedSyncLocallyCreatedOrUpdatedFile(
document {
} document
), [relativePath] }
),
[relativePath]
); );
} }
public async syncLocallyDeletedFile( public async syncLocallyDeletedFile(
relativePath: RelativePath relativePath: RelativePath
): Promise<void> { ): Promise<void> {
const document = this.database.getLatestDocumentByRelativePath(relativePath); let document =
this.database.getLatestDocumentByRelativePath(relativePath);
if (document == null || document.isDeleted === true) {
if (
document
?.isDeleted === true
) {
// This is must be a consequence of us deleting a file because of a remote update // This is must be a consequence of us deleting a file because of a remote update
// which triggered a local delete, so we don't need to do anything here. // which triggered a local delete, so we don't need to do anything here.
this.logger.debug( this.logger.debug(
@ -137,25 +132,13 @@ export class Syncer {
// document which finishes after the delete has succeeded and would introduce a phantom metadata record. // document which finishes after the delete has succeeded and would introduce a phantom metadata record.
this.database.delete(relativePath); this.database.delete(relativePath);
await this.enqueueSyncOperation(async () => { await this.enqueueSyncOperation(async () => {
const document = this.database.getLatestDocumentByRelativePath(relativePath);
if (document === undefined) {
this.logger.debug(
`Cannot find document ${relativePath} in the database, must have been deleted already, skipping`
);
return;
}
await this.unrestrictedSyncer.unrestrictedSyncLocallyDeletedFile( await this.unrestrictedSyncer.unrestrictedSyncLocallyDeletedFile(
document document
); );
this.database.removeDocument(document); this.database.removeDocument(document);
}, [document?.metadata?.documentId, relativePath] }, [document?.metadata?.documentId, relativePath]);
);
} }
public async syncLocallyUpdatedFile({ public async syncLocallyUpdatedFile({
@ -165,18 +148,15 @@ export class Syncer {
oldPath?: RelativePath; oldPath?: RelativePath;
relativePath: RelativePath; relativePath: RelativePath;
}): Promise<void> { }): Promise<void> {
const documentAtNewPath = this.database.getLatestDocumentByRelativePath( const documentAtNewPath =
relativePath this.database.getLatestDocumentByRelativePath(relativePath);
);
if (oldPath !== undefined) { if (oldPath !== undefined) {
// We might have moved the document in the database before calling this method, // We might have moved the document in the database before calling this method,
// in that case, we mustn't move it again. // in that case, we mustn't move it again.
if ( if (
documentAtNewPath === documentAtNewPath === undefined ||
undefined || documentAtNewPath.isDeleted
documentAtNewPath
?.isDeleted === true
) { ) {
if (oldPath === relativePath) { if (oldPath === relativePath) {
throw new Error( throw new Error(
@ -188,7 +168,7 @@ export class Syncer {
} }
} }
let document = const document =
this.database.getLatestDocumentByRelativePath(relativePath); this.database.getLatestDocumentByRelativePath(relativePath);
if ( if (
@ -216,17 +196,16 @@ export class Syncer {
return; return;
} }
await this.enqueueSyncOperation(
await this.enqueueSyncOperation(async () => async () =>
this.unrestrictedSyncer.unrestrictedSyncLocallyCreatedOrUpdatedFile( this.unrestrictedSyncer.unrestrictedSyncLocallyCreatedOrUpdatedFile(
{ {
oldPath, oldPath,
document document
} }
), [document.metadata?.documentId, relativePath, oldPath] ),
[document.metadata?.documentId, relativePath, oldPath]
); );
} }
public async scheduleSyncForOfflineChanges(): Promise<void> { public async scheduleSyncForOfflineChanges(): Promise<void> {
@ -290,7 +269,7 @@ export class Syncer {
public reset(): void { public reset(): void {
this._isFirstSyncComplete = false; this._isFirstSyncComplete = false;
this.syncQueue.clear(); this.syncQueue.clear();
this.updatedDocumentsByPathAndKeysLock.reset(); this.updatedDocumentsByPathAndKeysLocks.reset();
this.runningScheduleSyncForOfflineChanges = undefined; this.runningScheduleSyncForOfflineChanges = undefined;
} }
@ -310,13 +289,17 @@ export class Syncer {
const document = this.database.getDocumentByDocumentId( const document = this.database.getDocumentByDocumentId(
remoteVersion.documentId remoteVersion.documentId
); );
this.enqueueSyncOperation(async () => await this.enqueueSyncOperation(
await this.syncQueue.add(async () => async () =>
this.unrestrictedSyncer.unrestrictedSyncRemotelyUpdatedFile( this.unrestrictedSyncer.unrestrictedSyncRemotelyUpdatedFile(
remoteVersion, remoteVersion,
document document
) ),
), [document?.relativePath, remoteVersion.relativePath, remoteVersion.documentId] [
document?.relativePath,
remoteVersion.relativePath,
remoteVersion.documentId
]
); );
this.database.addSeenUpdateId(remoteVersion.vaultUpdateId); this.database.addSeenUpdateId(remoteVersion.vaultUpdateId);
@ -465,10 +448,11 @@ export class Syncer {
private async enqueueSyncOperation<T>( private async enqueueSyncOperation<T>(
operation: () => Promise<T>, operation: () => Promise<T>,
keys: Array<DocumentId | RelativePath | undefined | null> keys: (DocumentId | undefined | null)[]
): Promise<T> { ): Promise<T> {
return this.updatedDocumentsByPathAndKeysLock.withLock(keys.filter(k => k !== undefined && k !== null), async () => return this.updatedDocumentsByPathAndKeysLocks.withLock(
this.syncQueue.add(operation) keys.filter((k) => k !== undefined && k !== null),
async () => this.syncQueue.add(operation)
); );
} }
} }

71
frontend/sync-client/src/sync-operations/unrestricted-syncer.ts
View file

@ -3,7 +3,6 @@ import type {
DocumentRecord, DocumentRecord,
RelativePath RelativePath
} from "../persistence/database"; } from "../persistence/database";
import { diff } from "reconcile-text"; import { diff } from "reconcile-text";
import type { SyncService } from "../services/sync-service"; import type { SyncService } from "../services/sync-service";
import type { Logger } from "../tracing/logger"; import type { Logger } from "../tracing/logger";
@ -18,11 +17,9 @@ import type {
} from "../tracing/sync-history"; } from "../tracing/sync-history";
import { SyncStatus, SyncType } from "../tracing/sync-history"; import { SyncStatus, SyncType } from "../tracing/sync-history";
import { EMPTY_HASH, hash } from "../utils/hash"; import { EMPTY_HASH, hash } from "../utils/hash";
import { base64ToBytes } from "byte-base64"; import { base64ToBytes } from "byte-base64";
import type { Settings } from "../persistence/settings"; import type { Settings } from "../persistence/settings";
import type { FileOperations } from "../file-operations/file-operations"; import type { FileOperations } from "../file-operations/file-operations";
import { createPromise } from "../utils/create-promise";
import { FileNotFoundError } from "../errors/file-not-found-error"; import { FileNotFoundError } from "../errors/file-not-found-error";
import { SyncResetError } from "../errors/sync-reset-error"; import { SyncResetError } from "../errors/sync-reset-error";
import { globsToRegexes } from "../utils/globs-to-regexes"; import { globsToRegexes } from "../utils/globs-to-regexes";
@ -33,7 +30,6 @@ import type { FixedSizeDocumentCache } from "../utils/data-structures/fix-sized-
import { isFileTypeMergable } from "../utils/is-file-type-mergable"; import { isFileTypeMergable } from "../utils/is-file-type-mergable";
import { isBinary } from "../utils/is-binary"; import { isBinary } from "../utils/is-binary";
import type { ServerConfig } from "../services/server-config"; import type { ServerConfig } from "../services/server-config";
import { Locks } from "../utils/data-structures/locks";
export class UnrestrictedSyncer { export class UnrestrictedSyncer {
private ignorePatterns: RegExp[]; private ignorePatterns: RegExp[];
@ -66,7 +62,7 @@ export class UnrestrictedSyncer {
// We use the same code path for both local and remote updates. We need to force the update // We use the same code path for both local and remote updates. We need to force the update
// if there are no local changes but we know that the remote version is newer. // if there are no local changes but we know that the remote version is newer.
force = false, force = false,
document, document
}: { }: {
oldPath?: RelativePath; oldPath?: RelativePath;
force?: boolean; force?: boolean;
@ -123,7 +119,6 @@ export class UnrestrictedSyncer {
originalContentBytes: contentBytes, originalContentBytes: contentBytes,
isCreate: true isCreate: true
}); });
} else { } else {
const areThereLocalChanges = const areThereLocalChanges =
document.metadata.hash !== contentHash || document.metadata.hash !== contentHash ||
@ -351,7 +346,7 @@ export class UnrestrictedSyncer {
remoteRelativePath: remoteVersion.relativePath remoteRelativePath: remoteVersion.relativePath
}, },
this.database.createNewPendingDocument( this.database.createNewPendingDocument(
remoteVersion.relativePath, remoteVersion.relativePath
) )
); );
@ -365,7 +360,6 @@ export class UnrestrictedSyncer {
remoteVersion.relativePath remoteVersion.relativePath
); );
this.history.addHistoryEntry({ this.history.addHistoryEntry({
status: SyncStatus.SUCCESS, status: SyncStatus.SUCCESS,
details: updateDetails, details: updateDetails,
@ -376,8 +370,6 @@ export class UnrestrictedSyncer {
}); });
} }
private async executeSync<T>( private async executeSync<T>(
details: SyncDetails, details: SyncDetails,
fn: () => Promise<T> fn: () => Promise<T>
@ -481,9 +473,9 @@ export class UnrestrictedSyncer {
} }
let actualPath = document.relativePath; let actualPath = document.relativePath;
let mustCreate = false;
if (isCreate) {
if (isCreate === true) {
// We have a file locally that got moved by another client to the same path as the one we're trying to create. // We have a file locally that got moved by another client to the same path as the one we're trying to create.
// The server returns a merging update for the document ID that already exists locally (but at another path). // The server returns a merging update for the document ID that already exists locally (but at another path).
// We have to merge these two documents by extending the provenance of the existing document and deleting // We have to merge these two documents by extending the provenance of the existing document and deleting
@ -492,14 +484,18 @@ export class UnrestrictedSyncer {
response.documentId response.documentId
); );
if (existingDocument !== undefined) { if (existingDocument !== undefined) {
this.logger.info(`Merging document ${existingDocument.relativePath} into existing document ${document.relativePath} after concurrent move & creation`); this.logger.info(
`Merging document ${existingDocument.relativePath} into existing document ${document.relativePath
} after concurrent move & creation`
);
this.database.removeDocument(document); // this was a (fake) pending document this.database.removeDocument(document); // this was a (fake) pending document
if (!existingDocument.isDeleted) { if (!existingDocument.isDeleted) {
this.operations.delete(document.relativePath); this.database.delete(existingDocument.relativePath); // make sure syncLocallyDeletedFile doesn't actually schedule deleting the new file
await this.operations.delete(existingDocument.relativePath);
} }
mustCreate = true;
document = existingDocument; document = existingDocument;
} }
} }
// this can't happen on the creation path as we can only get a merging response if a document already exists remotely on the same path // this can't happen on the creation path as we can only get a merging response if a document already exists remotely on the same path
@ -516,26 +512,41 @@ export class UnrestrictedSyncer {
); // this can throw FileNotFoundError ); // this can throw FileNotFoundError
} }
if (!("type" in response) || response.type === "MergingUpdate") { if (!("type" in response) || response.type === "MergingUpdate") {
const responseBytes = base64ToBytes(response.contentBase64); const responseBytes = base64ToBytes(response.contentBase64);
contentHash = hash(responseBytes); contentHash = hash(responseBytes);
this.database.updateDocumentMetadata(
{ if (mustCreate) {
documentId: response.documentId, this.database.createNewPendingDocument(actualPath);
parentVersionId: response.vaultUpdateId, this.database.updateDocumentMetadata(
hash: contentHash, {
remoteRelativePath: response.relativePath documentId: response.documentId,
}, parentVersionId: response.vaultUpdateId,
document hash: contentHash,
); remoteRelativePath: response.relativePath
await this.operations.write( },
actualPath, document
originalContentBytes, );
responseBytes
); await this.operations.create(actualPath, responseBytes);
} else {
this.database.updateDocumentMetadata(
{
documentId: response.documentId,
parentVersionId: response.vaultUpdateId,
hash: contentHash,
remoteRelativePath: response.relativePath
},
document
);
await this.operations.write(
actualPath,
originalContentBytes,
responseBytes
);
}
await this.updateCache( await this.updateCache(
response.vaultUpdateId, response.vaultUpdateId,
responseBytes, responseBytes,

76
frontend/sync-client/src/utils/data-structures/locks.test.ts
View file

@ -60,27 +60,30 @@ describe("withLock", () => {
await locks.waitForLock(testPath); await locks.waitForLock(testPath);
const promise = awaitAll([locks.withLock([testPath2, testPath3, testPath], async () => { const promise = awaitAll([
executionOrder.push("operation1-start"); locks.withLock([testPath2, testPath3, testPath], async () => {
executionOrder.push("operation1-end"); executionOrder.push("operation1-start");
return "result1"; executionOrder.push("operation1-end");
}), return "result1";
}),
locks.withLock([testPath3, testPath, testPath2], async () => {
executionOrder.push("operation2-start");
executionOrder.push("operation2-end");
return "result2";
})]);
locks.withLock([testPath3, testPath, testPath2], async () => {
executionOrder.push("operation2-start");
executionOrder.push("operation2-end");
return "result2";
})
]);
locks.unlock(testPath); locks.unlock(testPath);
const [result1, result2] = await Promise.race([promise, new Promise<never>((_, reject) => { const [result1, result2] = await Promise.race([
setTimeout(() => { promise,
reject(new Error("Deadlock detected")); new Promise<never>((_, reject) => {
}, 1000); setTimeout(() => {
})]); reject(new Error("Deadlock detected"));
}, 1000);
})
]);
assert.strictEqual(result1, "result1"); assert.strictEqual(result1, "result1");
assert.strictEqual(result2, "result2"); assert.strictEqual(result2, "result2");
@ -243,6 +246,7 @@ describe("withLock", () => {
describe("reset", () => { describe("reset", () => {
const testPath: RelativePath = "test/document/path"; const testPath: RelativePath = "test/document/path";
const testPath2: RelativePath = "test/document/path2";
const logger = new Logger(); const logger = new Logger();
// eslint-disable-next-line @typescript-eslint/init-declarations // eslint-disable-next-line @typescript-eslint/init-declarations
@ -261,7 +265,7 @@ describe("reset", () => {
await sleep(1); await sleep(1);
const secondPromise = locks.withLock(testPath, async () => "second"); const secondPromise = locks.withLock(testPath, async () => "second");
void secondPromise.catch(() => { }); // eslint-disable-line @typescript-eslint/no-empty-function void secondPromise.catch(() => {}); // eslint-disable-line @typescript-eslint/no-empty-function
locks.reset(); locks.reset();
@ -282,7 +286,7 @@ describe("reset", () => {
await sleep(1); await sleep(1);
const secondPromise = locks.withLock(testPath, async () => "second"); const secondPromise = locks.withLock(testPath, async () => "second");
void secondPromise.catch(() => { }); // eslint-disable-line @typescript-eslint/no-empty-function void secondPromise.catch(() => {}); // eslint-disable-line @typescript-eslint/no-empty-function
locks.reset(); locks.reset();
@ -298,4 +302,38 @@ describe("reset", () => {
const result = await locks.withLock(testPath, () => "success"); const result = await locks.withLock(testPath, () => "success");
assert.strictEqual(result, "success"); assert.strictEqual(result, "success");
}); });
it("should release partially acquired locks when reset interrupts multi-key acquisition", async () => {
// Hold testPath2 so multi-key acquisition will block on it
await locks.waitForLock(testPath2);
// Start multi-key lock that will acquire testPath first, then block on testPath2
const multiKeyPromise = locks.withLock(
[testPath, testPath2],
async () => "multi"
);
void multiKeyPromise.catch(() => {}); // eslint-disable-line @typescript-eslint/no-empty-function
// Wait for the multi-key operation to acquire testPath and start waiting on testPath2
await sleep(10);
// Reset should reject the waiting operation
locks.reset();
await assert.rejects(multiKeyPromise, (err: Error) => {
assert.ok(err instanceof SyncResetError);
return true;
});
// The key that was already acquired (testPath) should now be released
// This would hang/timeout if the lock was leaked
const result = await Promise.race([
locks.withLock(testPath, () => "success"),
sleep(100).then(() => {
throw new Error("Lock was not released - deadlock detected");
})
]);
assert.strictEqual(result, "success");
});
}); });

83
frontend/sync-client/src/utils/data-structures/locks.ts
View file

@ -1,6 +1,5 @@
import { SyncResetError } from "../../errors/sync-reset-error"; import { SyncResetError } from "../../errors/sync-reset-error";
import type { Logger } from "../../tracing/logger"; import type { Logger } from "../../tracing/logger";
import { awaitAll } from "../await-all";
/** /**
* Manages exclusive locks on items to prevent concurrent modifications. * Manages exclusive locks on items to prevent concurrent modifications.
@ -8,15 +7,18 @@ import { awaitAll } from "../await-all";
* *
* @template T The type of the key used for locking * @template T The type of the key used for locking
*/ */
/** Waiter entry with callbacks */
interface WaiterEntry<T> {
resolve: () => unknown;
reject: (err: unknown) => unknown;
}
export class Locks<T> { export class Locks<T> {
/** Currently locked keys */ /** Currently locked keys */
private readonly locked = new Set<T>(); private readonly locked = new Set<T>();
/** Queue of resolve functions waiting for each key */ /** Queue of waiters for each key */
private readonly waiters = new Map< private readonly waiters = new Map<T, WaiterEntry<T>[]>();
T,
[() => unknown, (err: unknown) => unknown][]
>();
public constructor(private readonly logger?: Logger) { } public constructor(private readonly logger?: Logger) { }
@ -59,15 +61,17 @@ export class Locks<T> {
const uniqueKeys = Array.from(new Set(keys)); const uniqueKeys = Array.from(new Set(keys));
uniqueKeys.sort((a, b) => String(a).localeCompare(String(b))); // Ensure consistent order to prevent deadlocks uniqueKeys.sort((a, b) => String(a).localeCompare(String(b))); // Ensure consistent order to prevent deadlocks
for (const key of uniqueKeys) { const lockedKeys = [];
// Must acquire locks in-order (not concurrently) to prevent deadlocks
await this.waitForLock(key);
}
try { try {
for (const key of uniqueKeys) {
// Must acquire locks in-order (not concurrently) to prevent deadlocks
await this.waitForLock(key);
lockedKeys.push(key);
}
return await fn(); return await fn();
} finally { } finally {
uniqueKeys.forEach((key) => { lockedKeys.forEach((key) => {
this.unlock(key); this.unlock(key);
}); });
} }
@ -77,7 +81,7 @@ export class Locks<T> {
// Resolve all waiting promises before clearing to prevent deadlock // Resolve all waiting promises before clearing to prevent deadlock
// Any operation waiting for a lock will be granted access immediately // Any operation waiting for a lock will be granted access immediately
for (const waiting of this.waiters.values()) { for (const waiting of this.waiters.values()) {
for (const [_, reject] of waiting) { for (const { reject } of waiting) {
reject(new SyncResetError()); reject(new SyncResetError());
} }
} }
@ -89,40 +93,6 @@ export class Locks<T> {
return this.locked.has(key); return this.locked.has(key);
} }
public getDebugString(): string {
const lockedKeys = Array.from(this.locked).map((key) => String(key));
const waiterEntries = Array.from(this.waiters.entries()).filter(
([_, waiting]) => waiting.length > 0
);
const lines: string[] = [];
lines.push("=== Locks Debug ===");
lines.push(`Locked keys (${lockedKeys.length}):`);
if (lockedKeys.length === 0) {
lines.push(" (none)");
} else {
for (const key of lockedKeys) {
const waiterCount =
this.waiters.get(key as T)?.length ?? 0;
lines.push(
` - ${key}${waiterCount > 0 ? ` (${waiterCount} waiting)` : ""}`
);
}
}
lines.push(`Waiters (${waiterEntries.length} keys):`);
if (waiterEntries.length === 0) {
lines.push(" (none)");
} else {
for (const [key, waiting] of waiterEntries) {
lines.push(` - ${String(key)}: ${waiting.length} waiting`);
}
}
lines.push("===================");
return lines.join("\n");
}
/** /**
* Attempts to acquire a lock immediately without waiting. * Attempts to acquire a lock immediately without waiting.
* Must call `unlock()` if successful. * Must call `unlock()` if successful.
@ -162,11 +132,13 @@ export class Locks<T> {
this.waiters.set(key, waiting); this.waiters.set(key, waiting);
} }
waiting.push([resolve, reject]); waiting.push({
resolve,
reject,
});
}); });
} }
/** /**
* Releases a lock and grants access to the next waiting operation in FIFO order. * Releases a lock and grants access to the next waiting operation in FIFO order.
* Removes the key from locked set if no waiters. * Removes the key from locked set if no waiters.
@ -176,15 +148,20 @@ export class Locks<T> {
*/ */
public unlock(key: T): void { public unlock(key: T): void {
if (!this.locked.has(key)) { if (!this.locked.has(key)) {
this.logger?.debug(
`Attempted to unlock ${key} which is not locked`
);
return; return;
} }
// Remove first waiter to ensure FIFO order this.logger?.debug(`Releasing lock on ${key}`);
const [resolveNextWaiting, _] = this.waiters.get(key)?.shift() ?? [];
if (resolveNextWaiting) { // Remove first waiter to ensure FIFO order
const nextWaiter = this.waiters.get(key)?.shift();
if (nextWaiter) {
this.logger?.debug(`Granted lock on ${key}`); this.logger?.debug(`Granted lock on ${key}`);
resolveNextWaiting(); nextWaiter.resolve();
} else { } else {
this.locked.delete(key); this.locked.delete(key);
} }

1
frontend/sync-client/src/utils/debugging/in-memory-file-system.ts
View file

@ -45,7 +45,6 @@ export class InMemoryFileSystem implements FileSystemOperations {
return this.files.has(path); return this.files.has(path);
} }
// eslint-disable-next-line @typescript-eslint/no-empty-function
public async createDirectory(_path: RelativePath): Promise<void> { public async createDirectory(_path: RelativePath): Promise<void> {
// This doesn't mean anything in our virtual FS representation // This doesn't mean anything in our virtual FS representation
} }

6
frontend/test-client/src/agent/mock-agent.ts
View file

@ -17,7 +17,7 @@ export class MockAgent extends MockClient {
// The renamed file finding algorithm isn't too smart so we can't both update and rename the same file // The renamed file finding algorithm isn't too smart so we can't both update and rename the same file
private readonly doNotTouchWhileOffline: string[] = []; private readonly doNotTouchWhileOffline: string[] = [];
private lastSyncEnabledState: boolean = true; private lastSyncEnabledState = true;
public constructor( public constructor(
initialSettings: Partial<SyncSettings>, initialSettings: Partial<SyncSettings>,
@ -107,14 +107,12 @@ export class MockAgent extends MockClient {
public async waitUntilSynced(): Promise<void> { public async waitUntilSynced(): Promise<void> {
await withTimeout( await withTimeout(
(async (): Promise<void> => { (async (): Promise<void> => {
this.client.setSetting("isSyncEnabled", true); await this.client.setSetting("isSyncEnabled", true);
await this.client.waitUntilFinished(); await this.client.waitUntilFinished();
})(), })(),
TIMEOUT_MS, TIMEOUT_MS,
"waitUntilSynced()" "waitUntilSynced()"
); );
} }
public async act(): Promise<void> { public async act(): Promise<void> {

14
frontend/test-client/src/agent/mock-client.ts
View file

@ -83,13 +83,13 @@ export class MockClient extends debugging.InMemoryFileSystem {
.map((part) => part.trim()); .map((part) => part.trim());
const newParts = newContent.split(" ").map((part) => part.trim()); const newParts = newContent.split(" ").map((part) => part.trim());
existingParts.forEach((part) => existingParts.forEach((part) =>
// all changes should be additive // all changes should be additive
{ {
assert( assert(
newParts.includes(part), newParts.includes(part),
`Part ${part} not found in new content: ${newContent}` `Part ${part} not found in new content: ${newContent}`
); );
} }
); );
} }

8
frontend/test-client/src/cli.ts
View file

@ -80,8 +80,6 @@ async function runTest({
await utils.awaitAll(clients.map(async (client) => client.init())); await utils.awaitAll(clients.map(async (client) => client.init()));
for (let i = 0; i < iterations; i++) { for (let i = 0; i < iterations; i++) {
logger.info(`Iteration ${i + 1}/${iterations}`); logger.info(`Iteration ${i + 1}/${iterations}`);
await utils.awaitAll(clients.map(async (client) => client.act())); await utils.awaitAll(clients.map(async (client) => client.act()));
@ -184,7 +182,7 @@ process.on("uncaughtException", (error) => {
} }
logger.error(`Error - uncaught exception: ${error}`); logger.error(`Error - uncaught exception: ${error}`);
if (error instanceof Error && error.stack) { if (error instanceof Error && error.stack != null) {
logger.error(error.stack); logger.error(error.stack);
} }
process.exit(1); process.exit(1);
@ -215,7 +213,7 @@ process.on("unhandledRejection", (error, _promise) => {
} }
logger.error(`Error - unhandled rejection: ${error}`); logger.error(`Error - unhandled rejection: ${error}`);
if (error instanceof Error && error.stack) { if (error instanceof Error && error.stack != null) {
logger.error(error.stack); logger.error(error.stack);
} }
process.exit(1); process.exit(1);
@ -227,7 +225,7 @@ runTests()
}) })
.catch((error: unknown) => { .catch((error: unknown) => {
logger.error(`Error - tests failed with ${error}`); logger.error(`Error - tests failed with ${error}`);
if (error instanceof Error && error.stack) { if (error instanceof Error && error.stack != null) {
logger.error(error.stack); logger.error(error.stack);
} }
process.exit(1); process.exit(1);

18
frontend/test-client/src/utils/with-timeout.ts
View file

@ -1,4 +1,9 @@
import { __debug_locks } from "sync-client"; export class TimeoutError extends Error {
public constructor(message: string) {
super(message);
this.name = "TimeoutError";
}
}
export async function withTimeout<T>( export async function withTimeout<T>(
promise: Promise<T>, promise: Promise<T>,
@ -10,16 +15,11 @@ export async function withTimeout<T>(
new Promise<T>((_, reject) => new Promise<T>((_, reject) =>
setTimeout(() => { setTimeout(() => {
reject( reject(
new TimeoutError(`${operationName} timed out after ${timeoutMs}ms ${__debug_locks.map(lock => lock.getDebugString()).join(", ")}`) new TimeoutError(
`${operationName} timed out after ${timeoutMs}ms`
)
); );
}, timeoutMs) }, timeoutMs)
) )
]); ]);
} }
export class TimeoutError extends Error {
constructor(message: string) {
super(message);
this.name = "TimeoutError";
}
}

8
package-lock.json generated
View file

@ -1,6 +1,6 @@
{ {
"name": "vault-link", "name": "vault-link",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": {} "packages": {}
} }