Start fixing tests
This commit is contained in:
parent
727b6b7ed5
commit
7fcd0f0bfa
19 changed files with 209 additions and 217 deletions
|
|
@ -80,5 +80,4 @@ And to clean up the logs & database files, run `task clean`
|
||||||
|
|
||||||
- [Sync server](./sync-server/README.md)
|
- [Sync server](./sync-server/README.md)
|
||||||
|
|
||||||
|
|
||||||
remove force merge everywhere
|
remove force merge everywhere
|
||||||
|
|
|
||||||
|
|
@ -80,10 +80,10 @@ async function main(): Promise<void> {
|
||||||
|
|
||||||
if (!result.success) {
|
if (!result.success) {
|
||||||
allPassed = false;
|
allPassed = false;
|
||||||
logger.error(`\n✗ FAILED: ${test.name}`);
|
logger.error(`✗ FAILED: ${test.name}`);
|
||||||
logger.error(`Error: ${result.error}`);
|
logger.error(`Error: ${result.error}`);
|
||||||
} else {
|
} else {
|
||||||
logger.info(`\n✓ PASSED: ${test.name} (${result.duration}ms)`);
|
logger.info(`✓ PASSED: ${test.name} (${result.duration}ms)`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
|
|
|
||||||
|
|
@ -46,12 +46,11 @@ export class TestRunner {
|
||||||
for (let i = 0; i < test.steps.length; i++) {
|
for (let i = 0; i < test.steps.length; i++) {
|
||||||
const step = test.steps[i];
|
const step = test.steps[i];
|
||||||
this.logger.info(
|
this.logger.info(
|
||||||
`\nStep ${i + 1}/${test.steps.length}: ${JSON.stringify(step)}`
|
`Step ${i + 1}/${test.steps.length}: ${JSON.stringify(step)}`
|
||||||
);
|
);
|
||||||
await this.executeStep(step);
|
await this.executeStep(step);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Cleanup
|
|
||||||
await this.cleanup();
|
await this.cleanup();
|
||||||
|
|
||||||
const duration = Date.now() - startTime;
|
const duration = Date.now() - startTime;
|
||||||
|
|
|
||||||
|
|
@ -33,14 +33,13 @@ export type { AuthenticationError } from "./errors/authentication-error";
|
||||||
export type { MaybeOutdatedClientCursors } from "./types/maybe-outdated-client-cursors";
|
export type { MaybeOutdatedClientCursors } from "./types/maybe-outdated-client-cursors";
|
||||||
export { DocumentSyncStatus } from "./types/document-sync-status";
|
export { DocumentSyncStatus } from "./types/document-sync-status";
|
||||||
export { SyncClient } from "./sync-client";
|
export { SyncClient } from "./sync-client";
|
||||||
export { __debug_locks } from "./sync-operations/syncer";
|
|
||||||
export type { TextWithCursors, CursorPosition } from "reconcile-text";
|
export type { TextWithCursors, CursorPosition } from "reconcile-text";
|
||||||
|
|
||||||
export const debugging = {
|
export const debugging = {
|
||||||
slowFetchFactory,
|
slowFetchFactory,
|
||||||
slowWebSocketFactory,
|
slowWebSocketFactory,
|
||||||
logToConsole,
|
logToConsole,
|
||||||
InMemoryFileSystem,
|
InMemoryFileSystem
|
||||||
};
|
};
|
||||||
|
|
||||||
export const utils = {
|
export const utils = {
|
||||||
|
|
|
||||||
|
|
@ -130,11 +130,7 @@ export class Database {
|
||||||
target.metadata,
|
target.metadata,
|
||||||
null,
|
null,
|
||||||
2
|
2
|
||||||
)} to ${JSON.stringify(
|
)} to ${JSON.stringify(metadata, null, 2)}`
|
||||||
metadata,
|
|
||||||
null,
|
|
||||||
2
|
|
||||||
)}`
|
|
||||||
);
|
);
|
||||||
|
|
||||||
target.metadata = metadata;
|
target.metadata = metadata;
|
||||||
|
|
@ -142,7 +138,6 @@ export class Database {
|
||||||
this.saveInTheBackground();
|
this.saveInTheBackground();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public getLatestDocumentByRelativePath(
|
public getLatestDocumentByRelativePath(
|
||||||
find: RelativePath
|
find: RelativePath
|
||||||
): DocumentRecord | undefined {
|
): DocumentRecord | undefined {
|
||||||
|
|
@ -153,9 +148,8 @@ export class Database {
|
||||||
return candidates[0];
|
return candidates[0];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public createNewPendingDocument(
|
public createNewPendingDocument(
|
||||||
relativePath: RelativePath,
|
relativePath: RelativePath
|
||||||
): DocumentRecord {
|
): DocumentRecord {
|
||||||
this.logger.debug(`Creating new pending document: ${relativePath}`);
|
this.logger.debug(`Creating new pending document: ${relativePath}`);
|
||||||
const previousEntry =
|
const previousEntry =
|
||||||
|
|
@ -223,7 +217,6 @@ export class Database {
|
||||||
candidate.isDeleted = true;
|
candidate.isDeleted = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public removeDocument(find: DocumentRecord): void {
|
public removeDocument(find: DocumentRecord): void {
|
||||||
removeFromArray(this.documents, find);
|
removeFromArray(this.documents, find);
|
||||||
this.saveInTheBackground();
|
this.saveInTheBackground();
|
||||||
|
|
|
||||||
|
|
@ -40,7 +40,6 @@ export class SyncClient {
|
||||||
private readonly history: SyncHistory,
|
private readonly history: SyncHistory,
|
||||||
private readonly settings: Settings,
|
private readonly settings: Settings,
|
||||||
private readonly database: Database,
|
private readonly database: Database,
|
||||||
private readonly unrestrictedSyncer: UnrestrictedSyncer,
|
|
||||||
private readonly syncer: Syncer,
|
private readonly syncer: Syncer,
|
||||||
private readonly webSocketManager: WebSocketManager,
|
private readonly webSocketManager: WebSocketManager,
|
||||||
public readonly logger: Logger,
|
public readonly logger: Logger,
|
||||||
|
|
@ -221,7 +220,6 @@ export class SyncClient {
|
||||||
history,
|
history,
|
||||||
settings,
|
settings,
|
||||||
database,
|
database,
|
||||||
unrestrictedSyncer,
|
|
||||||
syncer,
|
syncer,
|
||||||
webSocketManager,
|
webSocketManager,
|
||||||
logger,
|
logger,
|
||||||
|
|
@ -410,7 +408,6 @@ export class SyncClient {
|
||||||
return DocumentSyncStatus.SYNCING;
|
return DocumentSyncStatus.SYNCING;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
return this.syncer.hasPendingOperationsForDocument(relativePath)
|
return this.syncer.hasPendingOperationsForDocument(relativePath)
|
||||||
? DocumentSyncStatus.SYNCING
|
? DocumentSyncStatus.SYNCING
|
||||||
: DocumentSyncStatus.UP_TO_DATE;
|
: DocumentSyncStatus.UP_TO_DATE;
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,6 @@ import type { Settings } from "../persistence/settings";
|
||||||
import type { FileOperations } from "../file-operations/file-operations";
|
import type { FileOperations } from "../file-operations/file-operations";
|
||||||
import { findMatchingFile } from "../utils/find-matching-file";
|
import { findMatchingFile } from "../utils/find-matching-file";
|
||||||
import type { UnrestrictedSyncer } from "./unrestricted-syncer";
|
import type { UnrestrictedSyncer } from "./unrestricted-syncer";
|
||||||
import { createPromise } from "../utils/create-promise";
|
|
||||||
import { SyncResetError } from "../errors/sync-reset-error";
|
import { SyncResetError } from "../errors/sync-reset-error";
|
||||||
import { Locks } from "../utils/data-structures/locks";
|
import { Locks } from "../utils/data-structures/locks";
|
||||||
import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent";
|
import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent";
|
||||||
|
|
@ -21,14 +20,12 @@ import type { WebSocketClientMessage } from "../services/types/WebSocketClientMe
|
||||||
import { awaitAll } from "../utils/await-all";
|
import { awaitAll } from "../utils/await-all";
|
||||||
import { EventListeners } from "../utils/data-structures/event-listeners";
|
import { EventListeners } from "../utils/data-structures/event-listeners";
|
||||||
|
|
||||||
export const __debug_locks: Locks<any>[] = []; // Used only for debugging timeouts
|
|
||||||
|
|
||||||
export class Syncer {
|
export class Syncer {
|
||||||
public readonly onRemainingOperationsCountChanged = new EventListeners<
|
public readonly onRemainingOperationsCountChanged = new EventListeners<
|
||||||
(remainingOperations: number) => unknown
|
(remainingOperations: number) => unknown
|
||||||
>();
|
>();
|
||||||
|
|
||||||
public readonly updatedDocumentsByPathAndKeysLock: Locks<DocumentId | RelativePath>;
|
public readonly updatedDocumentsByPathAndKeysLocks: Locks<string>; // can be DocumentId or RelativePath
|
||||||
|
|
||||||
// FIFO to limit the number of concurrent sync operations
|
// FIFO to limit the number of concurrent sync operations
|
||||||
private readonly syncQueue: PQueue;
|
private readonly syncQueue: PQueue;
|
||||||
|
|
@ -50,8 +47,9 @@ export class Syncer {
|
||||||
concurrency: settings.getSettings().syncConcurrency
|
concurrency: settings.getSettings().syncConcurrency
|
||||||
});
|
});
|
||||||
|
|
||||||
this.updatedDocumentsByPathAndKeysLock = new Locks<DocumentId>(this.logger);
|
this.updatedDocumentsByPathAndKeysLocks = new Locks<DocumentId>(
|
||||||
__debug_locks.push(this.updatedDocumentsByPathAndKeysLock); // Used only for debugging timeouts
|
this.logger
|
||||||
|
);
|
||||||
|
|
||||||
settings.onSettingsChanged.add((newSettings, oldSettings) => {
|
settings.onSettingsChanged.add((newSettings, oldSettings) => {
|
||||||
if (newSettings.syncConcurrency !== oldSettings.syncConcurrency) {
|
if (newSettings.syncConcurrency !== oldSettings.syncConcurrency) {
|
||||||
|
|
@ -84,7 +82,7 @@ export class Syncer {
|
||||||
}
|
}
|
||||||
|
|
||||||
public hasPendingOperationsForDocument(relativePath: string): boolean {
|
public hasPendingOperationsForDocument(relativePath: string): boolean {
|
||||||
return this.updatedDocumentsByPathAndKeysLock.isLocked(relativePath);
|
return this.updatedDocumentsByPathAndKeysLocks.isLocked(relativePath);
|
||||||
}
|
}
|
||||||
|
|
||||||
public async syncLocallyCreatedFile(
|
public async syncLocallyCreatedFile(
|
||||||
|
|
@ -102,29 +100,26 @@ export class Syncer {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const document = this.database.createNewPendingDocument(
|
const document = this.database.createNewPendingDocument(relativePath);
|
||||||
relativePath
|
|
||||||
);
|
|
||||||
|
|
||||||
await this.enqueueSyncOperation(async () =>
|
await this.enqueueSyncOperation(
|
||||||
|
async () =>
|
||||||
this.unrestrictedSyncer.unrestrictedSyncLocallyCreatedOrUpdatedFile(
|
this.unrestrictedSyncer.unrestrictedSyncLocallyCreatedOrUpdatedFile(
|
||||||
{
|
{
|
||||||
document
|
document
|
||||||
}
|
}
|
||||||
), [relativePath]
|
),
|
||||||
|
[relativePath]
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
public async syncLocallyDeletedFile(
|
public async syncLocallyDeletedFile(
|
||||||
relativePath: RelativePath
|
relativePath: RelativePath
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const document = this.database.getLatestDocumentByRelativePath(relativePath);
|
let document =
|
||||||
|
this.database.getLatestDocumentByRelativePath(relativePath);
|
||||||
|
|
||||||
|
if (document == null || document.isDeleted === true) {
|
||||||
if (
|
|
||||||
document
|
|
||||||
?.isDeleted === true
|
|
||||||
) {
|
|
||||||
// This is must be a consequence of us deleting a file because of a remote update
|
// This is must be a consequence of us deleting a file because of a remote update
|
||||||
// which triggered a local delete, so we don't need to do anything here.
|
// which triggered a local delete, so we don't need to do anything here.
|
||||||
this.logger.debug(
|
this.logger.debug(
|
||||||
|
|
@ -137,25 +132,13 @@ export class Syncer {
|
||||||
// document which finishes after the delete has succeeded and would introduce a phantom metadata record.
|
// document which finishes after the delete has succeeded and would introduce a phantom metadata record.
|
||||||
this.database.delete(relativePath);
|
this.database.delete(relativePath);
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
await this.enqueueSyncOperation(async () => {
|
await this.enqueueSyncOperation(async () => {
|
||||||
const document = this.database.getLatestDocumentByRelativePath(relativePath);
|
|
||||||
|
|
||||||
if (document === undefined) {
|
|
||||||
this.logger.debug(
|
|
||||||
`Cannot find document ${relativePath} in the database, must have been deleted already, skipping`
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
await this.unrestrictedSyncer.unrestrictedSyncLocallyDeletedFile(
|
await this.unrestrictedSyncer.unrestrictedSyncLocallyDeletedFile(
|
||||||
document
|
document
|
||||||
);
|
);
|
||||||
|
|
||||||
this.database.removeDocument(document);
|
this.database.removeDocument(document);
|
||||||
}, [document?.metadata?.documentId, relativePath]
|
}, [document?.metadata?.documentId, relativePath]);
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public async syncLocallyUpdatedFile({
|
public async syncLocallyUpdatedFile({
|
||||||
|
|
@ -165,18 +148,15 @@ export class Syncer {
|
||||||
oldPath?: RelativePath;
|
oldPath?: RelativePath;
|
||||||
relativePath: RelativePath;
|
relativePath: RelativePath;
|
||||||
}): Promise<void> {
|
}): Promise<void> {
|
||||||
const documentAtNewPath = this.database.getLatestDocumentByRelativePath(
|
const documentAtNewPath =
|
||||||
relativePath
|
this.database.getLatestDocumentByRelativePath(relativePath);
|
||||||
);
|
|
||||||
|
|
||||||
if (oldPath !== undefined) {
|
if (oldPath !== undefined) {
|
||||||
// We might have moved the document in the database before calling this method,
|
// We might have moved the document in the database before calling this method,
|
||||||
// in that case, we mustn't move it again.
|
// in that case, we mustn't move it again.
|
||||||
if (
|
if (
|
||||||
documentAtNewPath ===
|
documentAtNewPath === undefined ||
|
||||||
undefined ||
|
documentAtNewPath.isDeleted
|
||||||
documentAtNewPath
|
|
||||||
?.isDeleted === true
|
|
||||||
) {
|
) {
|
||||||
if (oldPath === relativePath) {
|
if (oldPath === relativePath) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
|
|
@ -188,7 +168,7 @@ export class Syncer {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let document =
|
const document =
|
||||||
this.database.getLatestDocumentByRelativePath(relativePath);
|
this.database.getLatestDocumentByRelativePath(relativePath);
|
||||||
|
|
||||||
if (
|
if (
|
||||||
|
|
@ -216,17 +196,16 @@ export class Syncer {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
await this.enqueueSyncOperation(
|
||||||
await this.enqueueSyncOperation(async () =>
|
async () =>
|
||||||
this.unrestrictedSyncer.unrestrictedSyncLocallyCreatedOrUpdatedFile(
|
this.unrestrictedSyncer.unrestrictedSyncLocallyCreatedOrUpdatedFile(
|
||||||
{
|
{
|
||||||
oldPath,
|
oldPath,
|
||||||
document
|
document
|
||||||
}
|
}
|
||||||
), [document.metadata?.documentId, relativePath, oldPath]
|
),
|
||||||
|
[document.metadata?.documentId, relativePath, oldPath]
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public async scheduleSyncForOfflineChanges(): Promise<void> {
|
public async scheduleSyncForOfflineChanges(): Promise<void> {
|
||||||
|
|
@ -290,7 +269,7 @@ export class Syncer {
|
||||||
public reset(): void {
|
public reset(): void {
|
||||||
this._isFirstSyncComplete = false;
|
this._isFirstSyncComplete = false;
|
||||||
this.syncQueue.clear();
|
this.syncQueue.clear();
|
||||||
this.updatedDocumentsByPathAndKeysLock.reset();
|
this.updatedDocumentsByPathAndKeysLocks.reset();
|
||||||
this.runningScheduleSyncForOfflineChanges = undefined;
|
this.runningScheduleSyncForOfflineChanges = undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -310,13 +289,17 @@ export class Syncer {
|
||||||
const document = this.database.getDocumentByDocumentId(
|
const document = this.database.getDocumentByDocumentId(
|
||||||
remoteVersion.documentId
|
remoteVersion.documentId
|
||||||
);
|
);
|
||||||
this.enqueueSyncOperation(async () =>
|
await this.enqueueSyncOperation(
|
||||||
await this.syncQueue.add(async () =>
|
async () =>
|
||||||
this.unrestrictedSyncer.unrestrictedSyncRemotelyUpdatedFile(
|
this.unrestrictedSyncer.unrestrictedSyncRemotelyUpdatedFile(
|
||||||
remoteVersion,
|
remoteVersion,
|
||||||
document
|
document
|
||||||
)
|
),
|
||||||
), [document?.relativePath, remoteVersion.relativePath, remoteVersion.documentId]
|
[
|
||||||
|
document?.relativePath,
|
||||||
|
remoteVersion.relativePath,
|
||||||
|
remoteVersion.documentId
|
||||||
|
]
|
||||||
);
|
);
|
||||||
|
|
||||||
this.database.addSeenUpdateId(remoteVersion.vaultUpdateId);
|
this.database.addSeenUpdateId(remoteVersion.vaultUpdateId);
|
||||||
|
|
@ -465,10 +448,11 @@ export class Syncer {
|
||||||
|
|
||||||
private async enqueueSyncOperation<T>(
|
private async enqueueSyncOperation<T>(
|
||||||
operation: () => Promise<T>,
|
operation: () => Promise<T>,
|
||||||
keys: Array<DocumentId | RelativePath | undefined | null>
|
keys: (DocumentId | undefined | null)[]
|
||||||
): Promise<T> {
|
): Promise<T> {
|
||||||
return this.updatedDocumentsByPathAndKeysLock.withLock(keys.filter(k => k !== undefined && k !== null), async () =>
|
return this.updatedDocumentsByPathAndKeysLocks.withLock(
|
||||||
this.syncQueue.add(operation)
|
keys.filter((k) => k !== undefined && k !== null),
|
||||||
|
async () => this.syncQueue.add(operation)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,6 @@ import type {
|
||||||
DocumentRecord,
|
DocumentRecord,
|
||||||
RelativePath
|
RelativePath
|
||||||
} from "../persistence/database";
|
} from "../persistence/database";
|
||||||
|
|
||||||
import { diff } from "reconcile-text";
|
import { diff } from "reconcile-text";
|
||||||
import type { SyncService } from "../services/sync-service";
|
import type { SyncService } from "../services/sync-service";
|
||||||
import type { Logger } from "../tracing/logger";
|
import type { Logger } from "../tracing/logger";
|
||||||
|
|
@ -18,11 +17,9 @@ import type {
|
||||||
} from "../tracing/sync-history";
|
} from "../tracing/sync-history";
|
||||||
import { SyncStatus, SyncType } from "../tracing/sync-history";
|
import { SyncStatus, SyncType } from "../tracing/sync-history";
|
||||||
import { EMPTY_HASH, hash } from "../utils/hash";
|
import { EMPTY_HASH, hash } from "../utils/hash";
|
||||||
|
|
||||||
import { base64ToBytes } from "byte-base64";
|
import { base64ToBytes } from "byte-base64";
|
||||||
import type { Settings } from "../persistence/settings";
|
import type { Settings } from "../persistence/settings";
|
||||||
import type { FileOperations } from "../file-operations/file-operations";
|
import type { FileOperations } from "../file-operations/file-operations";
|
||||||
import { createPromise } from "../utils/create-promise";
|
|
||||||
import { FileNotFoundError } from "../errors/file-not-found-error";
|
import { FileNotFoundError } from "../errors/file-not-found-error";
|
||||||
import { SyncResetError } from "../errors/sync-reset-error";
|
import { SyncResetError } from "../errors/sync-reset-error";
|
||||||
import { globsToRegexes } from "../utils/globs-to-regexes";
|
import { globsToRegexes } from "../utils/globs-to-regexes";
|
||||||
|
|
@ -33,7 +30,6 @@ import type { FixedSizeDocumentCache } from "../utils/data-structures/fix-sized-
|
||||||
import { isFileTypeMergable } from "../utils/is-file-type-mergable";
|
import { isFileTypeMergable } from "../utils/is-file-type-mergable";
|
||||||
import { isBinary } from "../utils/is-binary";
|
import { isBinary } from "../utils/is-binary";
|
||||||
import type { ServerConfig } from "../services/server-config";
|
import type { ServerConfig } from "../services/server-config";
|
||||||
import { Locks } from "../utils/data-structures/locks";
|
|
||||||
|
|
||||||
export class UnrestrictedSyncer {
|
export class UnrestrictedSyncer {
|
||||||
private ignorePatterns: RegExp[];
|
private ignorePatterns: RegExp[];
|
||||||
|
|
@ -66,7 +62,7 @@ export class UnrestrictedSyncer {
|
||||||
// We use the same code path for both local and remote updates. We need to force the update
|
// We use the same code path for both local and remote updates. We need to force the update
|
||||||
// if there are no local changes but we know that the remote version is newer.
|
// if there are no local changes but we know that the remote version is newer.
|
||||||
force = false,
|
force = false,
|
||||||
document,
|
document
|
||||||
}: {
|
}: {
|
||||||
oldPath?: RelativePath;
|
oldPath?: RelativePath;
|
||||||
force?: boolean;
|
force?: boolean;
|
||||||
|
|
@ -123,7 +119,6 @@ export class UnrestrictedSyncer {
|
||||||
originalContentBytes: contentBytes,
|
originalContentBytes: contentBytes,
|
||||||
isCreate: true
|
isCreate: true
|
||||||
});
|
});
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
const areThereLocalChanges =
|
const areThereLocalChanges =
|
||||||
document.metadata.hash !== contentHash ||
|
document.metadata.hash !== contentHash ||
|
||||||
|
|
@ -351,7 +346,7 @@ export class UnrestrictedSyncer {
|
||||||
remoteRelativePath: remoteVersion.relativePath
|
remoteRelativePath: remoteVersion.relativePath
|
||||||
},
|
},
|
||||||
this.database.createNewPendingDocument(
|
this.database.createNewPendingDocument(
|
||||||
remoteVersion.relativePath,
|
remoteVersion.relativePath
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
@ -365,7 +360,6 @@ export class UnrestrictedSyncer {
|
||||||
remoteVersion.relativePath
|
remoteVersion.relativePath
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
||||||
this.history.addHistoryEntry({
|
this.history.addHistoryEntry({
|
||||||
status: SyncStatus.SUCCESS,
|
status: SyncStatus.SUCCESS,
|
||||||
details: updateDetails,
|
details: updateDetails,
|
||||||
|
|
@ -376,8 +370,6 @@ export class UnrestrictedSyncer {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
private async executeSync<T>(
|
private async executeSync<T>(
|
||||||
details: SyncDetails,
|
details: SyncDetails,
|
||||||
fn: () => Promise<T>
|
fn: () => Promise<T>
|
||||||
|
|
@ -481,9 +473,9 @@ export class UnrestrictedSyncer {
|
||||||
}
|
}
|
||||||
|
|
||||||
let actualPath = document.relativePath;
|
let actualPath = document.relativePath;
|
||||||
|
let mustCreate = false;
|
||||||
|
|
||||||
|
if (isCreate) {
|
||||||
if (isCreate === true) {
|
|
||||||
// We have a file locally that got moved by another client to the same path as the one we're trying to create.
|
// We have a file locally that got moved by another client to the same path as the one we're trying to create.
|
||||||
// The server returns a merging update for the document ID that already exists locally (but at another path).
|
// The server returns a merging update for the document ID that already exists locally (but at another path).
|
||||||
// We have to merge these two documents by extending the provenance of the existing document and deleting
|
// We have to merge these two documents by extending the provenance of the existing document and deleting
|
||||||
|
|
@ -492,14 +484,18 @@ export class UnrestrictedSyncer {
|
||||||
response.documentId
|
response.documentId
|
||||||
);
|
);
|
||||||
if (existingDocument !== undefined) {
|
if (existingDocument !== undefined) {
|
||||||
this.logger.info(`Merging document ${existingDocument.relativePath} into existing document ${document.relativePath} after concurrent move & creation`);
|
this.logger.info(
|
||||||
|
`Merging document ${existingDocument.relativePath} into existing document ${document.relativePath
|
||||||
|
} after concurrent move & creation`
|
||||||
|
);
|
||||||
this.database.removeDocument(document); // this was a (fake) pending document
|
this.database.removeDocument(document); // this was a (fake) pending document
|
||||||
if (!existingDocument.isDeleted) {
|
if (!existingDocument.isDeleted) {
|
||||||
this.operations.delete(document.relativePath);
|
this.database.delete(existingDocument.relativePath); // make sure syncLocallyDeletedFile doesn't actually schedule deleting the new file
|
||||||
|
await this.operations.delete(existingDocument.relativePath);
|
||||||
}
|
}
|
||||||
|
mustCreate = true;
|
||||||
document = existingDocument;
|
document = existingDocument;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// this can't happen on the creation path as we can only get a merging response if a document already exists remotely on the same path
|
// this can't happen on the creation path as we can only get a merging response if a document already exists remotely on the same path
|
||||||
|
|
@ -516,12 +512,26 @@ export class UnrestrictedSyncer {
|
||||||
); // this can throw FileNotFoundError
|
); // this can throw FileNotFoundError
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
if (!("type" in response) || response.type === "MergingUpdate") {
|
if (!("type" in response) || response.type === "MergingUpdate") {
|
||||||
const responseBytes = base64ToBytes(response.contentBase64);
|
const responseBytes = base64ToBytes(response.contentBase64);
|
||||||
contentHash = hash(responseBytes);
|
contentHash = hash(responseBytes);
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
if (mustCreate) {
|
||||||
|
this.database.createNewPendingDocument(actualPath);
|
||||||
|
this.database.updateDocumentMetadata(
|
||||||
|
{
|
||||||
|
documentId: response.documentId,
|
||||||
|
parentVersionId: response.vaultUpdateId,
|
||||||
|
hash: contentHash,
|
||||||
|
remoteRelativePath: response.relativePath
|
||||||
|
},
|
||||||
|
document
|
||||||
|
);
|
||||||
|
|
||||||
|
await this.operations.create(actualPath, responseBytes);
|
||||||
|
} else {
|
||||||
this.database.updateDocumentMetadata(
|
this.database.updateDocumentMetadata(
|
||||||
{
|
{
|
||||||
documentId: response.documentId,
|
documentId: response.documentId,
|
||||||
|
|
@ -536,6 +546,7 @@ export class UnrestrictedSyncer {
|
||||||
originalContentBytes,
|
originalContentBytes,
|
||||||
responseBytes
|
responseBytes
|
||||||
);
|
);
|
||||||
|
}
|
||||||
await this.updateCache(
|
await this.updateCache(
|
||||||
response.vaultUpdateId,
|
response.vaultUpdateId,
|
||||||
responseBytes,
|
responseBytes,
|
||||||
|
|
|
||||||
|
|
@ -60,7 +60,8 @@ describe("withLock", () => {
|
||||||
|
|
||||||
await locks.waitForLock(testPath);
|
await locks.waitForLock(testPath);
|
||||||
|
|
||||||
const promise = awaitAll([locks.withLock([testPath2, testPath3, testPath], async () => {
|
const promise = awaitAll([
|
||||||
|
locks.withLock([testPath2, testPath3, testPath], async () => {
|
||||||
executionOrder.push("operation1-start");
|
executionOrder.push("operation1-start");
|
||||||
executionOrder.push("operation1-end");
|
executionOrder.push("operation1-end");
|
||||||
return "result1";
|
return "result1";
|
||||||
|
|
@ -70,17 +71,19 @@ describe("withLock", () => {
|
||||||
executionOrder.push("operation2-start");
|
executionOrder.push("operation2-start");
|
||||||
executionOrder.push("operation2-end");
|
executionOrder.push("operation2-end");
|
||||||
return "result2";
|
return "result2";
|
||||||
})]);
|
})
|
||||||
|
]);
|
||||||
|
|
||||||
locks.unlock(testPath);
|
locks.unlock(testPath);
|
||||||
|
|
||||||
const [result1, result2] = await Promise.race([promise, new Promise<never>((_, reject) => {
|
const [result1, result2] = await Promise.race([
|
||||||
|
promise,
|
||||||
|
new Promise<never>((_, reject) => {
|
||||||
setTimeout(() => {
|
setTimeout(() => {
|
||||||
reject(new Error("Deadlock detected"));
|
reject(new Error("Deadlock detected"));
|
||||||
}, 1000);
|
}, 1000);
|
||||||
})]);
|
})
|
||||||
|
]);
|
||||||
|
|
||||||
assert.strictEqual(result1, "result1");
|
assert.strictEqual(result1, "result1");
|
||||||
assert.strictEqual(result2, "result2");
|
assert.strictEqual(result2, "result2");
|
||||||
|
|
@ -243,6 +246,7 @@ describe("withLock", () => {
|
||||||
|
|
||||||
describe("reset", () => {
|
describe("reset", () => {
|
||||||
const testPath: RelativePath = "test/document/path";
|
const testPath: RelativePath = "test/document/path";
|
||||||
|
const testPath2: RelativePath = "test/document/path2";
|
||||||
const logger = new Logger();
|
const logger = new Logger();
|
||||||
|
|
||||||
// eslint-disable-next-line @typescript-eslint/init-declarations
|
// eslint-disable-next-line @typescript-eslint/init-declarations
|
||||||
|
|
@ -298,4 +302,38 @@ describe("reset", () => {
|
||||||
const result = await locks.withLock(testPath, () => "success");
|
const result = await locks.withLock(testPath, () => "success");
|
||||||
assert.strictEqual(result, "success");
|
assert.strictEqual(result, "success");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("should release partially acquired locks when reset interrupts multi-key acquisition", async () => {
|
||||||
|
// Hold testPath2 so multi-key acquisition will block on it
|
||||||
|
await locks.waitForLock(testPath2);
|
||||||
|
|
||||||
|
// Start multi-key lock that will acquire testPath first, then block on testPath2
|
||||||
|
const multiKeyPromise = locks.withLock(
|
||||||
|
[testPath, testPath2],
|
||||||
|
async () => "multi"
|
||||||
|
);
|
||||||
|
void multiKeyPromise.catch(() => {}); // eslint-disable-line @typescript-eslint/no-empty-function
|
||||||
|
|
||||||
|
// Wait for the multi-key operation to acquire testPath and start waiting on testPath2
|
||||||
|
await sleep(10);
|
||||||
|
|
||||||
|
// Reset should reject the waiting operation
|
||||||
|
locks.reset();
|
||||||
|
|
||||||
|
await assert.rejects(multiKeyPromise, (err: Error) => {
|
||||||
|
assert.ok(err instanceof SyncResetError);
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
|
||||||
|
// The key that was already acquired (testPath) should now be released
|
||||||
|
// This would hang/timeout if the lock was leaked
|
||||||
|
const result = await Promise.race([
|
||||||
|
locks.withLock(testPath, () => "success"),
|
||||||
|
sleep(100).then(() => {
|
||||||
|
throw new Error("Lock was not released - deadlock detected");
|
||||||
|
})
|
||||||
|
]);
|
||||||
|
|
||||||
|
assert.strictEqual(result, "success");
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,5 @@
|
||||||
import { SyncResetError } from "../../errors/sync-reset-error";
|
import { SyncResetError } from "../../errors/sync-reset-error";
|
||||||
import type { Logger } from "../../tracing/logger";
|
import type { Logger } from "../../tracing/logger";
|
||||||
import { awaitAll } from "../await-all";
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Manages exclusive locks on items to prevent concurrent modifications.
|
* Manages exclusive locks on items to prevent concurrent modifications.
|
||||||
|
|
@ -8,15 +7,18 @@ import { awaitAll } from "../await-all";
|
||||||
*
|
*
|
||||||
* @template T The type of the key used for locking
|
* @template T The type of the key used for locking
|
||||||
*/
|
*/
|
||||||
|
/** Waiter entry with callbacks */
|
||||||
|
interface WaiterEntry<T> {
|
||||||
|
resolve: () => unknown;
|
||||||
|
reject: (err: unknown) => unknown;
|
||||||
|
}
|
||||||
|
|
||||||
export class Locks<T> {
|
export class Locks<T> {
|
||||||
/** Currently locked keys */
|
/** Currently locked keys */
|
||||||
private readonly locked = new Set<T>();
|
private readonly locked = new Set<T>();
|
||||||
|
|
||||||
/** Queue of resolve functions waiting for each key */
|
/** Queue of waiters for each key */
|
||||||
private readonly waiters = new Map<
|
private readonly waiters = new Map<T, WaiterEntry<T>[]>();
|
||||||
T,
|
|
||||||
[() => unknown, (err: unknown) => unknown][]
|
|
||||||
>();
|
|
||||||
|
|
||||||
public constructor(private readonly logger?: Logger) { }
|
public constructor(private readonly logger?: Logger) { }
|
||||||
|
|
||||||
|
|
@ -59,15 +61,17 @@ export class Locks<T> {
|
||||||
const uniqueKeys = Array.from(new Set(keys));
|
const uniqueKeys = Array.from(new Set(keys));
|
||||||
uniqueKeys.sort((a, b) => String(a).localeCompare(String(b))); // Ensure consistent order to prevent deadlocks
|
uniqueKeys.sort((a, b) => String(a).localeCompare(String(b))); // Ensure consistent order to prevent deadlocks
|
||||||
|
|
||||||
|
const lockedKeys = [];
|
||||||
|
try {
|
||||||
for (const key of uniqueKeys) {
|
for (const key of uniqueKeys) {
|
||||||
// Must acquire locks in-order (not concurrently) to prevent deadlocks
|
// Must acquire locks in-order (not concurrently) to prevent deadlocks
|
||||||
await this.waitForLock(key);
|
await this.waitForLock(key);
|
||||||
|
lockedKeys.push(key);
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
|
||||||
return await fn();
|
return await fn();
|
||||||
} finally {
|
} finally {
|
||||||
uniqueKeys.forEach((key) => {
|
lockedKeys.forEach((key) => {
|
||||||
this.unlock(key);
|
this.unlock(key);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
@ -77,7 +81,7 @@ export class Locks<T> {
|
||||||
// Resolve all waiting promises before clearing to prevent deadlock
|
// Resolve all waiting promises before clearing to prevent deadlock
|
||||||
// Any operation waiting for a lock will be granted access immediately
|
// Any operation waiting for a lock will be granted access immediately
|
||||||
for (const waiting of this.waiters.values()) {
|
for (const waiting of this.waiters.values()) {
|
||||||
for (const [_, reject] of waiting) {
|
for (const { reject } of waiting) {
|
||||||
reject(new SyncResetError());
|
reject(new SyncResetError());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -89,40 +93,6 @@ export class Locks<T> {
|
||||||
return this.locked.has(key);
|
return this.locked.has(key);
|
||||||
}
|
}
|
||||||
|
|
||||||
public getDebugString(): string {
|
|
||||||
const lockedKeys = Array.from(this.locked).map((key) => String(key));
|
|
||||||
const waiterEntries = Array.from(this.waiters.entries()).filter(
|
|
||||||
([_, waiting]) => waiting.length > 0
|
|
||||||
);
|
|
||||||
|
|
||||||
const lines: string[] = [];
|
|
||||||
lines.push("=== Locks Debug ===");
|
|
||||||
lines.push(`Locked keys (${lockedKeys.length}):`);
|
|
||||||
if (lockedKeys.length === 0) {
|
|
||||||
lines.push(" (none)");
|
|
||||||
} else {
|
|
||||||
for (const key of lockedKeys) {
|
|
||||||
const waiterCount =
|
|
||||||
this.waiters.get(key as T)?.length ?? 0;
|
|
||||||
lines.push(
|
|
||||||
` - ${key}${waiterCount > 0 ? ` (${waiterCount} waiting)` : ""}`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
lines.push(`Waiters (${waiterEntries.length} keys):`);
|
|
||||||
if (waiterEntries.length === 0) {
|
|
||||||
lines.push(" (none)");
|
|
||||||
} else {
|
|
||||||
for (const [key, waiting] of waiterEntries) {
|
|
||||||
lines.push(` - ${String(key)}: ${waiting.length} waiting`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
lines.push("===================");
|
|
||||||
|
|
||||||
return lines.join("\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Attempts to acquire a lock immediately without waiting.
|
* Attempts to acquire a lock immediately without waiting.
|
||||||
* Must call `unlock()` if successful.
|
* Must call `unlock()` if successful.
|
||||||
|
|
@ -162,11 +132,13 @@ export class Locks<T> {
|
||||||
this.waiters.set(key, waiting);
|
this.waiters.set(key, waiting);
|
||||||
}
|
}
|
||||||
|
|
||||||
waiting.push([resolve, reject]);
|
waiting.push({
|
||||||
|
resolve,
|
||||||
|
reject,
|
||||||
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Releases a lock and grants access to the next waiting operation in FIFO order.
|
* Releases a lock and grants access to the next waiting operation in FIFO order.
|
||||||
* Removes the key from locked set if no waiters.
|
* Removes the key from locked set if no waiters.
|
||||||
|
|
@ -176,15 +148,20 @@ export class Locks<T> {
|
||||||
*/
|
*/
|
||||||
public unlock(key: T): void {
|
public unlock(key: T): void {
|
||||||
if (!this.locked.has(key)) {
|
if (!this.locked.has(key)) {
|
||||||
|
this.logger?.debug(
|
||||||
|
`Attempted to unlock ${key} which is not locked`
|
||||||
|
);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Remove first waiter to ensure FIFO order
|
this.logger?.debug(`Releasing lock on ${key}`);
|
||||||
const [resolveNextWaiting, _] = this.waiters.get(key)?.shift() ?? [];
|
|
||||||
|
|
||||||
if (resolveNextWaiting) {
|
// Remove first waiter to ensure FIFO order
|
||||||
|
const nextWaiter = this.waiters.get(key)?.shift();
|
||||||
|
|
||||||
|
if (nextWaiter) {
|
||||||
this.logger?.debug(`Granted lock on ${key}`);
|
this.logger?.debug(`Granted lock on ${key}`);
|
||||||
resolveNextWaiting();
|
nextWaiter.resolve();
|
||||||
} else {
|
} else {
|
||||||
this.locked.delete(key);
|
this.locked.delete(key);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -45,7 +45,6 @@ export class InMemoryFileSystem implements FileSystemOperations {
|
||||||
return this.files.has(path);
|
return this.files.has(path);
|
||||||
}
|
}
|
||||||
|
|
||||||
// eslint-disable-next-line @typescript-eslint/no-empty-function
|
|
||||||
public async createDirectory(_path: RelativePath): Promise<void> {
|
public async createDirectory(_path: RelativePath): Promise<void> {
|
||||||
// This doesn't mean anything in our virtual FS representation
|
// This doesn't mean anything in our virtual FS representation
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -17,7 +17,7 @@ export class MockAgent extends MockClient {
|
||||||
|
|
||||||
// The renamed file finding algorithm isn't too smart so we can't both update and rename the same file
|
// The renamed file finding algorithm isn't too smart so we can't both update and rename the same file
|
||||||
private readonly doNotTouchWhileOffline: string[] = [];
|
private readonly doNotTouchWhileOffline: string[] = [];
|
||||||
private lastSyncEnabledState: boolean = true;
|
private lastSyncEnabledState = true;
|
||||||
|
|
||||||
public constructor(
|
public constructor(
|
||||||
initialSettings: Partial<SyncSettings>,
|
initialSettings: Partial<SyncSettings>,
|
||||||
|
|
@ -107,14 +107,12 @@ export class MockAgent extends MockClient {
|
||||||
public async waitUntilSynced(): Promise<void> {
|
public async waitUntilSynced(): Promise<void> {
|
||||||
await withTimeout(
|
await withTimeout(
|
||||||
(async (): Promise<void> => {
|
(async (): Promise<void> => {
|
||||||
this.client.setSetting("isSyncEnabled", true);
|
await this.client.setSetting("isSyncEnabled", true);
|
||||||
await this.client.waitUntilFinished();
|
await this.client.waitUntilFinished();
|
||||||
})(),
|
})(),
|
||||||
TIMEOUT_MS,
|
TIMEOUT_MS,
|
||||||
"waitUntilSynced()"
|
"waitUntilSynced()"
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public async act(): Promise<void> {
|
public async act(): Promise<void> {
|
||||||
|
|
|
||||||
|
|
@ -80,8 +80,6 @@ async function runTest({
|
||||||
|
|
||||||
await utils.awaitAll(clients.map(async (client) => client.init()));
|
await utils.awaitAll(clients.map(async (client) => client.init()));
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
for (let i = 0; i < iterations; i++) {
|
for (let i = 0; i < iterations; i++) {
|
||||||
logger.info(`Iteration ${i + 1}/${iterations}`);
|
logger.info(`Iteration ${i + 1}/${iterations}`);
|
||||||
await utils.awaitAll(clients.map(async (client) => client.act()));
|
await utils.awaitAll(clients.map(async (client) => client.act()));
|
||||||
|
|
@ -184,7 +182,7 @@ process.on("uncaughtException", (error) => {
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.error(`Error - uncaught exception: ${error}`);
|
logger.error(`Error - uncaught exception: ${error}`);
|
||||||
if (error instanceof Error && error.stack) {
|
if (error instanceof Error && error.stack != null) {
|
||||||
logger.error(error.stack);
|
logger.error(error.stack);
|
||||||
}
|
}
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
|
|
@ -215,7 +213,7 @@ process.on("unhandledRejection", (error, _promise) => {
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.error(`Error - unhandled rejection: ${error}`);
|
logger.error(`Error - unhandled rejection: ${error}`);
|
||||||
if (error instanceof Error && error.stack) {
|
if (error instanceof Error && error.stack != null) {
|
||||||
logger.error(error.stack);
|
logger.error(error.stack);
|
||||||
}
|
}
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
|
|
@ -227,7 +225,7 @@ runTests()
|
||||||
})
|
})
|
||||||
.catch((error: unknown) => {
|
.catch((error: unknown) => {
|
||||||
logger.error(`Error - tests failed with ${error}`);
|
logger.error(`Error - tests failed with ${error}`);
|
||||||
if (error instanceof Error && error.stack) {
|
if (error instanceof Error && error.stack != null) {
|
||||||
logger.error(error.stack);
|
logger.error(error.stack);
|
||||||
}
|
}
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,9 @@
|
||||||
import { __debug_locks } from "sync-client";
|
export class TimeoutError extends Error {
|
||||||
|
public constructor(message: string) {
|
||||||
|
super(message);
|
||||||
|
this.name = "TimeoutError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export async function withTimeout<T>(
|
export async function withTimeout<T>(
|
||||||
promise: Promise<T>,
|
promise: Promise<T>,
|
||||||
|
|
@ -10,16 +15,11 @@ export async function withTimeout<T>(
|
||||||
new Promise<T>((_, reject) =>
|
new Promise<T>((_, reject) =>
|
||||||
setTimeout(() => {
|
setTimeout(() => {
|
||||||
reject(
|
reject(
|
||||||
new TimeoutError(`${operationName} timed out after ${timeoutMs}ms ${__debug_locks.map(lock => lock.getDebugString()).join(", ")}`)
|
new TimeoutError(
|
||||||
|
`${operationName} timed out after ${timeoutMs}ms`
|
||||||
|
)
|
||||||
);
|
);
|
||||||
}, timeoutMs)
|
}, timeoutMs)
|
||||||
)
|
)
|
||||||
]);
|
]);
|
||||||
}
|
}
|
||||||
|
|
||||||
export class TimeoutError extends Error {
|
|
||||||
constructor(message: string) {
|
|
||||||
super(message);
|
|
||||||
this.name = "TimeoutError";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue