andras/vault-link
1
0
Fork 0
Code Issues Pull requests 18 Projects Releases 55 Packages Wiki Activity Actions 1

Add deterministic tests and lint

Browse source
This commit is contained in:
Andras Schmelczer 2026-01-13 21:52:42 +00:00
parent ea5a123cb8
commit 16afe31e89
29 changed files with 1738 additions and 222 deletions
Download patch file Download diff file Expand all files Collapse all files

4
frontend/sync-client/src/persistence/database.ts
View file

@ -103,7 +103,7 @@ export class Database {
i === 0
? false
: records[i - 1].parallelVersion ===
current.parallelVersion
current.parallelVersion
)
) {
throw new Error(
@ -350,7 +350,7 @@ export class Database {
if (duplicates.length > 0) {
throw new Error(
"Document IDs are not unique, found duplicates: " +
duplicates.join("; ")
duplicates.join("; ")
);
}
}

10
frontend/sync-client/src/services/sync-service.ts
View file

@ -67,7 +67,7 @@ export class SyncService {
public async create({
relativePath,
contentBytes,
contentBytes
}: {
relativePath: RelativePath;
contentBytes: Uint8Array;
@ -151,7 +151,8 @@ export class SyncService {
(await response.json()) as DocumentUpdateResponse; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
this.logger.debug(
`Updated document ${JSON.stringify(result)} with id ${result.documentId
`Updated document ${JSON.stringify(result)} with id ${
result.documentId
}}`
);
@ -203,7 +204,8 @@ export class SyncService {
(await response.json()) as DocumentUpdateResponse; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
this.logger.debug(
`Updated document ${JSON.stringify(result)} with id ${result.documentId
`Updated document ${JSON.stringify(result)} with id ${
result.documentId
}}`
);
@ -330,7 +332,7 @@ export class SyncService {
return this.retryForever(async () => {
this.logger.debug(
"Getting all documents" +
(since != null ? ` since ${since}` : "")
(since != null ? ` since ${since}` : "")
);
const url = new URL(this.getUrl("/documents"));

1
frontend/sync-client/src/services/types/CreateDocumentVersion.ts
View file

@ -2,6 +2,5 @@
export interface CreateDocumentVersion {
relative_path: string;
force_merge: boolean | null;
content: number[];
}

4
frontend/sync-client/src/sync-client.ts
View file

@ -56,7 +56,7 @@ export class SyncClient {
database: Partial<StoredDatabase>;
}>
>
) { }
) {}
public get documentCount(): number {
return this.database.length;
@ -369,7 +369,7 @@ export class SyncClient {
this.checkIfDestroyed("syncLocallyCreatedFile");
this.fileChangeNotifier.notifyOfFileChange(relativePath);
return this.syncer.syncLocallyCreatedFile(relativePath,);
return this.syncer.syncLocallyCreatedFile(relativePath);
}
public async syncLocallyDeletedFile(

122
frontend/sync-client/src/sync-operations/syncer.ts
View file

@ -81,7 +81,7 @@ export class Syncer {
}
public async syncLocallyCreatedFile(
relativePath: RelativePath,
relativePath: RelativePath
): Promise<void> {
if (
this.database.getLatestDocumentByRelativePath(relativePath)
@ -96,7 +96,6 @@ export class Syncer {
}
const [promise, resolve, reject] = createPromise();
this.logger.warn(`creating ${relativePath} locally`);
const document = this.database.createNewPendingDocument(
relativePath,
@ -106,12 +105,9 @@ export class Syncer {
try {
await this.syncQueue.add(async () =>
this.unrestrictedSyncer.unrestrictedSyncLocallyCreatedOrUpdatedFile(
{ document, forceMerge }
{ document }
)
)
this.logger.warn(`done creating ${relativePath} locally`);
);
resolve();
} catch (e) {
@ -149,7 +145,9 @@ export class Syncer {
try {
await this.syncQueue.add(async () =>
this.unrestrictedSyncer.unrestrictedSyncLocallyDeletedFile(document)
this.unrestrictedSyncer.unrestrictedSyncLocallyDeletedFile(
document
)
);
resolve();
@ -174,7 +172,7 @@ export class Syncer {
// in that case, we mustn't move it again.
if (
this.database.getLatestDocumentByRelativePath(relativePath) ===
undefined ||
undefined ||
this.database.getLatestDocumentByRelativePath(relativePath)
?.isDeleted === true
) {
@ -191,8 +189,6 @@ export class Syncer {
let document =
this.database.getLatestDocumentByRelativePath(relativePath);
this.logger.warn(`sync doc ${JSON.stringify(document)} for path ${relativePath} (old path: ${oldPath}), len docs: ${document?.updates.length}`);
if (
oldPath !== undefined &&
document?.metadata?.remoteRelativePath === relativePath
@ -224,14 +220,15 @@ export class Syncer {
relativePath,
promise
);
this.logger.warn(`updating ${document.relativePath} locally`);
try {
await this.syncQueue.add(async () =>
this.unrestrictedSyncer.unrestrictedSyncLocallyCreatedOrUpdatedFile({
oldPath,
document
})
this.unrestrictedSyncer.unrestrictedSyncLocallyCreatedOrUpdatedFile(
{
oldPath,
document
}
)
);
resolve();
@ -324,45 +321,37 @@ export class Syncer {
remoteVersion.documentId
);
this.logger.warn(`${remoteVersion.documentId} got remote update ${JSON.stringify(remoteVersion)}`);
if (document === undefined) {
this.logger.warn(`${remoteVersion.documentId} but document doesn't exist`)
return this.remoteDocumentsLock.withLock(
// Avoid the same documents getting created in parallel multiple times through fetching multiple updates of the same
// new remote document concurrently.
// There might be multiple tasks waiting for the lock
remoteVersion.documentId,
async () => {
// We have to wait for any ongoing creates sent for this file to finish,
// This is to avoid fetching one's own creates before the corresponding local create has finished syncing. This is a concern because
// documents being created don't yet have a document id in the local database and we could be notified of the remote create
// before the local create has finished syncing, so we can't just ignore the update based on the local DB content as we
// can't find the corresponding document yet.
// documents being created don't yet have a document id in the local database and we could be notified of the remote create
// before the local create has finished syncing, so we can't just ignore the update based on the local DB content as we
// can't find the corresponding document yet.
if (document?.metadata === undefined) {
await this.unrestrictedSyncer.fileCreationLock.waitForLockWithoutAcquiringLock(remoteVersion.relativePath);
await this.unrestrictedSyncer.fileCreationLock.waitForLockWithoutAcquiringLock(
remoteVersion.relativePath
);
}
document = this.database.getDocumentByDocumentId(
remoteVersion.documentId
);
this.logger.warn(`${remoteVersion.documentId} rechecking, document is now ${JSON.stringify(document)}`)
// We're the first one to get the lock, so we have to create the document in `unrestrictedSyncRemotelyUpdatedFile`
if (document === undefined) {
this.logger.warn(`${remoteVersion.documentId} document is undefined, creating new document`)
await this.syncQueue.add(async () =>
this.unrestrictedSyncer.unrestrictedSyncRemotelyUpdatedFile(
remoteVersion
)
);
} else {
const [promise, resolve, reject] =
createPromise();
const [promise, resolve, reject] = createPromise();
document =
await this.database.getResolvedDocumentByRelativePath(
@ -382,19 +371,13 @@ export class Syncer {
} catch (e) {
reject(e);
} finally {
this.database.removeDocumentPromise(
promise
);
this.database.removeDocumentPromise(promise);
}
}
this.database.addSeenUpdateId(
remoteVersion.vaultUpdateId
);
this.database.addSeenUpdateId(remoteVersion.vaultUpdateId);
}
)
} else {
this.logger.warn(`${remoteVersion.documentId} and document exists (path: ${JSON.stringify(document)})`);
);
}
// We're either the first one to get the lock, so we have to create the document in `unrestrictedSyncRemotelyUpdatedFile`
@ -440,7 +423,11 @@ export class Syncer {
}
}
interface Instruction { "type": "update" | "create", relativePath: string, oldPath?: string }
interface Instruction {
type: "update" | "create";
relativePath: string;
oldPath?: string;
}
const instructions: (Instruction | undefined)[] = await awaitAll(
allLocalFiles.map(async (relativePath) => {
if (
@ -499,7 +486,6 @@ export class Syncer {
oldPath: originalFile.relativePath,
relativePath
} as Instruction;
}
this.logger.debug(
@ -513,7 +499,6 @@ export class Syncer {
})
);
// this has to happen strictly after the previous awaitAll, as that one
// might have removed some of the documents from the list
await awaitAll(
@ -527,35 +512,38 @@ export class Syncer {
})
);
await awaitAll(
instructions.map(async (instruction) => {
if (instruction === undefined) {
return;
}
await awaitAll(instructions.map(async (instruction) => {
if (instruction === undefined) {
return;
}
if (instruction.type === "update") {
// We're outside of the pqueue, so we need to call the public wrapper
await this.syncLocallyUpdatedFile({
oldPath: instruction.oldPath,
relativePath: instruction.relativePath
}); return;
}
}));
if (instruction.type === "update") {
// We're outside of the pqueue, so we need to call the public wrapper
await this.syncLocallyUpdatedFile({
oldPath: instruction.oldPath,
relativePath: instruction.relativePath
});
return;
}
})
);
// we have to ensure the deletes & updates have finished before starting creates,
// otherwise the server might return an existing document (that we're about to delete)
// instead of actually creating a new one
await awaitAll(instructions.map(async (instruction) => {
if (instruction === undefined) {
return;
}
if (instruction.type === "create") {
// We're outside of the pqueue, so we need to call the public wrapper
await this.syncLocallyCreatedFile(instruction.relativePath,); return;
}
}));
await awaitAll(
instructions.map(async (instruction) => {
if (instruction === undefined) {
return;
}
if (instruction.type === "create") {
// We're outside of the pqueue, so we need to call the public wrapper
await this.syncLocallyCreatedFile(instruction.relativePath);
return;
}
})
);
}
}

138
frontend/sync-client/src/sync-operations/unrestricted-syncer.ts
View file

@ -36,9 +36,9 @@ import type { ServerConfig } from "../services/server-config";
import { Locks } from "../utils/data-structures/locks";
export class UnrestrictedSyncer {
public readonly fileCreationLock: Locks<RelativePath> =
new Locks<RelativePath>();
private ignorePatterns: RegExp[];
public readonly fileCreationLock: Locks<RelativePath> = new Locks<RelativePath>();
public constructor(
private readonly logger: Logger,
@ -74,32 +74,31 @@ export class UnrestrictedSyncer {
force?: boolean;
document: DocumentRecord;
}): Promise<void> {
// this.history.addHistoryEntry({
// status: SyncStatus.SUCCESS,
// details: updateDetails,
// message: `Successfully uploaded locally created file`
// });
let updateDetails: SyncCreateDetails | SyncUpdateDetails | SyncMovedDetails;
if (document.metadata === undefined) {
updateDetails = {
type: SyncType.CREATE,
relativePath: document.relativePath
};
}
else if (oldPath !== undefined) {
updateDetails = {
type: SyncType.MOVE,
relativePath: document.relativePath,
movedFrom: oldPath
};
} else {
updateDetails = {
type: SyncType.UPDATE,
relativePath: document.relativePath
};
}
const updateDetails:
| SyncCreateDetails
| SyncUpdateDetails
| SyncMovedDetails =
document.metadata === undefined
? {
type: SyncType.CREATE,
relativePath: document.relativePath
}
: oldPath !== undefined
? {
type: SyncType.MOVE,
relativePath: document.relativePath,
movedFrom: oldPath
}
: {
type: SyncType.UPDATE,
relativePath: document.relativePath
};
await this.executeSync(updateDetails, async () => {
const originalRelativePath = document.relativePath;
@ -116,31 +115,33 @@ export class UnrestrictedSyncer {
); // this can throw FileNotFoundError
const contentHash = hash(contentBytes);
this.logger.warn(`updating ${document.relativePath} locally, inner`);
let response: DocumentVersion | DocumentUpdateResponse | undefined =
undefined;
if (document.metadata === undefined) {
response = await this.fileCreationLock.withLock(document.relativePath, async () => {
const response = await this.syncService.create({
relativePath: originalRelativePath,
contentBytes,
});
response = await this.fileCreationLock.withLock(
document.relativePath,
async () => {
const createResponse = await this.syncService.create({
relativePath: originalRelativePath,
contentBytes
});
await this.handleMaybeMergingResponse({
document,
response,
contentHash,
originalRelativePath,
originalContentBytes: contentBytes
});
await this.handleMaybeMergingResponse({
document,
response: createResponse,
contentHash,
originalRelativePath,
originalContentBytes: contentBytes
});
return response;
});
return createResponse;
}
);
} else {
const areThereLocalChanges =
document.metadata.hash !== contentHash || oldPath !== undefined;
document.metadata.hash !== contentHash ||
oldPath !== undefined;
if (areThereLocalChanges) {
const isText =
@ -157,22 +158,22 @@ export class UnrestrictedSyncer {
response =
isText && cachedVersion !== undefined
? await this.syncService.putText({
documentId: document.metadata.documentId,
parentVersionId:
document.metadata.parentVersionId,
relativePath: document.relativePath,
content: diff(
new TextDecoder().decode(cachedVersion),
new TextDecoder().decode(contentBytes)
)
})
documentId: document.metadata.documentId,
parentVersionId:
document.metadata.parentVersionId,
relativePath: document.relativePath,
content: diff(
new TextDecoder().decode(cachedVersion),
new TextDecoder().decode(contentBytes)
)
})
: await this.syncService.putBinary({
documentId: document.metadata.documentId,
parentVersionId:
document.metadata.parentVersionId,
relativePath: document.relativePath,
contentBytes
});
documentId: document.metadata.documentId,
parentVersionId:
document.metadata.parentVersionId,
relativePath: document.relativePath,
contentBytes
});
} else {
if (!force) {
this.logger.debug(
@ -196,8 +197,6 @@ export class UnrestrictedSyncer {
});
}
if (!("type" in response) || response.type === "MergingUpdate") {
if (!force) {
this.history.addHistoryEntry({
@ -211,16 +210,16 @@ export class UnrestrictedSyncer {
const actualUpdateDetails: SyncUpdateDetails | SyncMovedDetails =
oldPath !== undefined ||
response.relativePath != originalRelativePath
response.relativePath != originalRelativePath
? {
type: SyncType.MOVE,
relativePath: response.relativePath,
movedFrom: originalRelativePath
}
type: SyncType.MOVE,
relativePath: response.relativePath,
movedFrom: originalRelativePath
}
: {
type: SyncType.UPDATE,
relativePath: response.relativePath
};
type: SyncType.UPDATE,
relativePath: response.relativePath
};
// if (areThereLocalChanges) {
// this.history.addHistoryEntry({
@ -229,7 +228,7 @@ export class UnrestrictedSyncer {
// message: `Successfully uploaded locally updated file to the server`,
// author: response.userId
// });
// } else
// } else
if (!response.isDeleted) {
this.history.addHistoryEntry({
@ -255,7 +254,6 @@ export class UnrestrictedSyncer {
});
}
public async unrestrictedSyncLocallyDeletedFile(
document: DocumentRecord
): Promise<void> {
@ -307,7 +305,6 @@ export class UnrestrictedSyncer {
relativePath: remoteVersion.relativePath
};
await this.executeSync(updateDetails, async () => {
if (document?.metadata !== undefined) {
// If the file exists locally, let's pretend the user has updated it
@ -474,8 +471,6 @@ export class UnrestrictedSyncer {
}
}
private async handleMaybeMergingResponse({
document,
response,
@ -584,8 +579,9 @@ export class UnrestrictedSyncer {
type: SyncType.SKIPPED,
relativePath
},
message: `File size of ${sizeInMB} MB exceeds the maximum file size limit of ${maxFileSizeMB
} MB`
message: `File size of ${sizeInMB} MB exceeds the maximum file size limit of ${
maxFileSizeMB
} MB`
};
}
}

2
frontend/sync-client/src/utils/data-structures/locks.ts
View file

@ -18,7 +18,7 @@ export class Locks<T> {
[() => unknown, (err: unknown) => unknown][]
>();
public constructor(private readonly logger?: Logger) { }
public constructor(private readonly logger?: Logger) {}
/**
* Executes a function while holding exclusive locks on one or more keys.