andras/vault-link
1
0
Fork 0
Code Issues Pull requests 18 Projects Releases 55 Packages Wiki Activity Actions 1

Simplify syncing logic

Browse source
This commit is contained in:
Andras Schmelczer 2026-03-28 11:55:37 +00:00
parent e8c57b3a37
commit 4493365076
48 changed files with 1054 additions and 918 deletions
Download patch file Download diff file Expand all files Collapse all files

167
frontend/sync-client/src/persistence/database.ts
View file

@ -9,6 +9,7 @@ export type DocumentId = string;
export type RelativePath = string;
export interface DocumentMetadata {
documentId: DocumentId;
parentVersionId: VaultUpdateId;
hash: string;
remoteRelativePath?: RelativePath;
@ -25,7 +26,6 @@ export interface StoredDocumentMetadata {
export interface StoredDatabase {
documents: StoredDocumentMetadata[];
lastSeenUpdateId: VaultUpdateId | undefined;
hasInitialSyncCompleted: boolean;
}
/**
@ -36,17 +36,14 @@ export interface StoredDatabase {
*/
export interface DocumentRecord {
relativePath: RelativePath;
documentId: DocumentId;
metadata: DocumentMetadata | undefined;
isDeleted: boolean;
updates: Promise<unknown>[];
parallelVersion: number;
}
export class Database {
private documents: DocumentRecord[];
private lastSeenUpdateIds: CoveredValues;
private hasInitialSyncCompleted: boolean;
public constructor(
private readonly logger: Logger,
@ -56,16 +53,12 @@ export class Database {
initialState ??= {};
this.documents =
initialState.documents?.map(
({ relativePath, documentId, ...metadata }) => ({
relativePath,
documentId,
metadata,
isDeleted: false,
updates: [],
parallelVersion: 0
})
) ?? [];
initialState.documents?.map(({ relativePath, ...metadata }) => ({
relativePath,
metadata,
isDeleted: false,
parallelVersion: 0
})) ?? [];
this.ensureConsistency();
this.logger.debug(`Loaded ${this.documents.length} documents`);
@ -79,12 +72,6 @@ export class Database {
this.documents.forEach((doc) => {
this.lastSeenUpdateIds.add(doc.metadata?.parentVersionId);
});
this.hasInitialSyncCompleted =
initialState.hasInitialSyncCompleted ?? false;
this.logger.debug(
`Loaded hasInitialSyncCompleted: ${this.hasInitialSyncCompleted}`
);
}
public get length(): number {
@ -127,91 +114,51 @@ export class Database {
public updateDocumentMetadata(
metadata: {
documentId: DocumentId;
parentVersionId: VaultUpdateId;
hash: string;
remoteRelativePath: RelativePath;
},
toUpdate: DocumentRecord
target: DocumentRecord
): void {
if (!this.documents.includes(toUpdate)) {
if (!this.documents.includes(target)) {
throw new Error("Document not found in database");
}
toUpdate.metadata = metadata;
this.saveInTheBackground();
}
public removeDocumentPromise(promise: Promise<unknown>): void {
const entry = this.documents.find(({ updates }) =>
updates.includes(promise)
this.logger.debug(
`Updating document metadata for ${target.relativePath} from ${JSON.stringify(
target.metadata,
null,
2
)} to ${JSON.stringify(metadata, null, 2)}`
);
if (entry === undefined) {
// This method should be idempotent and tolerant of
// stragglers calling it after the databse has been reset.
return;
}
target.metadata = metadata;
removeFromArray(entry.updates, promise);
// No need to save as Promises don't get serialized
}
public removeDocument(find: DocumentRecord): void {
removeFromArray(this.documents, find);
this.saveInTheBackground();
}
public getLatestDocumentByRelativePath(
find: RelativePath
target: RelativePath
): DocumentRecord | undefined {
const candidates = this.documents.filter(
({ relativePath }) => relativePath === find
({ relativePath }) => relativePath === target
);
candidates.sort((a, b) => b.parallelVersion - a.parallelVersion); // descending
return candidates[0];
}
public async getResolvedDocumentByRelativePath(
relativePath: RelativePath,
promise: Promise<unknown>
): Promise<DocumentRecord> {
const entry = this.getLatestDocumentByRelativePath(relativePath);
if (entry === undefined) {
throw new Error(
`Document not found by relative path: ${relativePath}, ${JSON.stringify(
this.documents,
null,
2
)}`
);
}
const currentPromises = entry.updates;
entry.updates = [...currentPromises, promise];
await awaitAll(currentPromises);
return entry;
}
public createNewPendingDocument(
documentId: DocumentId,
relativePath: RelativePath,
promise: Promise<unknown>
relativePath: RelativePath
): DocumentRecord {
this.logger.debug(
`Creating new pending document: ${relativePath} (${documentId})`
);
this.logger.debug(`Creating new pending document: ${relativePath}`);
const previousEntry =
this.getLatestDocumentByRelativePath(relativePath);
const entry = {
relativePath,
documentId,
metadata: undefined,
isDeleted: false,
updates: [promise],
parallelVersion:
previousEntry?.parallelVersion === undefined
? 0
@ -219,39 +166,18 @@ export class Database {
};
this.documents.push(entry);
this.saveInTheBackground();
return entry;
}
public createNewEmptyDocument(
documentId: DocumentId,
parentVersionId: VaultUpdateId,
relativePath: RelativePath
): DocumentRecord {
const entry = {
relativePath,
documentId,
metadata: {
parentVersionId,
hash: EMPTY_HASH,
remoteRelativePath: relativePath
},
isDeleted: false,
updates: [],
parallelVersion: 0
};
this.documents.push(entry);
this.saveInTheBackground();
// no need to save as we only save documents which have metadata
return entry;
}
public getDocumentByDocumentId(
find: DocumentId
target: DocumentId
): DocumentRecord | undefined {
return this.documents.find(({ documentId }) => documentId === find);
return this.documents.find(
({ metadata }) => metadata?.documentId === target
);
}
public move(
@ -274,7 +200,7 @@ export class Database {
}
oldDocument.relativePath = newRelativePath;
// We're in a strange state where the target of the move has just got deleted,
// We might be in a strange state where the target of the move has just got deleted,
// however, its metadata might already have a bunch of updates queued up for
// the document at the new location. We need to keep these updates.
oldDocument.parallelVersion =
@ -286,19 +212,13 @@ export class Database {
public delete(relativePath: RelativePath): void {
const candidate = this.getLatestDocumentByRelativePath(relativePath);
if (candidate === undefined) {
throw new Error(
`Document not found by relative path: ${relativePath}`
);
return;
}
candidate.isDeleted = true;
}
public getHasInitialSyncCompleted(): boolean {
return this.hasInitialSyncCompleted;
}
public setHasInitialSyncCompleted(value: boolean): void {
this.hasInitialSyncCompleted = value;
public removeDocument(target: DocumentRecord): void {
removeFromArray(this.documents, target);
this.saveInTheBackground();
}
@ -324,43 +244,50 @@ export class Database {
this.lastSeenUpdateIds = new CoveredValues(
0 // the first updateId will be 1 which is the first integer after -1
);
this.hasInitialSyncCompleted = false;
this.saveInTheBackground();
}
public async save(): Promise<void> {
return this.saveData({
documents: this.resolvedDocuments.map(
({ relativePath, documentId, metadata }) => ({
documentId,
({ relativePath, metadata }) => ({
relativePath,
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
...metadata! // `resolvedDocuments` only returns docs with metadata set
})
),
lastSeenUpdateId: this.lastSeenUpdateIds.min,
hasInitialSyncCompleted: this.hasInitialSyncCompleted
lastSeenUpdateId: this.lastSeenUpdateIds.min
});
}
private ensureConsistency(): void {
const idToPath = new Map<string, string[]>();
this.resolvedDocuments.forEach(({ relativePath, documentId }) => {
idToPath.set(documentId, [
...(idToPath.get(documentId) ?? []),
this.resolvedDocuments.forEach(({ relativePath, metadata }) => {
if (metadata === undefined) {
return;
}
idToPath.set(metadata.documentId, [
...(idToPath.get(metadata.documentId) ?? []),
relativePath
]);
});
const duplicates = Array.from(idToPath.entries())
.filter(([_, paths]) => paths.length > 1)
.map(([id, paths]) => `${id} (${paths.join(", ")})`);
.map(([id, paths]) => {
let details = "";
for (const path of paths) {
const doc = this.getLatestDocumentByRelativePath(path);
details += `\n- ${JSON.stringify(doc, null, 2)}`;
}
return `${id} (${paths.join(", ")}): ${details}`;
});
if (duplicates.length > 0) {
throw new Error(
"Document IDs are not unique, found duplicates: " +
duplicates.join("; ")
duplicates.join("; ")
);
}
}

4
frontend/sync-client/src/persistence/settings.ts
View file

@ -38,7 +38,7 @@ export class Settings {
>();
private settings: SyncSettings;
private readonly lock: Lock = new Lock();
private readonly lock: Lock;
public constructor(
private readonly logger: Logger,
@ -50,6 +50,8 @@ export class Settings {
...(initialState ?? {})
};
this.lock = new Lock(Settings.name, this.logger);
this.logger.debug(
`Loaded settings: ${JSON.stringify(this.settings, null, 2)}`
);