Fix syncing when network latency is present (#4)
* WIP * Add debug * Dedupe inserts * Add deterministic ordering * Fix whitespaces * Update insta * Add integration test script * Rename * Add test * Working for non-deletes * omg it mostly works for deletes * Isdeleted fix * remove created dates * update api * Take document id * No max attempt * works * Use string uuids * . * working!!!! (hopefully) * Improve bundling * Add module * lint * . * lint * Fix CI * use toolchain * clean up * Add useSlowFileEvents * Delete fuzz * Fix CI * use docker * fix script * clean up * Clean up * change node version * Build docker image on every commit * fix ci * 1 db per vault * Add scritps folder * Bump versions * Lint * . * Fix tests for real * Style * . * try * Consistent ordering * Fix tests * hmm * . * Clean up diff * Fixes * . * Fix version bump * . * . * .
This commit is contained in:
GitHub
parent
bcf48c428d
commit
8b8f1d91d9
91 changed files with 2252 additions and 1586 deletions
|
|
@ -1,23 +1,43 @@
|
|||
import type { Logger } from "../tracing/logger";
|
||||
|
||||
export type VaultUpdateId = number;
|
||||
export type DocumentId = string;
|
||||
export type RelativePath = string;
|
||||
|
||||
export interface DocumentMetadata {
|
||||
parentVersionId: VaultUpdateId;
|
||||
documentId: DocumentId;
|
||||
hash: string;
|
||||
}
|
||||
|
||||
import type { Logger } from "src/tracing/logger";
|
||||
export interface StoredDocumentMetadata {
|
||||
relativePath: RelativePath;
|
||||
documentId: DocumentId;
|
||||
parentVersionId: VaultUpdateId;
|
||||
hash: string;
|
||||
}
|
||||
|
||||
export interface StoredDatabase {
|
||||
documents: Record<RelativePath, DocumentMetadata>;
|
||||
documents: StoredDocumentMetadata[];
|
||||
lastSeenUpdateId: VaultUpdateId | undefined;
|
||||
}
|
||||
|
||||
export class Database {
|
||||
private documents = new Map<RelativePath, DocumentMetadata>();
|
||||
/**
|
||||
* Represents a document in the database.
|
||||
*
|
||||
* It is mutable and its content should always represent the latest
|
||||
* state of the document on disk based on the update events we have seen.
|
||||
*/
|
||||
export interface DocumentRecord {
|
||||
relativePath: RelativePath;
|
||||
documentId: DocumentId;
|
||||
metadata: DocumentMetadata | undefined;
|
||||
isDeleted: boolean;
|
||||
updates: Promise<void>[];
|
||||
parallelVersion: number;
|
||||
}
|
||||
|
||||
export class Database {
|
||||
private documents: DocumentRecord[];
|
||||
private lastSeenUpdateId: VaultUpdateId | undefined;
|
||||
|
||||
public constructor(
|
||||
|
|
@ -26,16 +46,21 @@ export class Database {
|
|||
private readonly saveData: (data: StoredDatabase) => Promise<void>
|
||||
) {
|
||||
initialState ??= {};
|
||||
if (initialState.documents) {
|
||||
for (const [relativePath, metadata] of Object.entries(
|
||||
initialState.documents
|
||||
)) {
|
||||
this.documents.set(relativePath, metadata);
|
||||
}
|
||||
}
|
||||
this.ensureConsistency();
|
||||
|
||||
this.logger.debug(`Loaded ${this.documents.size} documents`);
|
||||
this.documents =
|
||||
initialState.documents?.map(
|
||||
({ relativePath, documentId, ...metadata }) => ({
|
||||
relativePath,
|
||||
documentId,
|
||||
metadata,
|
||||
isDeleted: false,
|
||||
updates: [],
|
||||
parallelVersion: 0
|
||||
})
|
||||
) ?? [];
|
||||
|
||||
this.ensureConsistency();
|
||||
this.logger.debug(`Loaded ${this.documents.length} documents`);
|
||||
|
||||
this.lastSeenUpdateId = initialState.lastSeenUpdateId;
|
||||
this.logger.debug(
|
||||
|
|
@ -43,109 +68,213 @@ export class Database {
|
|||
);
|
||||
}
|
||||
|
||||
public getDocuments(): Map<RelativePath, DocumentMetadata> {
|
||||
return this.documents;
|
||||
public get length(): number {
|
||||
return this.documents.length;
|
||||
}
|
||||
|
||||
public get resolvedDocuments(): DocumentRecord[] {
|
||||
const paths = new Map<string, DocumentRecord[]>();
|
||||
this.documents
|
||||
.filter(({ metadata }) => metadata !== undefined)
|
||||
.forEach((record) =>
|
||||
paths.set(record.relativePath, [
|
||||
record,
|
||||
...(paths.get(record.relativePath) ?? [])
|
||||
])
|
||||
);
|
||||
|
||||
return Array.from(paths.values()).map((records) => {
|
||||
records.sort(
|
||||
(a, b) => b.parallelVersion - a.parallelVersion // descending
|
||||
);
|
||||
|
||||
if (
|
||||
records.length > 1 &&
|
||||
records.some((current, i) =>
|
||||
i === 0
|
||||
? false
|
||||
: records[i - 1].parallelVersion ===
|
||||
current.parallelVersion
|
||||
)
|
||||
) {
|
||||
throw new Error(
|
||||
`Multiple documents with the same parallel version and path at ${records[0].relativePath}`
|
||||
);
|
||||
}
|
||||
return records[0];
|
||||
});
|
||||
}
|
||||
|
||||
public getLastSeenUpdateId(): VaultUpdateId | undefined {
|
||||
return this.lastSeenUpdateId;
|
||||
}
|
||||
|
||||
public async setLastSeenUpdateId(
|
||||
value: VaultUpdateId | undefined
|
||||
): Promise<void> {
|
||||
public setLastSeenUpdateId(value: VaultUpdateId | undefined): void {
|
||||
this.lastSeenUpdateId = value;
|
||||
await this.save();
|
||||
this.save();
|
||||
}
|
||||
|
||||
public async resetSyncState(): Promise<void> {
|
||||
this.documents = new Map();
|
||||
public resetSyncState(): void {
|
||||
this.documents = [];
|
||||
this.lastSeenUpdateId = 0;
|
||||
await this.save();
|
||||
this.save();
|
||||
}
|
||||
|
||||
public updateDocumentMetadata(
|
||||
metadata: {
|
||||
parentVersionId: VaultUpdateId;
|
||||
hash: string;
|
||||
},
|
||||
toUpdate: DocumentRecord
|
||||
): void {
|
||||
if (!this.documents.includes(toUpdate)) {
|
||||
throw new Error("Document not found in database");
|
||||
}
|
||||
|
||||
toUpdate.metadata = metadata;
|
||||
|
||||
this.save();
|
||||
}
|
||||
|
||||
public removeDocumentPromise(promise: Promise<void>): void {
|
||||
const entry = this.documents.find(({ updates }) =>
|
||||
updates.includes(promise)
|
||||
);
|
||||
|
||||
if (entry === undefined) {
|
||||
throw new Error("Document not found by update promise");
|
||||
}
|
||||
|
||||
entry.updates = entry.updates.filter((update) => update !== promise);
|
||||
// No need to save as Promises don't get serialized
|
||||
}
|
||||
|
||||
public getLatestDocumentByRelativePath(
|
||||
find: RelativePath
|
||||
): DocumentRecord | undefined {
|
||||
const candidates = this.documents.filter(
|
||||
({ relativePath }) => relativePath === find
|
||||
);
|
||||
candidates.sort((a, b) => b.parallelVersion - a.parallelVersion); // descending
|
||||
return candidates[0];
|
||||
}
|
||||
|
||||
public async getResolvedDocumentByRelativePath(
|
||||
relativePath: RelativePath,
|
||||
promise: Promise<void>
|
||||
): Promise<DocumentRecord> {
|
||||
const entry = this.getLatestDocumentByRelativePath(relativePath);
|
||||
|
||||
if (entry === undefined) {
|
||||
throw new Error(
|
||||
`Document not found by relative path: ${relativePath}, ${JSON.stringify(
|
||||
this.documents,
|
||||
null,
|
||||
2
|
||||
)}`
|
||||
);
|
||||
}
|
||||
|
||||
const currentPromises = entry.updates;
|
||||
entry.updates = [...currentPromises, promise];
|
||||
await Promise.all(currentPromises);
|
||||
|
||||
return entry;
|
||||
}
|
||||
|
||||
public createNewPendingDocument(
|
||||
documentId: DocumentId,
|
||||
relativePath: RelativePath,
|
||||
promise: Promise<void>
|
||||
): DocumentRecord {
|
||||
const previousEntry =
|
||||
this.getLatestDocumentByRelativePath(relativePath);
|
||||
|
||||
const entry = {
|
||||
relativePath,
|
||||
documentId,
|
||||
metadata: undefined,
|
||||
isDeleted: false,
|
||||
updates: [promise],
|
||||
parallelVersion:
|
||||
previousEntry?.parallelVersion === undefined
|
||||
? 0
|
||||
: previousEntry.parallelVersion + 1
|
||||
};
|
||||
|
||||
this.documents.push(entry);
|
||||
this.save();
|
||||
|
||||
return entry;
|
||||
}
|
||||
|
||||
public getDocumentByDocumentId(
|
||||
documentId: DocumentId
|
||||
): [RelativePath, DocumentMetadata] | undefined {
|
||||
return [...this.documents.entries()].find(
|
||||
([_, metadata]) => metadata.documentId === documentId
|
||||
);
|
||||
find: DocumentId
|
||||
): DocumentRecord | undefined {
|
||||
return this.documents.find(({ documentId }) => documentId === find);
|
||||
}
|
||||
|
||||
public async setDocument({
|
||||
documentId,
|
||||
relativePath,
|
||||
parentVersionId,
|
||||
hash
|
||||
}: {
|
||||
documentId: DocumentId;
|
||||
relativePath: RelativePath;
|
||||
parentVersionId: VaultUpdateId;
|
||||
hash: string;
|
||||
}): Promise<void> {
|
||||
this.documents.set(relativePath, {
|
||||
documentId,
|
||||
parentVersionId,
|
||||
hash
|
||||
});
|
||||
await this.save();
|
||||
}
|
||||
|
||||
public async removeDocument(relativePath: RelativePath): Promise<void> {
|
||||
this.documents.delete(relativePath);
|
||||
await this.save();
|
||||
}
|
||||
|
||||
public getDocument(
|
||||
relativePath: RelativePath
|
||||
): DocumentMetadata | undefined {
|
||||
return this.documents.get(relativePath);
|
||||
}
|
||||
|
||||
public async deleteDocument(relativePath: RelativePath): Promise<void> {
|
||||
this.documents.delete(relativePath);
|
||||
await this.save();
|
||||
}
|
||||
|
||||
public async updatePath(
|
||||
public move(
|
||||
oldRelativePath: RelativePath,
|
||||
newRelativePath: RelativePath
|
||||
): Promise<void> {
|
||||
const document = this.documents.get(oldRelativePath);
|
||||
if (!document) {
|
||||
): void {
|
||||
const oldDocument =
|
||||
this.getLatestDocumentByRelativePath(oldRelativePath);
|
||||
|
||||
if (oldDocument === undefined) {
|
||||
return;
|
||||
}
|
||||
|
||||
const newDocument =
|
||||
this.getLatestDocumentByRelativePath(newRelativePath);
|
||||
if (newDocument?.isDeleted === false) {
|
||||
throw new Error(
|
||||
`Cannot update physical path for document that does not exist: ${oldRelativePath}`
|
||||
`Document already exists at new location: ${newRelativePath}`
|
||||
);
|
||||
}
|
||||
|
||||
if (this.documents.has(newRelativePath)) {
|
||||
throw new Error(
|
||||
`Cannot update physical path to path that is already in use: ${newRelativePath}`
|
||||
);
|
||||
}
|
||||
oldDocument.relativePath = newRelativePath;
|
||||
// We're in a strange state where the target of the move has just got deleted,
|
||||
// however, its metadata might already have a bunch of updates queued up for
|
||||
// the document at the new location. We need to keep these updates.
|
||||
oldDocument.parallelVersion =
|
||||
newDocument !== undefined ? newDocument.parallelVersion + 1 : 0;
|
||||
|
||||
this.documents.delete(oldRelativePath);
|
||||
this.documents.set(newRelativePath, document);
|
||||
|
||||
await this.save();
|
||||
this.save();
|
||||
}
|
||||
|
||||
private async save(): Promise<void> {
|
||||
public delete(relativePath: RelativePath): void {
|
||||
const candidate = this.getLatestDocumentByRelativePath(relativePath);
|
||||
if (candidate === undefined) {
|
||||
throw new Error(
|
||||
`Document not found by relative path: ${relativePath}`
|
||||
);
|
||||
}
|
||||
candidate.isDeleted = true;
|
||||
}
|
||||
|
||||
private save(): void {
|
||||
this.ensureConsistency();
|
||||
await this.saveData({
|
||||
documents: Object.fromEntries(this.documents.entries()),
|
||||
void this.saveData({
|
||||
documents: this.resolvedDocuments.map(
|
||||
({ relativePath, documentId, metadata }) => ({
|
||||
documentId,
|
||||
relativePath,
|
||||
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
||||
...metadata! // resolvedDocuments only returns docs with metadata set
|
||||
})
|
||||
),
|
||||
lastSeenUpdateId: this.lastSeenUpdateId
|
||||
});
|
||||
}
|
||||
|
||||
private ensureConsistency(): void {
|
||||
const allMetadata = Array.from(this.documents.entries());
|
||||
const idToPath = new Map<string, Array<string>>();
|
||||
const idToPath = new Map<string, string[]>();
|
||||
|
||||
allMetadata.forEach(([name, metadata]) => {
|
||||
idToPath.set(metadata.documentId, [
|
||||
...(idToPath.get(metadata.documentId) ?? []),
|
||||
name
|
||||
this.resolvedDocuments.forEach(({ relativePath, documentId }) => {
|
||||
idToPath.set(documentId, [
|
||||
...(idToPath.get(documentId) ?? []),
|
||||
relativePath
|
||||
]);
|
||||
});
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue