clean up
This commit is contained in:
parent
2987afb20a
commit
d5112a7d0f
7 changed files with 147 additions and 272 deletions
|
|
@ -17,10 +17,6 @@
|
||||||
|
|
||||||
- Install [`rustup`](https://rustup.rs): `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh`
|
- Install [`rustup`](https://rustup.rs): `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh`
|
||||||
- `sudo apt install llvm -y`
|
- `sudo apt install llvm -y`
|
||||||
- `rustup self update`
|
|
||||||
- `rustup update`
|
|
||||||
- `rustup install nightly`
|
|
||||||
- `rustup default nightly`
|
|
||||||
- `rustup component add llvm-tools-preview`
|
- `rustup component add llvm-tools-preview`
|
||||||
- `cargo install cargo-generate cargo-fuzz cargo-insta rustfilt cargo-binutils`
|
- `cargo install cargo-generate cargo-fuzz cargo-insta rustfilt cargo-binutils`
|
||||||
- Install [`wasm-pack`](https://rustwasm.github.io/wasm-pack/installer): `curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh`
|
- Install [`wasm-pack`](https://rustwasm.github.io/wasm-pack/installer): `curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh`
|
||||||
|
|
|
||||||
|
|
@ -2,10 +2,7 @@ import type { Logger } from "../tracing/logger";
|
||||||
import type { FileSystemOperations } from "./filesystem-operations";
|
import type { FileSystemOperations } from "./filesystem-operations";
|
||||||
import type { Database, RelativePath } from "../persistence/database";
|
import type { Database, RelativePath } from "../persistence/database";
|
||||||
import { isBinary, isFileTypeMergable, mergeText } from "sync_lib";
|
import { isBinary, isFileTypeMergable, mergeText } from "sync_lib";
|
||||||
import {
|
import { SafeFileSystemOperations } from "./safe-filesystem-operations";
|
||||||
FileNotFoundError,
|
|
||||||
SafeFileSystemOperations
|
|
||||||
} from "./safe-filesystem-operations";
|
|
||||||
|
|
||||||
export class FileOperations {
|
export class FileOperations {
|
||||||
private static readonly PARENTHESES_REGEX = / \((\d+)\)$/;
|
private static readonly PARENTHESES_REGEX = / \((\d+)\)$/;
|
||||||
|
|
@ -67,7 +64,7 @@ export class FileOperations {
|
||||||
`Didn't expect ${path} to exist, deconflicting by moving it to '${deconflictedPath}'`
|
`Didn't expect ${path} to exist, deconflicting by moving it to '${deconflictedPath}'`
|
||||||
);
|
);
|
||||||
|
|
||||||
// this.database.move(path, deconflictedPath);
|
this.database.move(path, deconflictedPath);
|
||||||
await this.fs.rename(path, deconflictedPath);
|
await this.fs.rename(path, deconflictedPath);
|
||||||
} else {
|
} else {
|
||||||
await this.createParentDirectories(path);
|
await this.createParentDirectories(path);
|
||||||
|
|
@ -142,9 +139,9 @@ export class FileOperations {
|
||||||
if (oldPath === newPath) {
|
if (oldPath === newPath) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
await this.ensureClearPath(newPath);
|
await this.ensureClearPath(newPath);
|
||||||
|
|
||||||
|
this.database.move(oldPath, newPath);
|
||||||
await this.fs.rename(oldPath, newPath);
|
await this.fs.rename(oldPath, newPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -22,7 +22,6 @@ export interface StoredDatabase {
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface DocumentRecord {
|
export interface DocumentRecord {
|
||||||
identity: symbol;
|
|
||||||
relativePath: RelativePath;
|
relativePath: RelativePath;
|
||||||
documentId: DocumentId;
|
documentId: DocumentId;
|
||||||
metadata: DocumentMetadata | undefined;
|
metadata: DocumentMetadata | undefined;
|
||||||
|
|
@ -47,7 +46,6 @@ export class Database {
|
||||||
({ relativePath, documentId, ...metadata }) => ({
|
({ relativePath, documentId, ...metadata }) => ({
|
||||||
relativePath,
|
relativePath,
|
||||||
documentId,
|
documentId,
|
||||||
identity: Symbol(),
|
|
||||||
metadata,
|
metadata,
|
||||||
isDeleted: false,
|
isDeleted: false,
|
||||||
updates: [],
|
updates: [],
|
||||||
|
|
@ -118,85 +116,33 @@ export class Database {
|
||||||
|
|
||||||
public setDocument(
|
public setDocument(
|
||||||
{
|
{
|
||||||
documentId,
|
|
||||||
relativePath,
|
|
||||||
parentVersionId,
|
parentVersionId,
|
||||||
hash
|
hash
|
||||||
}: {
|
}: {
|
||||||
documentId: DocumentId;
|
|
||||||
relativePath: RelativePath;
|
|
||||||
parentVersionId: VaultUpdateId;
|
parentVersionId: VaultUpdateId;
|
||||||
hash: string;
|
hash: string;
|
||||||
},
|
},
|
||||||
identity?: symbol
|
toUpdate: DocumentRecord
|
||||||
): void {
|
): void {
|
||||||
if (identity !== undefined) {
|
if (!this.documents.includes(toUpdate)) {
|
||||||
const entry = this.getDocumentByIdentity(identity);
|
throw new Error("Document not found in database");
|
||||||
|
|
||||||
this.documents = this.documents.filter(
|
|
||||||
(doc) => doc.identity !== entry.identity
|
|
||||||
);
|
|
||||||
|
|
||||||
if (entry.relativePath !== relativePath) {
|
|
||||||
throw new Error(
|
|
||||||
"Document identity does not match the relative path"
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
this.documents.push({
|
toUpdate.metadata = { parentVersionId, hash };
|
||||||
...entry,
|
|
||||||
relativePath,
|
|
||||||
documentId,
|
|
||||||
metadata: {
|
|
||||||
parentVersionId,
|
|
||||||
hash
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
this.save();
|
this.save();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// We find a match based on relative path and we find one with a different document id
|
|
||||||
// meaning that two documents occupy the same path in terms of in-flight requests so we
|
|
||||||
// need to create a new parallel version.
|
|
||||||
const entry = this.getLatestDocumentByRelativePath(relativePath);
|
|
||||||
if (entry && entry.documentId !== documentId) {
|
|
||||||
this.documents.push({
|
|
||||||
// `entry` might be undefined if the document is new
|
|
||||||
identity: Symbol(),
|
|
||||||
relativePath,
|
|
||||||
documentId,
|
|
||||||
metadata: {
|
|
||||||
parentVersionId,
|
|
||||||
hash
|
|
||||||
},
|
|
||||||
isDeleted: false,
|
|
||||||
updates: [],
|
|
||||||
parallelVersion: entry.parallelVersion + 1
|
|
||||||
});
|
|
||||||
this.save();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
this.documents.push({
|
|
||||||
identity: Symbol(),
|
|
||||||
relativePath,
|
|
||||||
documentId,
|
|
||||||
metadata: {
|
|
||||||
parentVersionId,
|
|
||||||
hash
|
|
||||||
},
|
|
||||||
isDeleted: false,
|
|
||||||
updates: [],
|
|
||||||
parallelVersion: 0
|
|
||||||
});
|
|
||||||
|
|
||||||
this.save();
|
|
||||||
}
|
|
||||||
|
|
||||||
public removeDocumentPromise(promise: Promise<void>): void {
|
public removeDocumentPromise(promise: Promise<void>): void {
|
||||||
const entry = this.getDocumentByUpdatePromise(promise);
|
const entry = this.documents.find(({ updates }) =>
|
||||||
|
updates.includes(promise)
|
||||||
|
);
|
||||||
|
|
||||||
|
if (entry === undefined) {
|
||||||
|
throw new Error("Document not found by update promise");
|
||||||
|
}
|
||||||
|
|
||||||
entry.updates = entry.updates.filter((update) => update !== promise);
|
entry.updates = entry.updates.filter((update) => update !== promise);
|
||||||
// No need to save as Promises don't get serialized
|
// No need to save as Promises don't get serialized
|
||||||
}
|
}
|
||||||
|
|
@ -214,7 +160,7 @@ export class Database {
|
||||||
public async getResolvedDocumentByRelativePath(
|
public async getResolvedDocumentByRelativePath(
|
||||||
relativePath: RelativePath,
|
relativePath: RelativePath,
|
||||||
promise: Promise<void>
|
promise: Promise<void>
|
||||||
): Promise<void> {
|
): Promise<DocumentRecord> {
|
||||||
const entry = this.getLatestDocumentByRelativePath(relativePath);
|
const entry = this.getLatestDocumentByRelativePath(relativePath);
|
||||||
|
|
||||||
if (entry === undefined) {
|
if (entry === undefined) {
|
||||||
|
|
@ -230,20 +176,21 @@ export class Database {
|
||||||
const currentPromises = entry.updates;
|
const currentPromises = entry.updates;
|
||||||
entry.updates = [...currentPromises, promise];
|
entry.updates = [...currentPromises, promise];
|
||||||
await Promise.all(currentPromises);
|
await Promise.all(currentPromises);
|
||||||
|
|
||||||
|
return entry;
|
||||||
}
|
}
|
||||||
|
|
||||||
public getNewResolvedDocumentByRelativePath(
|
public createNewPendingDocument(
|
||||||
documentId: DocumentId,
|
documentId: DocumentId,
|
||||||
relativePath: RelativePath,
|
relativePath: RelativePath,
|
||||||
promise: Promise<void>
|
promise: Promise<void>
|
||||||
): void {
|
): DocumentRecord {
|
||||||
const previousEntry =
|
const previousEntry =
|
||||||
this.getLatestDocumentByRelativePath(relativePath);
|
this.getLatestDocumentByRelativePath(relativePath);
|
||||||
|
|
||||||
const entry = {
|
const entry = {
|
||||||
relativePath,
|
relativePath,
|
||||||
documentId,
|
documentId,
|
||||||
identity: Symbol(),
|
|
||||||
metadata: undefined,
|
metadata: undefined,
|
||||||
isDeleted: false,
|
isDeleted: false,
|
||||||
updates: [promise],
|
updates: [promise],
|
||||||
|
|
@ -255,18 +202,8 @@ export class Database {
|
||||||
|
|
||||||
this.documents.push(entry);
|
this.documents.push(entry);
|
||||||
this.save();
|
this.save();
|
||||||
}
|
|
||||||
|
|
||||||
public getDocumentByUpdatePromise(promise: Promise<void>): DocumentRecord {
|
return entry;
|
||||||
const result = this.documents.find(({ updates }) =>
|
|
||||||
updates.includes(promise)
|
|
||||||
);
|
|
||||||
|
|
||||||
if (result === undefined) {
|
|
||||||
throw new Error("Document not found by update promise");
|
|
||||||
}
|
|
||||||
|
|
||||||
return result;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public getDocumentByDocumentId(
|
public getDocumentByDocumentId(
|
||||||
|
|
@ -275,16 +212,6 @@ export class Database {
|
||||||
return this.documents.find(({ documentId }) => documentId === find);
|
return this.documents.find(({ documentId }) => documentId === find);
|
||||||
}
|
}
|
||||||
|
|
||||||
public getDocumentByIdentity(find: symbol): DocumentRecord {
|
|
||||||
const result = this.documents.find(({ identity }) => identity === find);
|
|
||||||
|
|
||||||
if (result === undefined) {
|
|
||||||
throw new Error("Document not found by identity symbol");
|
|
||||||
}
|
|
||||||
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
public move(
|
public move(
|
||||||
oldRelativePath: RelativePath,
|
oldRelativePath: RelativePath,
|
||||||
newRelativePath: RelativePath
|
newRelativePath: RelativePath
|
||||||
|
|
@ -296,10 +223,6 @@ export class Database {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
this.documents = this.documents.filter(
|
|
||||||
({ identity }) => identity !== oldDocument.identity
|
|
||||||
);
|
|
||||||
|
|
||||||
const newDocument =
|
const newDocument =
|
||||||
this.getLatestDocumentByRelativePath(newRelativePath);
|
this.getLatestDocumentByRelativePath(newRelativePath);
|
||||||
if (newDocument !== undefined && !newDocument.isDeleted) {
|
if (newDocument !== undefined && !newDocument.isDeleted) {
|
||||||
|
|
@ -308,17 +231,12 @@ export class Database {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// It's either an invalid state of newDocument is pending deletion and we have
|
oldDocument.relativePath = newRelativePath;
|
||||||
// to wait for it to complete.
|
|
||||||
this.documents.push({
|
|
||||||
...oldDocument,
|
|
||||||
relativePath: newRelativePath,
|
|
||||||
// We're in a strange state where the target of the move has just got deleted,
|
// We're in a strange state where the target of the move has just got deleted,
|
||||||
// however, its metadata might already have a bunch of updates queued up for
|
// however, its metadata might already have a bunch of updates queued up for
|
||||||
// the document at the new location. We need to keep these updates.
|
// the document at the new location. We need to keep these updates.
|
||||||
parallelVersion:
|
oldDocument.parallelVersion =
|
||||||
newDocument !== undefined ? newDocument.parallelVersion + 1 : 0
|
newDocument !== undefined ? newDocument.parallelVersion + 1 : 0;
|
||||||
});
|
|
||||||
|
|
||||||
this.save();
|
this.save();
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -3,8 +3,8 @@ import type { SyncService } from "../services/sync-service";
|
||||||
import type { Logger } from "../tracing/logger";
|
import type { Logger } from "../tracing/logger";
|
||||||
import type { SyncHistory } from "../tracing/sync-history";
|
import type { SyncHistory } from "../tracing/sync-history";
|
||||||
import PQueue from "p-queue";
|
import PQueue from "p-queue";
|
||||||
import { v4 as uuidv4 } from "uuid";
|
|
||||||
import { hash } from "../utils/hash";
|
import { hash } from "../utils/hash";
|
||||||
|
import { v4 as uuidv4 } from "uuid";
|
||||||
import type { components } from "../services/types";
|
import type { components } from "../services/types";
|
||||||
import type { Settings } from "../persistence/settings";
|
import type { Settings } from "../persistence/settings";
|
||||||
import type { FileOperations } from "../file-operations/file-operations";
|
import type { FileOperations } from "../file-operations/file-operations";
|
||||||
|
|
@ -98,20 +98,16 @@ export class Syncer {
|
||||||
}
|
}
|
||||||
|
|
||||||
const [promise, resolve, reject] = createPromise();
|
const [promise, resolve, reject] = createPromise();
|
||||||
const proposedDocumentId = uuidv4();
|
|
||||||
|
|
||||||
this.database.getNewResolvedDocumentByRelativePath(
|
const document = this.database.createNewPendingDocument(
|
||||||
proposedDocumentId,
|
uuidv4(),
|
||||||
relativePath,
|
relativePath,
|
||||||
promise
|
promise
|
||||||
);
|
);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await this.syncQueue.add(async () =>
|
await this.syncQueue.add(async () =>
|
||||||
this.internalSyncer.unrestrictedSyncLocallyCreatedFile(
|
this.internalSyncer.unrestrictedSyncLocallyCreatedFile(document)
|
||||||
proposedDocumentId,
|
|
||||||
() => this.database.getDocumentByUpdatePromise(promise)
|
|
||||||
)
|
|
||||||
);
|
);
|
||||||
|
|
||||||
resolve();
|
resolve();
|
||||||
|
|
@ -131,16 +127,14 @@ export class Syncer {
|
||||||
|
|
||||||
const [promise, resolve, reject] = createPromise();
|
const [promise, resolve, reject] = createPromise();
|
||||||
|
|
||||||
await this.database.getResolvedDocumentByRelativePath(
|
const document = await this.database.getResolvedDocumentByRelativePath(
|
||||||
relativePath,
|
relativePath,
|
||||||
promise
|
promise
|
||||||
);
|
);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await this.syncQueue.add(async () =>
|
await this.syncQueue.add(async () =>
|
||||||
this.internalSyncer.unrestrictedSyncLocallyDeletedFile(() =>
|
this.internalSyncer.unrestrictedSyncLocallyDeletedFile(document)
|
||||||
this.database.getDocumentByUpdatePromise(promise)
|
|
||||||
)
|
|
||||||
);
|
);
|
||||||
|
|
||||||
resolve();
|
resolve();
|
||||||
|
|
@ -158,17 +152,13 @@ export class Syncer {
|
||||||
oldPath?: RelativePath;
|
oldPath?: RelativePath;
|
||||||
relativePath: RelativePath;
|
relativePath: RelativePath;
|
||||||
}): Promise<void> {
|
}): Promise<void> {
|
||||||
if (oldPath !== undefined) {
|
|
||||||
if (
|
if (
|
||||||
this.database.getLatestDocumentByRelativePath(oldPath)
|
oldPath !== undefined &&
|
||||||
?.isDeleted === true
|
(this.database.getLatestDocumentByRelativePath(relativePath) ===
|
||||||
|
undefined ||
|
||||||
|
this.database.getLatestDocumentByRelativePath(relativePath)
|
||||||
|
?.isDeleted === true)
|
||||||
) {
|
) {
|
||||||
this.logger.debug(
|
|
||||||
`Document ${oldPath} has been deleted locally, skipping`
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (oldPath === relativePath) {
|
if (oldPath === relativePath) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`Old path and new path are the same: ${oldPath}`
|
`Old path and new path are the same: ${oldPath}`
|
||||||
|
|
@ -178,10 +168,17 @@ export class Syncer {
|
||||||
this.database.move(oldPath, relativePath);
|
this.database.move(oldPath, relativePath);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (
|
let document =
|
||||||
this.database.getLatestDocumentByRelativePath(relativePath)
|
this.database.getLatestDocumentByRelativePath(relativePath);
|
||||||
?.isDeleted === true
|
|
||||||
) {
|
if (document === undefined) {
|
||||||
|
this.logger.debug(
|
||||||
|
`Cannot find document ${relativePath} in the database, skipping`
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (document.isDeleted) {
|
||||||
this.logger.debug(
|
this.logger.debug(
|
||||||
`Document ${relativePath} has been deleted locally, skipping`
|
`Document ${relativePath} has been deleted locally, skipping`
|
||||||
);
|
);
|
||||||
|
|
@ -190,7 +187,7 @@ export class Syncer {
|
||||||
|
|
||||||
const [promise, resolve, reject] = createPromise();
|
const [promise, resolve, reject] = createPromise();
|
||||||
|
|
||||||
await this.database.getResolvedDocumentByRelativePath(
|
document = await this.database.getResolvedDocumentByRelativePath(
|
||||||
relativePath,
|
relativePath,
|
||||||
promise
|
promise
|
||||||
);
|
);
|
||||||
|
|
@ -199,8 +196,7 @@ export class Syncer {
|
||||||
await this.syncQueue.add(async () =>
|
await this.syncQueue.add(async () =>
|
||||||
this.internalSyncer.unrestrictedSyncLocallyUpdatedFile({
|
this.internalSyncer.unrestrictedSyncLocallyUpdatedFile({
|
||||||
oldPath,
|
oldPath,
|
||||||
getLatestDocument: () =>
|
document
|
||||||
this.database.getDocumentByUpdatePromise(promise)
|
|
||||||
})
|
})
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
@ -299,7 +295,7 @@ export class Syncer {
|
||||||
private async syncRemotelyUpdatedFile(
|
private async syncRemotelyUpdatedFile(
|
||||||
remoteVersion: components["schemas"]["DocumentVersionWithoutContent"]
|
remoteVersion: components["schemas"]["DocumentVersionWithoutContent"]
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const document = this.database.getDocumentByDocumentId(
|
let document = this.database.getDocumentByDocumentId(
|
||||||
remoteVersion.documentId
|
remoteVersion.documentId
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
@ -308,15 +304,11 @@ export class Syncer {
|
||||||
if (document === undefined) {
|
if (document === undefined) {
|
||||||
await this.syncQueue.add(async () =>
|
await this.syncQueue.add(async () =>
|
||||||
this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
|
this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
|
||||||
remoteVersion,
|
remoteVersion
|
||||||
() =>
|
|
||||||
this.database.getDocumentByDocumentId(
|
|
||||||
remoteVersion.documentId
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
await this.database.getResolvedDocumentByRelativePath(
|
document = await this.database.getResolvedDocumentByRelativePath(
|
||||||
document.relativePath,
|
document.relativePath,
|
||||||
promise
|
promise
|
||||||
);
|
);
|
||||||
|
|
@ -325,7 +317,7 @@ export class Syncer {
|
||||||
await this.syncQueue.add(async () =>
|
await this.syncQueue.add(async () =>
|
||||||
this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
|
this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
|
||||||
remoteVersion,
|
remoteVersion,
|
||||||
() => this.database.getDocumentByUpdatePromise(promise)
|
document
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,5 @@
|
||||||
import type {
|
import type {
|
||||||
Database,
|
Database,
|
||||||
DocumentId,
|
|
||||||
DocumentRecord,
|
DocumentRecord,
|
||||||
RelativePath
|
RelativePath
|
||||||
} from "../persistence/database";
|
} from "../persistence/database";
|
||||||
|
|
@ -33,31 +32,24 @@ export class UnrestrictedSyncer {
|
||||||
}
|
}
|
||||||
|
|
||||||
public async unrestrictedSyncLocallyCreatedFile(
|
public async unrestrictedSyncLocallyCreatedFile(
|
||||||
proposedDocumentId: DocumentId,
|
document: DocumentRecord
|
||||||
getLatestDocument: () => DocumentRecord
|
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
let document = getLatestDocument();
|
|
||||||
|
|
||||||
return this.executeSync(
|
return this.executeSync(
|
||||||
[document.relativePath],
|
document.relativePath,
|
||||||
SyncType.CREATE,
|
SyncType.CREATE,
|
||||||
SyncSource.PUSH,
|
SyncSource.PUSH,
|
||||||
async () => {
|
async () => {
|
||||||
document = getLatestDocument();
|
|
||||||
|
|
||||||
const contentBytes = await this.operations.read(
|
const contentBytes = await this.operations.read(
|
||||||
document.relativePath
|
document.relativePath
|
||||||
); // this can throw FileNotFoundError
|
); // this can throw FileNotFoundError
|
||||||
const contentHash = hash(contentBytes);
|
const contentHash = hash(contentBytes);
|
||||||
|
|
||||||
const response = await this.syncService.create({
|
const response = await this.syncService.create({
|
||||||
documentId: proposedDocumentId,
|
documentId: document.documentId,
|
||||||
relativePath: document.relativePath,
|
relativePath: document.relativePath,
|
||||||
contentBytes
|
contentBytes
|
||||||
});
|
});
|
||||||
|
|
||||||
document = getLatestDocument();
|
|
||||||
|
|
||||||
this.history.addHistoryEntry({
|
this.history.addHistoryEntry({
|
||||||
status: SyncStatus.SUCCESS,
|
status: SyncStatus.SUCCESS,
|
||||||
source: SyncSource.PUSH,
|
source: SyncSource.PUSH,
|
||||||
|
|
@ -68,12 +60,10 @@ export class UnrestrictedSyncer {
|
||||||
|
|
||||||
this.database.setDocument(
|
this.database.setDocument(
|
||||||
{
|
{
|
||||||
relativePath: document.relativePath,
|
|
||||||
documentId: response.documentId,
|
|
||||||
parentVersionId: response.vaultUpdateId,
|
parentVersionId: response.vaultUpdateId,
|
||||||
hash: contentHash
|
hash: contentHash
|
||||||
},
|
},
|
||||||
document.identity
|
document
|
||||||
);
|
);
|
||||||
|
|
||||||
this.tryIncrementVaultUpdateId(response.vaultUpdateId);
|
this.tryIncrementVaultUpdateId(response.vaultUpdateId);
|
||||||
|
|
@ -82,16 +72,13 @@ export class UnrestrictedSyncer {
|
||||||
}
|
}
|
||||||
|
|
||||||
public async unrestrictedSyncLocallyDeletedFile(
|
public async unrestrictedSyncLocallyDeletedFile(
|
||||||
getLatestDocument: () => DocumentRecord
|
document: DocumentRecord
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
let document = getLatestDocument();
|
|
||||||
await this.executeSync(
|
await this.executeSync(
|
||||||
[document.relativePath],
|
document.relativePath,
|
||||||
SyncType.DELETE,
|
SyncType.DELETE,
|
||||||
SyncSource.PUSH,
|
SyncSource.PUSH,
|
||||||
async () => {
|
async () => {
|
||||||
document = getLatestDocument();
|
|
||||||
|
|
||||||
const response = await this.syncService.delete({
|
const response = await this.syncService.delete({
|
||||||
documentId: document.documentId,
|
documentId: document.documentId,
|
||||||
relativePath: document.relativePath
|
relativePath: document.relativePath
|
||||||
|
|
@ -105,16 +92,12 @@ export class UnrestrictedSyncer {
|
||||||
type: SyncType.DELETE
|
type: SyncType.DELETE
|
||||||
});
|
});
|
||||||
|
|
||||||
document = getLatestDocument();
|
|
||||||
|
|
||||||
this.database.setDocument(
|
this.database.setDocument(
|
||||||
{
|
{
|
||||||
relativePath: document.relativePath,
|
|
||||||
documentId: response.documentId,
|
|
||||||
parentVersionId: response.vaultUpdateId,
|
parentVersionId: response.vaultUpdateId,
|
||||||
hash: EMPTY_HASH
|
hash: EMPTY_HASH
|
||||||
},
|
},
|
||||||
document.identity
|
document
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
@ -122,21 +105,16 @@ export class UnrestrictedSyncer {
|
||||||
|
|
||||||
public async unrestrictedSyncLocallyUpdatedFile({
|
public async unrestrictedSyncLocallyUpdatedFile({
|
||||||
oldPath,
|
oldPath,
|
||||||
getLatestDocument
|
document
|
||||||
}: {
|
}: {
|
||||||
oldPath?: RelativePath;
|
oldPath?: RelativePath;
|
||||||
getLatestDocument: () => DocumentRecord;
|
document: DocumentRecord;
|
||||||
}): Promise<void> {
|
}): Promise<void> {
|
||||||
let document = getLatestDocument();
|
|
||||||
|
|
||||||
await this.executeSync(
|
await this.executeSync(
|
||||||
[oldPath, document.relativePath].filter(
|
document.relativePath,
|
||||||
(path) => path !== undefined
|
|
||||||
),
|
|
||||||
SyncType.UPDATE,
|
SyncType.UPDATE,
|
||||||
SyncSource.PUSH,
|
SyncSource.PUSH,
|
||||||
async () => {
|
async () => {
|
||||||
document = getLatestDocument();
|
|
||||||
const originalRelativePath = document.relativePath;
|
const originalRelativePath = document.relativePath;
|
||||||
|
|
||||||
if (document.metadata === undefined || document.isDeleted) {
|
if (document.metadata === undefined || document.isDeleted) {
|
||||||
|
|
@ -168,8 +146,8 @@ export class UnrestrictedSyncer {
|
||||||
contentBytes
|
contentBytes
|
||||||
});
|
});
|
||||||
|
|
||||||
document = getLatestDocument();
|
// `document` is mutable and reflects the latest state in the local database
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
||||||
if (document.isDeleted) {
|
if (document.isDeleted) {
|
||||||
this.logger.info(
|
this.logger.info(
|
||||||
`Document ${document.relativePath} has been deleted before we could finish updating it`
|
`Document ${document.relativePath} has been deleted before we could finish updating it`
|
||||||
|
|
@ -177,7 +155,8 @@ export class UnrestrictedSyncer {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!document.metadata) {
|
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
||||||
|
if (document.metadata === undefined) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`Document ${document.relativePath} no longer has metadata after updating it`
|
`Document ${document.relativePath} no longer has metadata after updating it`
|
||||||
);
|
);
|
||||||
|
|
@ -213,12 +192,10 @@ export class UnrestrictedSyncer {
|
||||||
this.database.delete(document.relativePath);
|
this.database.delete(document.relativePath);
|
||||||
this.database.setDocument(
|
this.database.setDocument(
|
||||||
{
|
{
|
||||||
documentId: response.documentId,
|
|
||||||
relativePath: document.relativePath,
|
|
||||||
parentVersionId: response.vaultUpdateId,
|
parentVersionId: response.vaultUpdateId,
|
||||||
hash: EMPTY_HASH
|
hash: EMPTY_HASH
|
||||||
},
|
},
|
||||||
document.identity
|
document
|
||||||
);
|
);
|
||||||
|
|
||||||
await this.operations.delete(document.relativePath);
|
await this.operations.delete(document.relativePath);
|
||||||
|
|
@ -231,11 +208,6 @@ export class UnrestrictedSyncer {
|
||||||
let actualPath = document.relativePath;
|
let actualPath = document.relativePath;
|
||||||
|
|
||||||
if (response.relativePath != originalRelativePath) {
|
if (response.relativePath != originalRelativePath) {
|
||||||
// this.database.getNewResolvedDocumentByRelativePath(
|
|
||||||
// response.relativePath,
|
|
||||||
// promise
|
|
||||||
// );
|
|
||||||
|
|
||||||
actualPath = response.relativePath;
|
actualPath = response.relativePath;
|
||||||
await this.operations.move(
|
await this.operations.move(
|
||||||
document.relativePath,
|
document.relativePath,
|
||||||
|
|
@ -243,6 +215,14 @@ export class UnrestrictedSyncer {
|
||||||
); // this can throw FileNotFoundError
|
); // this can throw FileNotFoundError
|
||||||
}
|
}
|
||||||
|
|
||||||
|
this.database.setDocument(
|
||||||
|
{
|
||||||
|
parentVersionId: response.vaultUpdateId,
|
||||||
|
hash: contentHash
|
||||||
|
},
|
||||||
|
document
|
||||||
|
);
|
||||||
|
|
||||||
if (response.type === "MergingUpdate") {
|
if (response.type === "MergingUpdate") {
|
||||||
const responseBytes = deserialize(response.contentBase64);
|
const responseBytes = deserialize(response.contentBase64);
|
||||||
contentHash = hash(responseBytes);
|
contentHash = hash(responseBytes);
|
||||||
|
|
@ -262,16 +242,6 @@ export class UnrestrictedSyncer {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
this.database.setDocument(
|
|
||||||
{
|
|
||||||
documentId: response.documentId,
|
|
||||||
relativePath: actualPath,
|
|
||||||
parentVersionId: response.vaultUpdateId,
|
|
||||||
hash: contentHash
|
|
||||||
},
|
|
||||||
document.identity
|
|
||||||
);
|
|
||||||
|
|
||||||
this.tryIncrementVaultUpdateId(response.vaultUpdateId);
|
this.tryIncrementVaultUpdateId(response.vaultUpdateId);
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
@ -279,20 +249,18 @@ export class UnrestrictedSyncer {
|
||||||
|
|
||||||
public async unrestrictedSyncRemotelyUpdatedFile(
|
public async unrestrictedSyncRemotelyUpdatedFile(
|
||||||
remoteVersion: components["schemas"]["DocumentVersionWithoutContent"],
|
remoteVersion: components["schemas"]["DocumentVersionWithoutContent"],
|
||||||
getLatestDocument: () => DocumentRecord | undefined
|
document?: DocumentRecord
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
await this.executeSync(
|
await this.executeSync(
|
||||||
[remoteVersion.relativePath],
|
remoteVersion.relativePath,
|
||||||
SyncType.UPDATE,
|
SyncType.UPDATE,
|
||||||
SyncSource.PULL,
|
SyncSource.PULL,
|
||||||
async () => {
|
async () => {
|
||||||
let localMetadata = getLatestDocument();
|
if (document?.metadata !== undefined) {
|
||||||
|
|
||||||
if (localMetadata?.metadata !== undefined) {
|
|
||||||
// If the file exists locally, let's pretend the user has updated it
|
// If the file exists locally, let's pretend the user has updated it
|
||||||
// and deal with remote update/deletion within `unrestrictedSyncLocallyUpdatedFile`
|
// and deal with remote update/deletion within `unrestrictedSyncLocallyUpdatedFile`
|
||||||
if (
|
if (
|
||||||
localMetadata.metadata.parentVersionId >=
|
document.metadata.parentVersionId >=
|
||||||
remoteVersion.vaultUpdateId
|
remoteVersion.vaultUpdateId
|
||||||
) {
|
) {
|
||||||
this.logger.debug(
|
this.logger.debug(
|
||||||
|
|
@ -302,11 +270,7 @@ export class UnrestrictedSyncer {
|
||||||
}
|
}
|
||||||
|
|
||||||
return this.unrestrictedSyncLocallyUpdatedFile({
|
return this.unrestrictedSyncLocallyUpdatedFile({
|
||||||
getLatestDocument: () =>
|
document
|
||||||
this.database.getDocumentByIdentity(
|
|
||||||
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
||||||
localMetadata!.identity
|
|
||||||
)
|
|
||||||
});
|
});
|
||||||
} else if (remoteVersion.isDeleted) {
|
} else if (remoteVersion.isDeleted) {
|
||||||
// Either the doc hasn't made it to us before and therefore we don't need to delete it,
|
// Either the doc hasn't made it to us before and therefore we don't need to delete it,
|
||||||
|
|
@ -323,9 +287,11 @@ export class UnrestrictedSyncer {
|
||||||
})
|
})
|
||||||
).contentBase64;
|
).contentBase64;
|
||||||
|
|
||||||
localMetadata = getLatestDocument();
|
document = this.database.getDocumentByDocumentId(
|
||||||
|
remoteVersion.documentId
|
||||||
|
);
|
||||||
|
|
||||||
if (localMetadata?.isDeleted === true) {
|
if (document?.isDeleted === true) {
|
||||||
this.logger.info(
|
this.logger.info(
|
||||||
`Document ${remoteVersion.relativePath} has been deleted locally before we could finish updating it`
|
`Document ${remoteVersion.relativePath} has been deleted locally before we could finish updating it`
|
||||||
);
|
);
|
||||||
|
|
@ -333,7 +299,7 @@ export class UnrestrictedSyncer {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (
|
if (
|
||||||
(localMetadata?.metadata?.parentVersionId ?? -1) >=
|
(document?.metadata?.parentVersionId ?? -1) >=
|
||||||
remoteVersion.vaultUpdateId
|
remoteVersion.vaultUpdateId
|
||||||
) {
|
) {
|
||||||
this.logger.debug(
|
this.logger.debug(
|
||||||
|
|
@ -344,16 +310,21 @@ export class UnrestrictedSyncer {
|
||||||
|
|
||||||
const contentBytes = deserialize(content);
|
const contentBytes = deserialize(content);
|
||||||
|
|
||||||
const [promise, resolve] = createPromise();
|
|
||||||
|
|
||||||
await this.operations.ensureClearPath(
|
await this.operations.ensureClearPath(
|
||||||
remoteVersion.relativePath
|
remoteVersion.relativePath
|
||||||
);
|
);
|
||||||
|
|
||||||
this.database.getNewResolvedDocumentByRelativePath(
|
const [promise, resolve] = createPromise();
|
||||||
|
this.database.setDocument(
|
||||||
|
{
|
||||||
|
parentVersionId: remoteVersion.vaultUpdateId,
|
||||||
|
hash: hash(contentBytes)
|
||||||
|
},
|
||||||
|
this.database.createNewPendingDocument(
|
||||||
remoteVersion.documentId,
|
remoteVersion.documentId,
|
||||||
remoteVersion.relativePath,
|
remoteVersion.relativePath,
|
||||||
promise
|
promise
|
||||||
|
)
|
||||||
);
|
);
|
||||||
|
|
||||||
await this.operations.create(
|
await this.operations.create(
|
||||||
|
|
@ -361,17 +332,6 @@ export class UnrestrictedSyncer {
|
||||||
contentBytes
|
contentBytes
|
||||||
);
|
);
|
||||||
|
|
||||||
const document =
|
|
||||||
this.database.getDocumentByUpdatePromise(promise);
|
|
||||||
this.database.setDocument(
|
|
||||||
{
|
|
||||||
documentId: remoteVersion.documentId,
|
|
||||||
relativePath: remoteVersion.relativePath,
|
|
||||||
parentVersionId: remoteVersion.vaultUpdateId,
|
|
||||||
hash: hash(contentBytes)
|
|
||||||
},
|
|
||||||
document.identity
|
|
||||||
);
|
|
||||||
resolve();
|
resolve();
|
||||||
this.database.removeDocumentPromise(promise);
|
this.database.removeDocumentPromise(promise);
|
||||||
|
|
||||||
|
|
@ -387,13 +347,11 @@ export class UnrestrictedSyncer {
|
||||||
}
|
}
|
||||||
|
|
||||||
public async executeSync<T>(
|
public async executeSync<T>(
|
||||||
paths: RelativePath[],
|
relativePath: RelativePath,
|
||||||
syncType: SyncType,
|
syncType: SyncType,
|
||||||
syncSource: SyncSource,
|
syncSource: SyncSource,
|
||||||
fn: () => Promise<T>
|
fn: () => Promise<T>
|
||||||
): Promise<T | undefined> {
|
): Promise<T | undefined> {
|
||||||
const relativePath = paths[paths.length - 1];
|
|
||||||
|
|
||||||
if (!this.operations.isFileEligibleForSync(relativePath)) {
|
if (!this.operations.isFileEligibleForSync(relativePath)) {
|
||||||
this.history.addHistoryEntry({
|
this.history.addHistoryEntry({
|
||||||
status: SyncStatus.ERROR,
|
status: SyncStatus.ERROR,
|
||||||
|
|
|
||||||
|
|
@ -63,7 +63,11 @@ export class MockClient implements FileSystemOperations {
|
||||||
`Creating file ${path} with content ${new TextDecoder().decode(newContent)}`
|
`Creating file ${path} with content ${new TextDecoder().decode(newContent)}`
|
||||||
);
|
);
|
||||||
this.localFiles.set(path, newContent);
|
this.localFiles.set(path, newContent);
|
||||||
|
|
||||||
|
// we aren't the best client and it takes some time to notice changes
|
||||||
|
setImmediate(() => {
|
||||||
void this.client.syncer.syncLocallyCreatedFile(path);
|
void this.client.syncer.syncLocallyCreatedFile(path);
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
public async createDirectory(_path: RelativePath): Promise<void> {
|
public async createDirectory(_path: RelativePath): Promise<void> {
|
||||||
|
|
@ -101,9 +105,12 @@ export class MockClient implements FileSystemOperations {
|
||||||
`Updated file ${path} with:\n current content: ${currentContent}\n new content: ${newContent}`
|
`Updated file ${path} with:\n current content: ${currentContent}\n new content: ${newContent}`
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// we aren't the best client and it takes some time to notice changes
|
||||||
|
setImmediate(() => {
|
||||||
void this.client.syncer.syncLocallyUpdatedFile({
|
void this.client.syncer.syncLocallyUpdatedFile({
|
||||||
relativePath: path
|
relativePath: path
|
||||||
});
|
});
|
||||||
|
});
|
||||||
|
|
||||||
return newContent;
|
return newContent;
|
||||||
}
|
}
|
||||||
|
|
@ -116,6 +123,8 @@ export class MockClient implements FileSystemOperations {
|
||||||
`Updated file ${path} with:\n new content: ${new TextDecoder().decode(content)}`
|
`Updated file ${path} with:\n new content: ${new TextDecoder().decode(content)}`
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// we aren't the best client and it takes some time to notice changes
|
||||||
|
setImmediate(() => {
|
||||||
if (hasExisted) {
|
if (hasExisted) {
|
||||||
void this.client.syncer.syncLocallyUpdatedFile({
|
void this.client.syncer.syncLocallyUpdatedFile({
|
||||||
relativePath: path
|
relativePath: path
|
||||||
|
|
@ -123,6 +132,7 @@ export class MockClient implements FileSystemOperations {
|
||||||
} else {
|
} else {
|
||||||
void this.client.syncer.syncLocallyCreatedFile(path);
|
void this.client.syncer.syncLocallyCreatedFile(path);
|
||||||
}
|
}
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
public async delete(path: RelativePath): Promise<void> {
|
public async delete(path: RelativePath): Promise<void> {
|
||||||
|
|
@ -130,7 +140,10 @@ export class MockClient implements FileSystemOperations {
|
||||||
`Deleting file: ${path} with:\n content ${new TextDecoder().decode(this.localFiles.get(path))}`
|
`Deleting file: ${path} with:\n content ${new TextDecoder().decode(this.localFiles.get(path))}`
|
||||||
);
|
);
|
||||||
this.localFiles.delete(path);
|
this.localFiles.delete(path);
|
||||||
|
// we aren't the best client and it takes some time to notice changes
|
||||||
|
setImmediate(() => {
|
||||||
void this.client.syncer.syncLocallyDeletedFile(path);
|
void this.client.syncer.syncLocallyDeletedFile(path);
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
public async rename(
|
public async rename(
|
||||||
|
|
@ -150,9 +163,12 @@ export class MockClient implements FileSystemOperations {
|
||||||
`Renamed file: ${oldPath} -> ${newPath} with:\n content ${new TextDecoder().decode(file)}`
|
`Renamed file: ${oldPath} -> ${newPath} with:\n content ${new TextDecoder().decode(file)}`
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// we aren't the best client and it takes some time to notice changes
|
||||||
|
setImmediate(() => {
|
||||||
void this.client.syncer.syncLocallyUpdatedFile({
|
void this.client.syncer.syncLocallyUpdatedFile({
|
||||||
oldPath,
|
oldPath,
|
||||||
relativePath: newPath
|
relativePath: newPath
|
||||||
});
|
});
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -91,21 +91,21 @@ async function runTest({
|
||||||
|
|
||||||
async function runTests(): Promise<void> {
|
async function runTests(): Promise<void> {
|
||||||
const agentCounts = [2, 8];
|
const agentCounts = [2, 8];
|
||||||
const jitterScaleInSeconds = [0.5, 0, 2];
|
const networkJitterScaleInSeconds = [0.5, 2];
|
||||||
const concurrencies = [16, 1];
|
const concurrencies = [
|
||||||
const iterations = [200];
|
16,
|
||||||
|
1 // test with concurrency 1 to check for deadlocks
|
||||||
|
];
|
||||||
const doDeletes = [true, false];
|
const doDeletes = [true, false];
|
||||||
|
|
||||||
for (const agentCount of agentCounts) {
|
for (const agentCount of agentCounts) {
|
||||||
for (const concurrency of concurrencies) {
|
for (const concurrency of concurrencies) {
|
||||||
for (const jitter of jitterScaleInSeconds) {
|
for (const jitter of networkJitterScaleInSeconds) {
|
||||||
for (const iteration of iterations) {
|
|
||||||
for (const deleteFiles of doDeletes) {
|
for (const deleteFiles of doDeletes) {
|
||||||
for (let i = 0; i < 3; i++) {
|
|
||||||
await runTest({
|
await runTest({
|
||||||
agentCount,
|
agentCount,
|
||||||
concurrency,
|
concurrency,
|
||||||
iterations: iteration,
|
iterations: 200,
|
||||||
doDeletes: deleteFiles,
|
doDeletes: deleteFiles,
|
||||||
jitterScaleInSeconds: jitter
|
jitterScaleInSeconds: jitter
|
||||||
});
|
});
|
||||||
|
|
@ -113,8 +113,6 @@ async function runTests(): Promise<void> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
process.on("uncaughtException", (error) => {
|
process.on("uncaughtException", (error) => {
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue