This commit is contained in:
Andras Schmelczer 2025-03-15 17:15:44 +00:00
commit d5112a7d0f
No known key found for this signature in database
GPG key ID: FC8F2C3D3D1A718C
7 changed files with 147 additions and 272 deletions

View file

@ -17,10 +17,6 @@
- Install [`rustup`](https://rustup.rs): `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh` - Install [`rustup`](https://rustup.rs): `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh`
- `sudo apt install llvm -y` - `sudo apt install llvm -y`
- `rustup self update`
- `rustup update`
- `rustup install nightly`
- `rustup default nightly`
- `rustup component add llvm-tools-preview` - `rustup component add llvm-tools-preview`
- `cargo install cargo-generate cargo-fuzz cargo-insta rustfilt cargo-binutils` - `cargo install cargo-generate cargo-fuzz cargo-insta rustfilt cargo-binutils`
- Install [`wasm-pack`](https://rustwasm.github.io/wasm-pack/installer): `curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh` - Install [`wasm-pack`](https://rustwasm.github.io/wasm-pack/installer): `curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh`

View file

@ -2,10 +2,7 @@ import type { Logger } from "../tracing/logger";
import type { FileSystemOperations } from "./filesystem-operations"; import type { FileSystemOperations } from "./filesystem-operations";
import type { Database, RelativePath } from "../persistence/database"; import type { Database, RelativePath } from "../persistence/database";
import { isBinary, isFileTypeMergable, mergeText } from "sync_lib"; import { isBinary, isFileTypeMergable, mergeText } from "sync_lib";
import { import { SafeFileSystemOperations } from "./safe-filesystem-operations";
FileNotFoundError,
SafeFileSystemOperations
} from "./safe-filesystem-operations";
export class FileOperations { export class FileOperations {
private static readonly PARENTHESES_REGEX = / \((\d+)\)$/; private static readonly PARENTHESES_REGEX = / \((\d+)\)$/;
@ -67,7 +64,7 @@ export class FileOperations {
`Didn't expect ${path} to exist, deconflicting by moving it to '${deconflictedPath}'` `Didn't expect ${path} to exist, deconflicting by moving it to '${deconflictedPath}'`
); );
// this.database.move(path, deconflictedPath); this.database.move(path, deconflictedPath);
await this.fs.rename(path, deconflictedPath); await this.fs.rename(path, deconflictedPath);
} else { } else {
await this.createParentDirectories(path); await this.createParentDirectories(path);
@ -142,9 +139,9 @@ export class FileOperations {
if (oldPath === newPath) { if (oldPath === newPath) {
return; return;
} }
await this.ensureClearPath(newPath); await this.ensureClearPath(newPath);
this.database.move(oldPath, newPath);
await this.fs.rename(oldPath, newPath); await this.fs.rename(oldPath, newPath);
} }

View file

@ -22,7 +22,6 @@ export interface StoredDatabase {
} }
export interface DocumentRecord { export interface DocumentRecord {
identity: symbol;
relativePath: RelativePath; relativePath: RelativePath;
documentId: DocumentId; documentId: DocumentId;
metadata: DocumentMetadata | undefined; metadata: DocumentMetadata | undefined;
@ -47,7 +46,6 @@ export class Database {
({ relativePath, documentId, ...metadata }) => ({ ({ relativePath, documentId, ...metadata }) => ({
relativePath, relativePath,
documentId, documentId,
identity: Symbol(),
metadata, metadata,
isDeleted: false, isDeleted: false,
updates: [], updates: [],
@ -118,85 +116,33 @@ export class Database {
public setDocument( public setDocument(
{ {
documentId,
relativePath,
parentVersionId, parentVersionId,
hash hash
}: { }: {
documentId: DocumentId;
relativePath: RelativePath;
parentVersionId: VaultUpdateId; parentVersionId: VaultUpdateId;
hash: string; hash: string;
}, },
identity?: symbol toUpdate: DocumentRecord
): void { ): void {
if (identity !== undefined) { if (!this.documents.includes(toUpdate)) {
const entry = this.getDocumentByIdentity(identity); throw new Error("Document not found in database");
this.documents = this.documents.filter(
(doc) => doc.identity !== entry.identity
);
if (entry.relativePath !== relativePath) {
throw new Error(
"Document identity does not match the relative path"
);
} }
this.documents.push({ toUpdate.metadata = { parentVersionId, hash };
...entry,
relativePath,
documentId,
metadata: {
parentVersionId,
hash
}
});
this.save(); this.save();
return; return;
} }
// We find a match based on relative path and we find one with a different document id
// meaning that two documents occupy the same path in terms of in-flight requests so we
// need to create a new parallel version.
const entry = this.getLatestDocumentByRelativePath(relativePath);
if (entry && entry.documentId !== documentId) {
this.documents.push({
// `entry` might be undefined if the document is new
identity: Symbol(),
relativePath,
documentId,
metadata: {
parentVersionId,
hash
},
isDeleted: false,
updates: [],
parallelVersion: entry.parallelVersion + 1
});
this.save();
return;
}
this.documents.push({
identity: Symbol(),
relativePath,
documentId,
metadata: {
parentVersionId,
hash
},
isDeleted: false,
updates: [],
parallelVersion: 0
});
this.save();
}
public removeDocumentPromise(promise: Promise<void>): void { public removeDocumentPromise(promise: Promise<void>): void {
const entry = this.getDocumentByUpdatePromise(promise); const entry = this.documents.find(({ updates }) =>
updates.includes(promise)
);
if (entry === undefined) {
throw new Error("Document not found by update promise");
}
entry.updates = entry.updates.filter((update) => update !== promise); entry.updates = entry.updates.filter((update) => update !== promise);
// No need to save as Promises don't get serialized // No need to save as Promises don't get serialized
} }
@ -214,7 +160,7 @@ export class Database {
public async getResolvedDocumentByRelativePath( public async getResolvedDocumentByRelativePath(
relativePath: RelativePath, relativePath: RelativePath,
promise: Promise<void> promise: Promise<void>
): Promise<void> { ): Promise<DocumentRecord> {
const entry = this.getLatestDocumentByRelativePath(relativePath); const entry = this.getLatestDocumentByRelativePath(relativePath);
if (entry === undefined) { if (entry === undefined) {
@ -230,20 +176,21 @@ export class Database {
const currentPromises = entry.updates; const currentPromises = entry.updates;
entry.updates = [...currentPromises, promise]; entry.updates = [...currentPromises, promise];
await Promise.all(currentPromises); await Promise.all(currentPromises);
return entry;
} }
public getNewResolvedDocumentByRelativePath( public createNewPendingDocument(
documentId: DocumentId, documentId: DocumentId,
relativePath: RelativePath, relativePath: RelativePath,
promise: Promise<void> promise: Promise<void>
): void { ): DocumentRecord {
const previousEntry = const previousEntry =
this.getLatestDocumentByRelativePath(relativePath); this.getLatestDocumentByRelativePath(relativePath);
const entry = { const entry = {
relativePath, relativePath,
documentId, documentId,
identity: Symbol(),
metadata: undefined, metadata: undefined,
isDeleted: false, isDeleted: false,
updates: [promise], updates: [promise],
@ -255,18 +202,8 @@ export class Database {
this.documents.push(entry); this.documents.push(entry);
this.save(); this.save();
}
public getDocumentByUpdatePromise(promise: Promise<void>): DocumentRecord { return entry;
const result = this.documents.find(({ updates }) =>
updates.includes(promise)
);
if (result === undefined) {
throw new Error("Document not found by update promise");
}
return result;
} }
public getDocumentByDocumentId( public getDocumentByDocumentId(
@ -275,16 +212,6 @@ export class Database {
return this.documents.find(({ documentId }) => documentId === find); return this.documents.find(({ documentId }) => documentId === find);
} }
public getDocumentByIdentity(find: symbol): DocumentRecord {
const result = this.documents.find(({ identity }) => identity === find);
if (result === undefined) {
throw new Error("Document not found by identity symbol");
}
return result;
}
public move( public move(
oldRelativePath: RelativePath, oldRelativePath: RelativePath,
newRelativePath: RelativePath newRelativePath: RelativePath
@ -296,10 +223,6 @@ export class Database {
return; return;
} }
this.documents = this.documents.filter(
({ identity }) => identity !== oldDocument.identity
);
const newDocument = const newDocument =
this.getLatestDocumentByRelativePath(newRelativePath); this.getLatestDocumentByRelativePath(newRelativePath);
if (newDocument !== undefined && !newDocument.isDeleted) { if (newDocument !== undefined && !newDocument.isDeleted) {
@ -308,17 +231,12 @@ export class Database {
); );
} }
// It's either an invalid state of newDocument is pending deletion and we have oldDocument.relativePath = newRelativePath;
// to wait for it to complete.
this.documents.push({
...oldDocument,
relativePath: newRelativePath,
// We're in a strange state where the target of the move has just got deleted, // We're in a strange state where the target of the move has just got deleted,
// however, its metadata might already have a bunch of updates queued up for // however, its metadata might already have a bunch of updates queued up for
// the document at the new location. We need to keep these updates. // the document at the new location. We need to keep these updates.
parallelVersion: oldDocument.parallelVersion =
newDocument !== undefined ? newDocument.parallelVersion + 1 : 0 newDocument !== undefined ? newDocument.parallelVersion + 1 : 0;
});
this.save(); this.save();
} }

View file

@ -3,8 +3,8 @@ import type { SyncService } from "../services/sync-service";
import type { Logger } from "../tracing/logger"; import type { Logger } from "../tracing/logger";
import type { SyncHistory } from "../tracing/sync-history"; import type { SyncHistory } from "../tracing/sync-history";
import PQueue from "p-queue"; import PQueue from "p-queue";
import { v4 as uuidv4 } from "uuid";
import { hash } from "../utils/hash"; import { hash } from "../utils/hash";
import { v4 as uuidv4 } from "uuid";
import type { components } from "../services/types"; import type { components } from "../services/types";
import type { Settings } from "../persistence/settings"; import type { Settings } from "../persistence/settings";
import type { FileOperations } from "../file-operations/file-operations"; import type { FileOperations } from "../file-operations/file-operations";
@ -98,20 +98,16 @@ export class Syncer {
} }
const [promise, resolve, reject] = createPromise(); const [promise, resolve, reject] = createPromise();
const proposedDocumentId = uuidv4();
this.database.getNewResolvedDocumentByRelativePath( const document = this.database.createNewPendingDocument(
proposedDocumentId, uuidv4(),
relativePath, relativePath,
promise promise
); );
try { try {
await this.syncQueue.add(async () => await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncLocallyCreatedFile( this.internalSyncer.unrestrictedSyncLocallyCreatedFile(document)
proposedDocumentId,
() => this.database.getDocumentByUpdatePromise(promise)
)
); );
resolve(); resolve();
@ -131,16 +127,14 @@ export class Syncer {
const [promise, resolve, reject] = createPromise(); const [promise, resolve, reject] = createPromise();
await this.database.getResolvedDocumentByRelativePath( const document = await this.database.getResolvedDocumentByRelativePath(
relativePath, relativePath,
promise promise
); );
try { try {
await this.syncQueue.add(async () => await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncLocallyDeletedFile(() => this.internalSyncer.unrestrictedSyncLocallyDeletedFile(document)
this.database.getDocumentByUpdatePromise(promise)
)
); );
resolve(); resolve();
@ -158,17 +152,13 @@ export class Syncer {
oldPath?: RelativePath; oldPath?: RelativePath;
relativePath: RelativePath; relativePath: RelativePath;
}): Promise<void> { }): Promise<void> {
if (oldPath !== undefined) {
if ( if (
this.database.getLatestDocumentByRelativePath(oldPath) oldPath !== undefined &&
?.isDeleted === true (this.database.getLatestDocumentByRelativePath(relativePath) ===
undefined ||
this.database.getLatestDocumentByRelativePath(relativePath)
?.isDeleted === true)
) { ) {
this.logger.debug(
`Document ${oldPath} has been deleted locally, skipping`
);
return;
}
if (oldPath === relativePath) { if (oldPath === relativePath) {
throw new Error( throw new Error(
`Old path and new path are the same: ${oldPath}` `Old path and new path are the same: ${oldPath}`
@ -178,10 +168,17 @@ export class Syncer {
this.database.move(oldPath, relativePath); this.database.move(oldPath, relativePath);
} }
if ( let document =
this.database.getLatestDocumentByRelativePath(relativePath) this.database.getLatestDocumentByRelativePath(relativePath);
?.isDeleted === true
) { if (document === undefined) {
this.logger.debug(
`Cannot find document ${relativePath} in the database, skipping`
);
return;
}
if (document.isDeleted) {
this.logger.debug( this.logger.debug(
`Document ${relativePath} has been deleted locally, skipping` `Document ${relativePath} has been deleted locally, skipping`
); );
@ -190,7 +187,7 @@ export class Syncer {
const [promise, resolve, reject] = createPromise(); const [promise, resolve, reject] = createPromise();
await this.database.getResolvedDocumentByRelativePath( document = await this.database.getResolvedDocumentByRelativePath(
relativePath, relativePath,
promise promise
); );
@ -199,8 +196,7 @@ export class Syncer {
await this.syncQueue.add(async () => await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncLocallyUpdatedFile({ this.internalSyncer.unrestrictedSyncLocallyUpdatedFile({
oldPath, oldPath,
getLatestDocument: () => document
this.database.getDocumentByUpdatePromise(promise)
}) })
); );
@ -299,7 +295,7 @@ export class Syncer {
private async syncRemotelyUpdatedFile( private async syncRemotelyUpdatedFile(
remoteVersion: components["schemas"]["DocumentVersionWithoutContent"] remoteVersion: components["schemas"]["DocumentVersionWithoutContent"]
): Promise<void> { ): Promise<void> {
const document = this.database.getDocumentByDocumentId( let document = this.database.getDocumentByDocumentId(
remoteVersion.documentId remoteVersion.documentId
); );
@ -308,15 +304,11 @@ export class Syncer {
if (document === undefined) { if (document === undefined) {
await this.syncQueue.add(async () => await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile( this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
remoteVersion, remoteVersion
() =>
this.database.getDocumentByDocumentId(
remoteVersion.documentId
)
) )
); );
} else { } else {
await this.database.getResolvedDocumentByRelativePath( document = await this.database.getResolvedDocumentByRelativePath(
document.relativePath, document.relativePath,
promise promise
); );
@ -325,7 +317,7 @@ export class Syncer {
await this.syncQueue.add(async () => await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile( this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
remoteVersion, remoteVersion,
() => this.database.getDocumentByUpdatePromise(promise) document
) )
); );

View file

@ -1,6 +1,5 @@
import type { import type {
Database, Database,
DocumentId,
DocumentRecord, DocumentRecord,
RelativePath RelativePath
} from "../persistence/database"; } from "../persistence/database";
@ -33,31 +32,24 @@ export class UnrestrictedSyncer {
} }
public async unrestrictedSyncLocallyCreatedFile( public async unrestrictedSyncLocallyCreatedFile(
proposedDocumentId: DocumentId, document: DocumentRecord
getLatestDocument: () => DocumentRecord
): Promise<void> { ): Promise<void> {
let document = getLatestDocument();
return this.executeSync( return this.executeSync(
[document.relativePath], document.relativePath,
SyncType.CREATE, SyncType.CREATE,
SyncSource.PUSH, SyncSource.PUSH,
async () => { async () => {
document = getLatestDocument();
const contentBytes = await this.operations.read( const contentBytes = await this.operations.read(
document.relativePath document.relativePath
); // this can throw FileNotFoundError ); // this can throw FileNotFoundError
const contentHash = hash(contentBytes); const contentHash = hash(contentBytes);
const response = await this.syncService.create({ const response = await this.syncService.create({
documentId: proposedDocumentId, documentId: document.documentId,
relativePath: document.relativePath, relativePath: document.relativePath,
contentBytes contentBytes
}); });
document = getLatestDocument();
this.history.addHistoryEntry({ this.history.addHistoryEntry({
status: SyncStatus.SUCCESS, status: SyncStatus.SUCCESS,
source: SyncSource.PUSH, source: SyncSource.PUSH,
@ -68,12 +60,10 @@ export class UnrestrictedSyncer {
this.database.setDocument( this.database.setDocument(
{ {
relativePath: document.relativePath,
documentId: response.documentId,
parentVersionId: response.vaultUpdateId, parentVersionId: response.vaultUpdateId,
hash: contentHash hash: contentHash
}, },
document.identity document
); );
this.tryIncrementVaultUpdateId(response.vaultUpdateId); this.tryIncrementVaultUpdateId(response.vaultUpdateId);
@ -82,16 +72,13 @@ export class UnrestrictedSyncer {
} }
public async unrestrictedSyncLocallyDeletedFile( public async unrestrictedSyncLocallyDeletedFile(
getLatestDocument: () => DocumentRecord document: DocumentRecord
): Promise<void> { ): Promise<void> {
let document = getLatestDocument();
await this.executeSync( await this.executeSync(
[document.relativePath], document.relativePath,
SyncType.DELETE, SyncType.DELETE,
SyncSource.PUSH, SyncSource.PUSH,
async () => { async () => {
document = getLatestDocument();
const response = await this.syncService.delete({ const response = await this.syncService.delete({
documentId: document.documentId, documentId: document.documentId,
relativePath: document.relativePath relativePath: document.relativePath
@ -105,16 +92,12 @@ export class UnrestrictedSyncer {
type: SyncType.DELETE type: SyncType.DELETE
}); });
document = getLatestDocument();
this.database.setDocument( this.database.setDocument(
{ {
relativePath: document.relativePath,
documentId: response.documentId,
parentVersionId: response.vaultUpdateId, parentVersionId: response.vaultUpdateId,
hash: EMPTY_HASH hash: EMPTY_HASH
}, },
document.identity document
); );
} }
); );
@ -122,21 +105,16 @@ export class UnrestrictedSyncer {
public async unrestrictedSyncLocallyUpdatedFile({ public async unrestrictedSyncLocallyUpdatedFile({
oldPath, oldPath,
getLatestDocument document
}: { }: {
oldPath?: RelativePath; oldPath?: RelativePath;
getLatestDocument: () => DocumentRecord; document: DocumentRecord;
}): Promise<void> { }): Promise<void> {
let document = getLatestDocument();
await this.executeSync( await this.executeSync(
[oldPath, document.relativePath].filter( document.relativePath,
(path) => path !== undefined
),
SyncType.UPDATE, SyncType.UPDATE,
SyncSource.PUSH, SyncSource.PUSH,
async () => { async () => {
document = getLatestDocument();
const originalRelativePath = document.relativePath; const originalRelativePath = document.relativePath;
if (document.metadata === undefined || document.isDeleted) { if (document.metadata === undefined || document.isDeleted) {
@ -168,8 +146,8 @@ export class UnrestrictedSyncer {
contentBytes contentBytes
}); });
document = getLatestDocument(); // `document` is mutable and reflects the latest state in the local database
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
if (document.isDeleted) { if (document.isDeleted) {
this.logger.info( this.logger.info(
`Document ${document.relativePath} has been deleted before we could finish updating it` `Document ${document.relativePath} has been deleted before we could finish updating it`
@ -177,7 +155,8 @@ export class UnrestrictedSyncer {
return; return;
} }
if (!document.metadata) { // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
if (document.metadata === undefined) {
throw new Error( throw new Error(
`Document ${document.relativePath} no longer has metadata after updating it` `Document ${document.relativePath} no longer has metadata after updating it`
); );
@ -213,12 +192,10 @@ export class UnrestrictedSyncer {
this.database.delete(document.relativePath); this.database.delete(document.relativePath);
this.database.setDocument( this.database.setDocument(
{ {
documentId: response.documentId,
relativePath: document.relativePath,
parentVersionId: response.vaultUpdateId, parentVersionId: response.vaultUpdateId,
hash: EMPTY_HASH hash: EMPTY_HASH
}, },
document.identity document
); );
await this.operations.delete(document.relativePath); await this.operations.delete(document.relativePath);
@ -231,11 +208,6 @@ export class UnrestrictedSyncer {
let actualPath = document.relativePath; let actualPath = document.relativePath;
if (response.relativePath != originalRelativePath) { if (response.relativePath != originalRelativePath) {
// this.database.getNewResolvedDocumentByRelativePath(
// response.relativePath,
// promise
// );
actualPath = response.relativePath; actualPath = response.relativePath;
await this.operations.move( await this.operations.move(
document.relativePath, document.relativePath,
@ -243,6 +215,14 @@ export class UnrestrictedSyncer {
); // this can throw FileNotFoundError ); // this can throw FileNotFoundError
} }
this.database.setDocument(
{
parentVersionId: response.vaultUpdateId,
hash: contentHash
},
document
);
if (response.type === "MergingUpdate") { if (response.type === "MergingUpdate") {
const responseBytes = deserialize(response.contentBase64); const responseBytes = deserialize(response.contentBase64);
contentHash = hash(responseBytes); contentHash = hash(responseBytes);
@ -262,16 +242,6 @@ export class UnrestrictedSyncer {
}); });
} }
this.database.setDocument(
{
documentId: response.documentId,
relativePath: actualPath,
parentVersionId: response.vaultUpdateId,
hash: contentHash
},
document.identity
);
this.tryIncrementVaultUpdateId(response.vaultUpdateId); this.tryIncrementVaultUpdateId(response.vaultUpdateId);
} }
); );
@ -279,20 +249,18 @@ export class UnrestrictedSyncer {
public async unrestrictedSyncRemotelyUpdatedFile( public async unrestrictedSyncRemotelyUpdatedFile(
remoteVersion: components["schemas"]["DocumentVersionWithoutContent"], remoteVersion: components["schemas"]["DocumentVersionWithoutContent"],
getLatestDocument: () => DocumentRecord | undefined document?: DocumentRecord
): Promise<void> { ): Promise<void> {
await this.executeSync( await this.executeSync(
[remoteVersion.relativePath], remoteVersion.relativePath,
SyncType.UPDATE, SyncType.UPDATE,
SyncSource.PULL, SyncSource.PULL,
async () => { async () => {
let localMetadata = getLatestDocument(); if (document?.metadata !== undefined) {
if (localMetadata?.metadata !== undefined) {
// If the file exists locally, let's pretend the user has updated it // If the file exists locally, let's pretend the user has updated it
// and deal with remote update/deletion within `unrestrictedSyncLocallyUpdatedFile` // and deal with remote update/deletion within `unrestrictedSyncLocallyUpdatedFile`
if ( if (
localMetadata.metadata.parentVersionId >= document.metadata.parentVersionId >=
remoteVersion.vaultUpdateId remoteVersion.vaultUpdateId
) { ) {
this.logger.debug( this.logger.debug(
@ -302,11 +270,7 @@ export class UnrestrictedSyncer {
} }
return this.unrestrictedSyncLocallyUpdatedFile({ return this.unrestrictedSyncLocallyUpdatedFile({
getLatestDocument: () => document
this.database.getDocumentByIdentity(
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
localMetadata!.identity
)
}); });
} else if (remoteVersion.isDeleted) { } else if (remoteVersion.isDeleted) {
// Either the doc hasn't made it to us before and therefore we don't need to delete it, // Either the doc hasn't made it to us before and therefore we don't need to delete it,
@ -323,9 +287,11 @@ export class UnrestrictedSyncer {
}) })
).contentBase64; ).contentBase64;
localMetadata = getLatestDocument(); document = this.database.getDocumentByDocumentId(
remoteVersion.documentId
);
if (localMetadata?.isDeleted === true) { if (document?.isDeleted === true) {
this.logger.info( this.logger.info(
`Document ${remoteVersion.relativePath} has been deleted locally before we could finish updating it` `Document ${remoteVersion.relativePath} has been deleted locally before we could finish updating it`
); );
@ -333,7 +299,7 @@ export class UnrestrictedSyncer {
} }
if ( if (
(localMetadata?.metadata?.parentVersionId ?? -1) >= (document?.metadata?.parentVersionId ?? -1) >=
remoteVersion.vaultUpdateId remoteVersion.vaultUpdateId
) { ) {
this.logger.debug( this.logger.debug(
@ -344,16 +310,21 @@ export class UnrestrictedSyncer {
const contentBytes = deserialize(content); const contentBytes = deserialize(content);
const [promise, resolve] = createPromise();
await this.operations.ensureClearPath( await this.operations.ensureClearPath(
remoteVersion.relativePath remoteVersion.relativePath
); );
this.database.getNewResolvedDocumentByRelativePath( const [promise, resolve] = createPromise();
this.database.setDocument(
{
parentVersionId: remoteVersion.vaultUpdateId,
hash: hash(contentBytes)
},
this.database.createNewPendingDocument(
remoteVersion.documentId, remoteVersion.documentId,
remoteVersion.relativePath, remoteVersion.relativePath,
promise promise
)
); );
await this.operations.create( await this.operations.create(
@ -361,17 +332,6 @@ export class UnrestrictedSyncer {
contentBytes contentBytes
); );
const document =
this.database.getDocumentByUpdatePromise(promise);
this.database.setDocument(
{
documentId: remoteVersion.documentId,
relativePath: remoteVersion.relativePath,
parentVersionId: remoteVersion.vaultUpdateId,
hash: hash(contentBytes)
},
document.identity
);
resolve(); resolve();
this.database.removeDocumentPromise(promise); this.database.removeDocumentPromise(promise);
@ -387,13 +347,11 @@ export class UnrestrictedSyncer {
} }
public async executeSync<T>( public async executeSync<T>(
paths: RelativePath[], relativePath: RelativePath,
syncType: SyncType, syncType: SyncType,
syncSource: SyncSource, syncSource: SyncSource,
fn: () => Promise<T> fn: () => Promise<T>
): Promise<T | undefined> { ): Promise<T | undefined> {
const relativePath = paths[paths.length - 1];
if (!this.operations.isFileEligibleForSync(relativePath)) { if (!this.operations.isFileEligibleForSync(relativePath)) {
this.history.addHistoryEntry({ this.history.addHistoryEntry({
status: SyncStatus.ERROR, status: SyncStatus.ERROR,

View file

@ -63,7 +63,11 @@ export class MockClient implements FileSystemOperations {
`Creating file ${path} with content ${new TextDecoder().decode(newContent)}` `Creating file ${path} with content ${new TextDecoder().decode(newContent)}`
); );
this.localFiles.set(path, newContent); this.localFiles.set(path, newContent);
// we aren't the best client and it takes some time to notice changes
setImmediate(() => {
void this.client.syncer.syncLocallyCreatedFile(path); void this.client.syncer.syncLocallyCreatedFile(path);
});
} }
public async createDirectory(_path: RelativePath): Promise<void> { public async createDirectory(_path: RelativePath): Promise<void> {
@ -101,9 +105,12 @@ export class MockClient implements FileSystemOperations {
`Updated file ${path} with:\n current content: ${currentContent}\n new content: ${newContent}` `Updated file ${path} with:\n current content: ${currentContent}\n new content: ${newContent}`
); );
// we aren't the best client and it takes some time to notice changes
setImmediate(() => {
void this.client.syncer.syncLocallyUpdatedFile({ void this.client.syncer.syncLocallyUpdatedFile({
relativePath: path relativePath: path
}); });
});
return newContent; return newContent;
} }
@ -116,6 +123,8 @@ export class MockClient implements FileSystemOperations {
`Updated file ${path} with:\n new content: ${new TextDecoder().decode(content)}` `Updated file ${path} with:\n new content: ${new TextDecoder().decode(content)}`
); );
// we aren't the best client and it takes some time to notice changes
setImmediate(() => {
if (hasExisted) { if (hasExisted) {
void this.client.syncer.syncLocallyUpdatedFile({ void this.client.syncer.syncLocallyUpdatedFile({
relativePath: path relativePath: path
@ -123,6 +132,7 @@ export class MockClient implements FileSystemOperations {
} else { } else {
void this.client.syncer.syncLocallyCreatedFile(path); void this.client.syncer.syncLocallyCreatedFile(path);
} }
});
} }
public async delete(path: RelativePath): Promise<void> { public async delete(path: RelativePath): Promise<void> {
@ -130,7 +140,10 @@ export class MockClient implements FileSystemOperations {
`Deleting file: ${path} with:\n content ${new TextDecoder().decode(this.localFiles.get(path))}` `Deleting file: ${path} with:\n content ${new TextDecoder().decode(this.localFiles.get(path))}`
); );
this.localFiles.delete(path); this.localFiles.delete(path);
// we aren't the best client and it takes some time to notice changes
setImmediate(() => {
void this.client.syncer.syncLocallyDeletedFile(path); void this.client.syncer.syncLocallyDeletedFile(path);
});
} }
public async rename( public async rename(
@ -150,9 +163,12 @@ export class MockClient implements FileSystemOperations {
`Renamed file: ${oldPath} -> ${newPath} with:\n content ${new TextDecoder().decode(file)}` `Renamed file: ${oldPath} -> ${newPath} with:\n content ${new TextDecoder().decode(file)}`
); );
// we aren't the best client and it takes some time to notice changes
setImmediate(() => {
void this.client.syncer.syncLocallyUpdatedFile({ void this.client.syncer.syncLocallyUpdatedFile({
oldPath, oldPath,
relativePath: newPath relativePath: newPath
}); });
});
} }
} }

View file

@ -91,21 +91,21 @@ async function runTest({
async function runTests(): Promise<void> { async function runTests(): Promise<void> {
const agentCounts = [2, 8]; const agentCounts = [2, 8];
const jitterScaleInSeconds = [0.5, 0, 2]; const networkJitterScaleInSeconds = [0.5, 2];
const concurrencies = [16, 1]; const concurrencies = [
const iterations = [200]; 16,
1 // test with concurrency 1 to check for deadlocks
];
const doDeletes = [true, false]; const doDeletes = [true, false];
for (const agentCount of agentCounts) { for (const agentCount of agentCounts) {
for (const concurrency of concurrencies) { for (const concurrency of concurrencies) {
for (const jitter of jitterScaleInSeconds) { for (const jitter of networkJitterScaleInSeconds) {
for (const iteration of iterations) {
for (const deleteFiles of doDeletes) { for (const deleteFiles of doDeletes) {
for (let i = 0; i < 3; i++) {
await runTest({ await runTest({
agentCount, agentCount,
concurrency, concurrency,
iterations: iteration, iterations: 200,
doDeletes: deleteFiles, doDeletes: deleteFiles,
jitterScaleInSeconds: jitter jitterScaleInSeconds: jitter
}); });
@ -113,8 +113,6 @@ async function runTests(): Promise<void> {
} }
} }
} }
}
}
} }
process.on("uncaughtException", (error) => { process.on("uncaughtException", (error) => {