andras/vault-link
andras/vault-link
1
0
Fork 0
Code Issues Pull requests 19 Projects Releases 55 Packages Wiki Activity Actions 24

Fix syncing when network latency is present (#4)

Browse source 
* WIP

* Add debug

* Dedupe inserts

* Add deterministic ordering

* Fix whitespaces

* Update insta

* Add integration test script

* Rename

* Add test

* Working for non-deletes

* omg it mostly works for deletes

* Isdeleted fix

* remove created dates

* update api

* Take document id

* No max attempt

* works

* Use string uuids

* .

* working!!!! (hopefully)

* Improve bundling

* Add module

* lint

* .

* lint

* Fix CI

* use toolchain

* clean up

* Add useSlowFileEvents

* Delete fuzz

* Fix CI

* use docker

* fix script

* clean up

* Clean up

* change node version

* Build docker image on every commit

* fix ci

* 1 db per vault

* Add scritps folder

* Bump versions

* Lint

* .

* Fix tests for real

* Style

* .

* try

* Consistent ordering

* Fix tests

* hmm

* .

* Clean up diff

* Fixes

* .

* Fix version bump

* .

* .

* .
This commit is contained in:
Andras Schmelczer 2025-03-16 20:13:49 +00:00 • committed by GitHub
commit 8b8f1d91d9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
91 changed files with 2216 additions and 1550 deletions
Download patch file Download diff file Expand all files Collapse all files

61
frontend/test-client/src/agent/mock-agent.ts
View file

@ -18,9 +18,10 @@ export class MockAgent extends MockClient {
initialSettings: Partial<SyncSettings>,
public readonly name: string,
private readonly doDeletes: boolean,
useSlowFileEvents: boolean,
private readonly jitterScaleInSeconds: number
) {
super(initialSettings);
super(initialSettings, useSlowFileEvents);
}
public async init(): Promise<void> {
@ -46,27 +47,33 @@ export class MockAgent extends MockClient {
? "(online) "
: "(offline)";
const formatted = `[${this.name} ${state}] ${logLine.timestamp.toISOString()} ${logLine.level} ${logLine.message}`;
// HACK: we have to ensure the file has been synced if we want to change it offline without data loss
const historyEntry = /.*History entry: (.*.md).*/.exec(
logLine.message
);
if (historyEntry) {
this.doNotTouchWhileOffline =
this.doNotTouchWhileOffline.filter(
(file) => file !== historyEntry[1]
);
}
switch (logLine.level) {
case LogLevel.ERROR:
console.error(formatted);
// Let's not ignore errors
process.exit(1);
if (!this.useSlowFileEvents) {
// Let's not ignore errors
// eslint-disable-next-line @typescript-eslint/no-floating-promises
sleep(100).then(() => process.exit(1));
}
break;
case LogLevel.WARNING:
console.warn(formatted);
break;
case LogLevel.INFO:
// HACK: we have to ensure the file has been synced if we want to change it offline without data loss
const result = /.*History entry: (.*.md).*/.exec(
logLine.message
);
if (result) {
this.doNotTouchWhileOffline =
this.doNotTouchWhileOffline.filter(
(file) => file !== result[1]
);
}
console.info(formatted);
break;
case LogLevel.DEBUG:
@ -84,11 +91,10 @@ export class MockAgent extends MockClient {
this.changeFetchChangesUpdateIntervalMsAction.bind(this)
];
if (
this.client.settings.getSettings().isSyncEnabled &&
this.doNotTouchWhileOffline.length === 0
) {
options.push(this.disableSyncAction.bind(this));
if (this.client.settings.getSettings().isSyncEnabled) {
if (this.doNotTouchWhileOffline.length === 0) {
options.push(this.disableSyncAction.bind(this));
}
} else {
options.push(this.enableSyncAction.bind(this));
}
@ -186,6 +192,14 @@ export class MockAgent extends MockClient {
}
public assertAllContentIsPresentOnce(): void {
if (this.useSlowFileEvents) {
this.client.logger.info(
// We can't ensure that we have seen every single update
`Skipping content check for ${this.name} because slow file events are enabled`
);
return;
}
for (const content of this.writtenContents) {
const found = Array.from(this.localFiles.keys()).filter((key) => {
return new TextDecoder()
@ -215,7 +229,7 @@ export class MockAgent extends MockClient {
);
assert(
fileContent.split(content).length == 2,
`Content ${content} (of ${this.name}) found more than once in file ${file}. File content:\n${fileContent}`
`Content ${content} (of ${this.name}) found more than once in '${file}'. File content:\n${fileContent}`
);
}
}
@ -237,10 +251,7 @@ export class MockAgent extends MockClient {
`Decided to create file ${file} with content ${content}`
);
return this.create(
file,
new TextEncoder().encode(` |${content}| `)
);
return this.create(file, new TextEncoder().encode(` ${content} `));
}
private async changeFetchChangesUpdateIntervalMsAction(): Promise<void> {
@ -314,7 +325,7 @@ export class MockAgent extends MockClient {
`Decided to update file ${file} with ${content}`
);
this.doNotTouchWhileOffline.push(file);
await this.atomicUpdateText(file, (old) => old + ` |${content}| `);
await this.atomicUpdateText(file, (old) => old + ` ${content} `);
}
private async deleteFileAction(files: RelativePath[]): Promise<void> {