andras/reconcile
1
0
Fork 0
Code Issues Pull requests 6 Projects Releases Packages Wiki Activity Actions

Fix syncing when network latency is present (#4)

Browse source 
* WIP

* Add debug

* Dedupe inserts

* Add deterministic ordering

* Fix whitespaces

* Update insta

* Add integration test script

* Rename

* Add test

* Working for non-deletes

* omg it mostly works for deletes

* Isdeleted fix

* remove created dates

* update api

* Take document id

* No max attempt

* works

* Use string uuids

* .

* working!!!! (hopefully)

* Improve bundling

* Add module

* lint

* .

* lint

* Fix CI

* use toolchain

* clean up

* Add useSlowFileEvents

* Delete fuzz

* Fix CI

* use docker

* fix script

* clean up

* Clean up

* change node version

* Build docker image on every commit

* fix ci

* 1 db per vault

* Add scritps folder

* Bump versions

* Lint

* .

* Fix tests for real

* Style

* .

* try

* Consistent ordering

* Fix tests

* hmm

* .

* Clean up diff

* Fixes

* .

* Fix version bump

* .

* .

* .
This commit is contained in:
Andras Schmelczer 2025-03-16 20:13:49 +00:00 • committed by GitHub
parent bcf48c428d
commit 8b8f1d91d9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
91 changed files with 2252 additions and 1586 deletions
Download patch file Download diff file Expand all files Collapse all files

61
frontend/test-client/src/agent/mock-agent.ts
View file

@ -18,9 +18,10 @@ export class MockAgent extends MockClient {
initialSettings: Partial<SyncSettings>,
public readonly name: string,
private readonly doDeletes: boolean,
useSlowFileEvents: boolean,
private readonly jitterScaleInSeconds: number
) {
super(initialSettings);
super(initialSettings, useSlowFileEvents);
}
public async init(): Promise<void> {
@ -46,27 +47,33 @@ export class MockAgent extends MockClient {
? "(online) "
: "(offline)";
const formatted = `[${this.name} ${state}] ${logLine.timestamp.toISOString()} ${logLine.level} ${logLine.message}`;
// HACK: we have to ensure the file has been synced if we want to change it offline without data loss
const historyEntry = /.*History entry: (.*.md).*/.exec(
logLine.message
);
if (historyEntry) {
this.doNotTouchWhileOffline =
this.doNotTouchWhileOffline.filter(
(file) => file !== historyEntry[1]
);
}
switch (logLine.level) {
case LogLevel.ERROR:
console.error(formatted);
// Let's not ignore errors
process.exit(1);
if (!this.useSlowFileEvents) {
// Let's not ignore errors
// eslint-disable-next-line @typescript-eslint/no-floating-promises
sleep(100).then(() => process.exit(1));
}
break;
case LogLevel.WARNING:
console.warn(formatted);
break;
case LogLevel.INFO:
// HACK: we have to ensure the file has been synced if we want to change it offline without data loss
const result = /.*History entry: (.*.md).*/.exec(
logLine.message
);
if (result) {
this.doNotTouchWhileOffline =
this.doNotTouchWhileOffline.filter(
(file) => file !== result[1]
);
}
console.info(formatted);
break;
case LogLevel.DEBUG:
@ -84,11 +91,10 @@ export class MockAgent extends MockClient {
this.changeFetchChangesUpdateIntervalMsAction.bind(this)
];
if (
this.client.settings.getSettings().isSyncEnabled &&
this.doNotTouchWhileOffline.length === 0
) {
options.push(this.disableSyncAction.bind(this));
if (this.client.settings.getSettings().isSyncEnabled) {
if (this.doNotTouchWhileOffline.length === 0) {
options.push(this.disableSyncAction.bind(this));
}
} else {
options.push(this.enableSyncAction.bind(this));
}
@ -186,6 +192,14 @@ export class MockAgent extends MockClient {
}
public assertAllContentIsPresentOnce(): void {
if (this.useSlowFileEvents) {
this.client.logger.info(
// We can't ensure that we have seen every single update
`Skipping content check for ${this.name} because slow file events are enabled`
);
return;
}
for (const content of this.writtenContents) {
const found = Array.from(this.localFiles.keys()).filter((key) => {
return new TextDecoder()
@ -215,7 +229,7 @@ export class MockAgent extends MockClient {
);
assert(
fileContent.split(content).length == 2,
`Content ${content} (of ${this.name}) found more than once in file ${file}. File content:\n${fileContent}`
`Content ${content} (of ${this.name}) found more than once in '${file}'. File content:\n${fileContent}`
);
}
}
@ -237,10 +251,7 @@ export class MockAgent extends MockClient {
`Decided to create file ${file} with content ${content}`
);
return this.create(
file,
new TextEncoder().encode(` |${content}| `)
);
return this.create(file, new TextEncoder().encode(` ${content} `));
}
private async changeFetchChangesUpdateIntervalMsAction(): Promise<void> {
@ -314,7 +325,7 @@ export class MockAgent extends MockClient {
`Decided to update file ${file} with ${content}`
);
this.doNotTouchWhileOffline.push(file);
await this.atomicUpdateText(file, (old) => old + ` |${content}| `);
await this.atomicUpdateText(file, (old) => old + ` ${content} `);
}
private async deleteFileAction(files: RelativePath[]): Promise<void> {

89
frontend/test-client/src/agent/mock-client.ts
View file

@ -1,9 +1,10 @@
import type {
RelativePath,
FileSystemOperations,
SyncSettings
import { assert } from "../utils/assert";
import {
type RelativePath,
type FileSystemOperations,
type SyncSettings,
SyncClient
} from "sync-client";
import { SyncClient } from "sync-client";
export class MockClient implements FileSystemOperations {
protected readonly localFiles = new Map<string, Uint8Array>();
@ -11,7 +12,8 @@ export class MockClient implements FileSystemOperations {
protected data: object | undefined = undefined;
public constructor(
private readonly initialSettings: Partial<SyncSettings>
private readonly initialSettings: Partial<SyncSettings>,
protected readonly useSlowFileEvents: boolean
) {}
public async init(): Promise<void> {
@ -22,9 +24,10 @@ export class MockClient implements FileSystemOperations {
await Promise.all(
Object.keys(this.initialSettings).map(async (key) => {
const settingKey = key as keyof SyncSettings; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
return this.client.settings.setSetting(
key as keyof SyncSettings, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
this.initialSettings[key as keyof SyncSettings] // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
settingKey,
this.initialSettings[settingKey]! // eslint-disable-line @typescript-eslint/no-non-null-assertion
);
})
);
@ -46,13 +49,6 @@ export class MockClient implements FileSystemOperations {
return (await this.read(path)).length;
}
public async getModificationTime(path: RelativePath): Promise<Date> {
if (!this.localFiles.has(path)) {
throw new Error(`File ${path} does not exist`);
}
return new Date();
}
public async exists(path: RelativePath): Promise<boolean> {
return this.localFiles.has(path);
}
@ -68,7 +64,10 @@ export class MockClient implements FileSystemOperations {
`Creating file ${path} with content ${new TextDecoder().decode(newContent)}`
);
this.localFiles.set(path, newContent);
void this.client.syncer.syncLocallyCreatedFile(path, new Date());
this.runCallback(() => {
void this.client.syncer.syncLocallyCreatedFile(path);
});
}
public async createDirectory(_path: RelativePath): Promise<void> {
@ -88,28 +87,51 @@ export class MockClient implements FileSystemOperations {
const newContentUint8Array = new TextEncoder().encode(newContent);
this.localFiles.set(path, newContentUint8Array);
if (!this.useSlowFileEvents) {
const existingParts = currentContent
.split(" ")
.map((part) => part.trim());
const newParts = newContent.split(" ").map((part) => part.trim());
existingParts.forEach((part) =>
// all changes should be additive
{
assert(
newParts.includes(part),
`Part ${part} not found in new content`
);
}
);
}
this.client.logger.info(
`Updated file ${path} with:\n current content: ${currentContent}\n new content: ${newContent}`
);
void this.client.syncer.syncLocallyUpdatedFile({
relativePath: path,
updateTime: new Date()
this.runCallback(() => {
void this.client.syncer.syncLocallyUpdatedFile({
relativePath: path
});
});
return newContent;
}
public async write(path: RelativePath, content: Uint8Array): Promise<void> {
const hasExisted = this.localFiles.has(path);
this.localFiles.set(path, content);
this.client.logger.info(
`Updated file ${path} with:\n new content: ${new TextDecoder().decode(content)}`
);
void this.client.syncer.syncLocallyUpdatedFile({
relativePath: path,
updateTime: new Date()
this.runCallback(() => {
if (hasExisted) {
void this.client.syncer.syncLocallyUpdatedFile({
relativePath: path
});
} else {
void this.client.syncer.syncLocallyCreatedFile(path);
}
});
}
@ -118,7 +140,10 @@ export class MockClient implements FileSystemOperations {
`Deleting file: ${path} with:\n content ${new TextDecoder().decode(this.localFiles.get(path))}`
);
this.localFiles.delete(path);
void this.client.syncer.syncLocallyDeletedFile(path);
this.runCallback(() => {
void this.client.syncer.syncLocallyDeletedFile(path);
});
}
public async rename(
@ -138,10 +163,20 @@ export class MockClient implements FileSystemOperations {
`Renamed file: ${oldPath} -> ${newPath} with:\n content ${new TextDecoder().decode(file)}`
);
void this.client.syncer.syncLocallyUpdatedFile({
oldPath,
relativePath: newPath,
updateTime: new Date()
this.runCallback(() => {
void this.client.syncer.syncLocallyUpdatedFile({
oldPath,
relativePath: newPath
});
});
}
private runCallback(callback: () => void): void {
if (this.useSlowFileEvents) {
// we aren't the best client and it takes some time to notice changes
setTimeout(callback, 100);
} else {
callback();
}
}
}