Fix syncing when network latency is present (#4)
* WIP * Add debug * Dedupe inserts * Add deterministic ordering * Fix whitespaces * Update insta * Add integration test script * Rename * Add test * Working for non-deletes * omg it mostly works for deletes * Isdeleted fix * remove created dates * update api * Take document id * No max attempt * works * Use string uuids * . * working!!!! (hopefully) * Improve bundling * Add module * lint * . * lint * Fix CI * use toolchain * clean up * Add useSlowFileEvents * Delete fuzz * Fix CI * use docker * fix script * clean up * Clean up * change node version * Build docker image on every commit * fix ci * 1 db per vault * Add scritps folder * Bump versions * Lint * . * Fix tests for real * Style * . * try * Consistent ordering * Fix tests * hmm * . * Clean up diff * Fixes * . * Fix version bump * . * . * .
This commit is contained in:
GitHub
parent
bcf48c428d
commit
8b8f1d91d9
91 changed files with 2252 additions and 1586 deletions
|
|
@ -18,9 +18,10 @@ export class MockAgent extends MockClient {
|
|||
initialSettings: Partial<SyncSettings>,
|
||||
public readonly name: string,
|
||||
private readonly doDeletes: boolean,
|
||||
useSlowFileEvents: boolean,
|
||||
private readonly jitterScaleInSeconds: number
|
||||
) {
|
||||
super(initialSettings);
|
||||
super(initialSettings, useSlowFileEvents);
|
||||
}
|
||||
|
||||
public async init(): Promise<void> {
|
||||
|
|
@ -46,27 +47,33 @@ export class MockAgent extends MockClient {
|
|||
? "(online) "
|
||||
: "(offline)";
|
||||
const formatted = `[${this.name} ${state}] ${logLine.timestamp.toISOString()} ${logLine.level} ${logLine.message}`;
|
||||
|
||||
// HACK: we have to ensure the file has been synced if we want to change it offline without data loss
|
||||
const historyEntry = /.*History entry: (.*.md).*/.exec(
|
||||
logLine.message
|
||||
);
|
||||
|
||||
if (historyEntry) {
|
||||
this.doNotTouchWhileOffline =
|
||||
this.doNotTouchWhileOffline.filter(
|
||||
(file) => file !== historyEntry[1]
|
||||
);
|
||||
}
|
||||
switch (logLine.level) {
|
||||
case LogLevel.ERROR:
|
||||
console.error(formatted);
|
||||
// Let's not ignore errors
|
||||
process.exit(1);
|
||||
|
||||
if (!this.useSlowFileEvents) {
|
||||
// Let's not ignore errors
|
||||
// eslint-disable-next-line @typescript-eslint/no-floating-promises
|
||||
sleep(100).then(() => process.exit(1));
|
||||
}
|
||||
|
||||
break;
|
||||
case LogLevel.WARNING:
|
||||
console.warn(formatted);
|
||||
break;
|
||||
case LogLevel.INFO:
|
||||
// HACK: we have to ensure the file has been synced if we want to change it offline without data loss
|
||||
const result = /.*History entry: (.*.md).*/.exec(
|
||||
logLine.message
|
||||
);
|
||||
if (result) {
|
||||
this.doNotTouchWhileOffline =
|
||||
this.doNotTouchWhileOffline.filter(
|
||||
(file) => file !== result[1]
|
||||
);
|
||||
}
|
||||
|
||||
console.info(formatted);
|
||||
break;
|
||||
case LogLevel.DEBUG:
|
||||
|
|
@ -84,11 +91,10 @@ export class MockAgent extends MockClient {
|
|||
this.changeFetchChangesUpdateIntervalMsAction.bind(this)
|
||||
];
|
||||
|
||||
if (
|
||||
this.client.settings.getSettings().isSyncEnabled &&
|
||||
this.doNotTouchWhileOffline.length === 0
|
||||
) {
|
||||
options.push(this.disableSyncAction.bind(this));
|
||||
if (this.client.settings.getSettings().isSyncEnabled) {
|
||||
if (this.doNotTouchWhileOffline.length === 0) {
|
||||
options.push(this.disableSyncAction.bind(this));
|
||||
}
|
||||
} else {
|
||||
options.push(this.enableSyncAction.bind(this));
|
||||
}
|
||||
|
|
@ -186,6 +192,14 @@ export class MockAgent extends MockClient {
|
|||
}
|
||||
|
||||
public assertAllContentIsPresentOnce(): void {
|
||||
if (this.useSlowFileEvents) {
|
||||
this.client.logger.info(
|
||||
// We can't ensure that we have seen every single update
|
||||
`Skipping content check for ${this.name} because slow file events are enabled`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
for (const content of this.writtenContents) {
|
||||
const found = Array.from(this.localFiles.keys()).filter((key) => {
|
||||
return new TextDecoder()
|
||||
|
|
@ -215,7 +229,7 @@ export class MockAgent extends MockClient {
|
|||
);
|
||||
assert(
|
||||
fileContent.split(content).length == 2,
|
||||
`Content ${content} (of ${this.name}) found more than once in file ${file}. File content:\n${fileContent}`
|
||||
`Content ${content} (of ${this.name}) found more than once in '${file}'. File content:\n${fileContent}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -237,10 +251,7 @@ export class MockAgent extends MockClient {
|
|||
`Decided to create file ${file} with content ${content}`
|
||||
);
|
||||
|
||||
return this.create(
|
||||
file,
|
||||
new TextEncoder().encode(` |${content}| `)
|
||||
);
|
||||
return this.create(file, new TextEncoder().encode(` ${content} `));
|
||||
}
|
||||
|
||||
private async changeFetchChangesUpdateIntervalMsAction(): Promise<void> {
|
||||
|
|
@ -314,7 +325,7 @@ export class MockAgent extends MockClient {
|
|||
`Decided to update file ${file} with ${content}`
|
||||
);
|
||||
this.doNotTouchWhileOffline.push(file);
|
||||
await this.atomicUpdateText(file, (old) => old + ` |${content}| `);
|
||||
await this.atomicUpdateText(file, (old) => old + ` ${content} `);
|
||||
}
|
||||
|
||||
private async deleteFileAction(files: RelativePath[]): Promise<void> {
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
import type {
|
||||
RelativePath,
|
||||
FileSystemOperations,
|
||||
SyncSettings
|
||||
import { assert } from "../utils/assert";
|
||||
import {
|
||||
type RelativePath,
|
||||
type FileSystemOperations,
|
||||
type SyncSettings,
|
||||
SyncClient
|
||||
} from "sync-client";
|
||||
import { SyncClient } from "sync-client";
|
||||
|
||||
export class MockClient implements FileSystemOperations {
|
||||
protected readonly localFiles = new Map<string, Uint8Array>();
|
||||
|
|
@ -11,7 +12,8 @@ export class MockClient implements FileSystemOperations {
|
|||
protected data: object | undefined = undefined;
|
||||
|
||||
public constructor(
|
||||
private readonly initialSettings: Partial<SyncSettings>
|
||||
private readonly initialSettings: Partial<SyncSettings>,
|
||||
protected readonly useSlowFileEvents: boolean
|
||||
) {}
|
||||
|
||||
public async init(): Promise<void> {
|
||||
|
|
@ -22,9 +24,10 @@ export class MockClient implements FileSystemOperations {
|
|||
|
||||
await Promise.all(
|
||||
Object.keys(this.initialSettings).map(async (key) => {
|
||||
const settingKey = key as keyof SyncSettings; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
|
||||
return this.client.settings.setSetting(
|
||||
key as keyof SyncSettings, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
|
||||
this.initialSettings[key as keyof SyncSettings] // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
|
||||
settingKey,
|
||||
this.initialSettings[settingKey]! // eslint-disable-line @typescript-eslint/no-non-null-assertion
|
||||
);
|
||||
})
|
||||
);
|
||||
|
|
@ -46,13 +49,6 @@ export class MockClient implements FileSystemOperations {
|
|||
return (await this.read(path)).length;
|
||||
}
|
||||
|
||||
public async getModificationTime(path: RelativePath): Promise<Date> {
|
||||
if (!this.localFiles.has(path)) {
|
||||
throw new Error(`File ${path} does not exist`);
|
||||
}
|
||||
return new Date();
|
||||
}
|
||||
|
||||
public async exists(path: RelativePath): Promise<boolean> {
|
||||
return this.localFiles.has(path);
|
||||
}
|
||||
|
|
@ -68,7 +64,10 @@ export class MockClient implements FileSystemOperations {
|
|||
`Creating file ${path} with content ${new TextDecoder().decode(newContent)}`
|
||||
);
|
||||
this.localFiles.set(path, newContent);
|
||||
void this.client.syncer.syncLocallyCreatedFile(path, new Date());
|
||||
|
||||
this.runCallback(() => {
|
||||
void this.client.syncer.syncLocallyCreatedFile(path);
|
||||
});
|
||||
}
|
||||
|
||||
public async createDirectory(_path: RelativePath): Promise<void> {
|
||||
|
|
@ -88,28 +87,51 @@ export class MockClient implements FileSystemOperations {
|
|||
const newContentUint8Array = new TextEncoder().encode(newContent);
|
||||
this.localFiles.set(path, newContentUint8Array);
|
||||
|
||||
if (!this.useSlowFileEvents) {
|
||||
const existingParts = currentContent
|
||||
.split(" ")
|
||||
.map((part) => part.trim());
|
||||
const newParts = newContent.split(" ").map((part) => part.trim());
|
||||
existingParts.forEach((part) =>
|
||||
// all changes should be additive
|
||||
{
|
||||
assert(
|
||||
newParts.includes(part),
|
||||
`Part ${part} not found in new content`
|
||||
);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
this.client.logger.info(
|
||||
`Updated file ${path} with:\n current content: ${currentContent}\n new content: ${newContent}`
|
||||
);
|
||||
|
||||
void this.client.syncer.syncLocallyUpdatedFile({
|
||||
relativePath: path,
|
||||
updateTime: new Date()
|
||||
this.runCallback(() => {
|
||||
void this.client.syncer.syncLocallyUpdatedFile({
|
||||
relativePath: path
|
||||
});
|
||||
});
|
||||
|
||||
return newContent;
|
||||
}
|
||||
|
||||
public async write(path: RelativePath, content: Uint8Array): Promise<void> {
|
||||
const hasExisted = this.localFiles.has(path);
|
||||
this.localFiles.set(path, content);
|
||||
|
||||
this.client.logger.info(
|
||||
`Updated file ${path} with:\n new content: ${new TextDecoder().decode(content)}`
|
||||
);
|
||||
|
||||
void this.client.syncer.syncLocallyUpdatedFile({
|
||||
relativePath: path,
|
||||
updateTime: new Date()
|
||||
this.runCallback(() => {
|
||||
if (hasExisted) {
|
||||
void this.client.syncer.syncLocallyUpdatedFile({
|
||||
relativePath: path
|
||||
});
|
||||
} else {
|
||||
void this.client.syncer.syncLocallyCreatedFile(path);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -118,7 +140,10 @@ export class MockClient implements FileSystemOperations {
|
|||
`Deleting file: ${path} with:\n content ${new TextDecoder().decode(this.localFiles.get(path))}`
|
||||
);
|
||||
this.localFiles.delete(path);
|
||||
void this.client.syncer.syncLocallyDeletedFile(path);
|
||||
|
||||
this.runCallback(() => {
|
||||
void this.client.syncer.syncLocallyDeletedFile(path);
|
||||
});
|
||||
}
|
||||
|
||||
public async rename(
|
||||
|
|
@ -138,10 +163,20 @@ export class MockClient implements FileSystemOperations {
|
|||
`Renamed file: ${oldPath} -> ${newPath} with:\n content ${new TextDecoder().decode(file)}`
|
||||
);
|
||||
|
||||
void this.client.syncer.syncLocallyUpdatedFile({
|
||||
oldPath,
|
||||
relativePath: newPath,
|
||||
updateTime: new Date()
|
||||
this.runCallback(() => {
|
||||
void this.client.syncer.syncLocallyUpdatedFile({
|
||||
oldPath,
|
||||
relativePath: newPath
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private runCallback(callback: () => void): void {
|
||||
if (this.useSlowFileEvents) {
|
||||
// we aren't the best client and it takes some time to notice changes
|
||||
setTimeout(callback, 100);
|
||||
} else {
|
||||
callback();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,20 +3,26 @@ import { MockAgent } from "./agent/mock-agent";
|
|||
import { sleep } from "./utils/sleep";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
|
||||
let slowFileEvents = false;
|
||||
|
||||
async function runTest({
|
||||
agentCount,
|
||||
concurrency,
|
||||
iterations,
|
||||
doDeletes,
|
||||
useSlowFileEvents,
|
||||
jitterScaleInSeconds
|
||||
}: {
|
||||
agentCount: number;
|
||||
concurrency: number;
|
||||
iterations: number;
|
||||
doDeletes: boolean;
|
||||
useSlowFileEvents: boolean;
|
||||
jitterScaleInSeconds: number;
|
||||
}): Promise<void> {
|
||||
const settings = `with ${agentCount} agents, concurrency ${concurrency}, iterations ${iterations}, doDeletes ${doDeletes}, jitterScaleInSeconds ${jitterScaleInSeconds}`;
|
||||
slowFileEvents = useSlowFileEvents;
|
||||
|
||||
const settings = `with ${agentCount} agents, concurrency ${concurrency}, iterations ${iterations}, doDeletes ${doDeletes}, jitterScaleInSeconds ${jitterScaleInSeconds}, useSlowFileEvents ${useSlowFileEvents}`;
|
||||
console.info(`Running test ${settings}`);
|
||||
|
||||
const initialSettings: Partial<SyncSettings> = {
|
||||
|
|
@ -34,6 +40,7 @@ async function runTest({
|
|||
initialSettings,
|
||||
`agent-${i}`,
|
||||
doDeletes,
|
||||
useSlowFileEvents,
|
||||
jitterScaleInSeconds
|
||||
)
|
||||
);
|
||||
|
|
@ -52,12 +59,24 @@ async function runTest({
|
|||
|
||||
// Each agent can have unpushed changes which might conflict with eachother so each has to resolve the conflicts & push, and
|
||||
for (const client of clients) {
|
||||
await client.finish();
|
||||
try {
|
||||
await client.finish();
|
||||
} catch (err) {
|
||||
if (!slowFileEvents) {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// then we need a second pass to ensure that all agents pull the same state.
|
||||
for (const client of clients) {
|
||||
await client.finish();
|
||||
try {
|
||||
await client.finish();
|
||||
} catch (err) {
|
||||
if (!slowFileEvents) {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.info("Agents finished successfully");
|
||||
|
|
@ -78,41 +97,49 @@ async function runTest({
|
|||
console.info(`Content check for ${client.name} passed`);
|
||||
});
|
||||
|
||||
console.info(`Test passed with ${settings}`);
|
||||
console.info(`Test passed ${settings}`);
|
||||
} catch (err) {
|
||||
console.error(`Test failed with ${settings}`);
|
||||
console.error(`Test failed ${settings}`);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async function runTests(): Promise<void> {
|
||||
const agentCounts = [2, 10];
|
||||
const jitterScaleInSeconds = [0.5, 3, 0];
|
||||
const concurrencies = [1, 16];
|
||||
const iterations = [50, 300];
|
||||
const doDeletes = [false, true];
|
||||
|
||||
for (const agentCount of agentCounts) {
|
||||
for (const concurrency of concurrencies) {
|
||||
for (const jitter of jitterScaleInSeconds) {
|
||||
for (const iteration of iterations) {
|
||||
for (const deleteFiles of doDeletes) {
|
||||
while (true) {
|
||||
await runTest({
|
||||
agentCount,
|
||||
concurrency,
|
||||
iterations: iteration,
|
||||
doDeletes: deleteFiles,
|
||||
jitterScaleInSeconds: jitter
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const useSlowFileEvents of [false, true]) {
|
||||
for (const concurrency of [
|
||||
16,
|
||||
1 // test with concurrency 1 to check for deadlocks
|
||||
]) {
|
||||
for (const doDeletes of [true, false]) {
|
||||
await runTest({
|
||||
agentCount: 3,
|
||||
concurrency,
|
||||
iterations: 100,
|
||||
doDeletes,
|
||||
useSlowFileEvents,
|
||||
jitterScaleInSeconds: 0.75
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
process.on("uncaughtException", (error) => {
|
||||
if (slowFileEvents) {
|
||||
return;
|
||||
}
|
||||
console.error("Uncaught Exception:", error);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
process.on("unhandledRejection", (reason, _promise) => {
|
||||
if (slowFileEvents) {
|
||||
return;
|
||||
}
|
||||
console.error("Unhandled Rejection:", reason);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
runTests()
|
||||
.then(() => {
|
||||
process.exit(0);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue