From 78e1372483d8a341efd96000d11152fee70239c0 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 15 Mar 2025 18:01:33 +0000 Subject: [PATCH] Add useSlowFileEvents --- .../sync-operations/unrestricted-syncer.ts | 2 +- frontend/test-client/src/agent/mock-agent.ts | 19 +++++-- frontend/test-client/src/agent/mock-client.ts | 56 +++++++++++-------- frontend/test-client/src/cli.ts | 51 +++++++++++++---- 4 files changed, 87 insertions(+), 41 deletions(-) diff --git a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts index 01cb665e..8300f10b 100644 --- a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts +++ b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts @@ -158,7 +158,7 @@ export class UnrestrictedSyncer { // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition if (document.metadata === undefined) { throw new Error( - `Document ${document.relativePath} no longer has metadata after updating it` + `Document ${document.relativePath} no longer has metadata after updating it, this cannot happen` ); } diff --git a/frontend/test-client/src/agent/mock-agent.ts b/frontend/test-client/src/agent/mock-agent.ts index 94c106da..bbed0ef2 100644 --- a/frontend/test-client/src/agent/mock-agent.ts +++ b/frontend/test-client/src/agent/mock-agent.ts @@ -18,9 +18,10 @@ export class MockAgent extends MockClient { initialSettings: Partial, public readonly name: string, private readonly doDeletes: boolean, + useSlowFileEvents: boolean, private readonly jitterScaleInSeconds: number ) { - super(initialSettings); + super(initialSettings, useSlowFileEvents); } public async init(): Promise { @@ -62,9 +63,11 @@ export class MockAgent extends MockClient { case LogLevel.ERROR: console.error(formatted); - // Let's not ignore errors - // eslint-disable-next-line @typescript-eslint/no-floating-promises - sleep(100).then(() => process.exit(1)); + if (!this.useSlowFileEvents) { + // Let's not ignore errors + // eslint-disable-next-line @typescript-eslint/no-floating-promises + sleep(100).then(() => process.exit(1)); + } break; case LogLevel.WARNING: @@ -189,6 +192,14 @@ export class MockAgent extends MockClient { } public assertAllContentIsPresentOnce(): void { + if (this.useSlowFileEvents) { + this.client.logger.info( + // We can't ensure that we have seen every single update + `Skipping content check for ${this.name} because slow file events are enabled` + ); + return; + } + for (const content of this.writtenContents) { const found = Array.from(this.localFiles.keys()).filter((key) => { return new TextDecoder() diff --git a/frontend/test-client/src/agent/mock-client.ts b/frontend/test-client/src/agent/mock-client.ts index 8aa0141d..7e4e14c3 100644 --- a/frontend/test-client/src/agent/mock-client.ts +++ b/frontend/test-client/src/agent/mock-client.ts @@ -12,7 +12,8 @@ export class MockClient implements FileSystemOperations { protected data: object | undefined = undefined; public constructor( - private readonly initialSettings: Partial + private readonly initialSettings: Partial, + protected readonly useSlowFileEvents: boolean ) {} public async init(): Promise { @@ -64,8 +65,7 @@ export class MockClient implements FileSystemOperations { ); this.localFiles.set(path, newContent); - // we aren't the best client and it takes some time to notice changes - setImmediate(() => { + this.runCallback(() => { void this.client.syncer.syncLocallyCreatedFile(path); }); } @@ -87,26 +87,27 @@ export class MockClient implements FileSystemOperations { const newContentUint8Array = new TextEncoder().encode(newContent); this.localFiles.set(path, newContentUint8Array); - const existingParts = currentContent - .split(" ") - .map((part) => part.trim()); - const newParts = newContent.split(" ").map((part) => part.trim()); - existingParts.forEach((part) => - // all changes should be additive - { - assert( - newParts.includes(part), - `Part ${part} not found in new content` - ); - } - ); + if (!this.useSlowFileEvents) { + const existingParts = currentContent + .split(" ") + .map((part) => part.trim()); + const newParts = newContent.split(" ").map((part) => part.trim()); + existingParts.forEach((part) => + // all changes should be additive + { + assert( + newParts.includes(part), + `Part ${part} not found in new content` + ); + } + ); + } this.client.logger.info( `Updated file ${path} with:\n current content: ${currentContent}\n new content: ${newContent}` ); - // we aren't the best client and it takes some time to notice changes - setImmediate(() => { + this.runCallback(() => { void this.client.syncer.syncLocallyUpdatedFile({ relativePath: path }); @@ -123,8 +124,7 @@ export class MockClient implements FileSystemOperations { `Updated file ${path} with:\n new content: ${new TextDecoder().decode(content)}` ); - // we aren't the best client and it takes some time to notice changes - setImmediate(() => { + this.runCallback(() => { if (hasExisted) { void this.client.syncer.syncLocallyUpdatedFile({ relativePath: path @@ -140,8 +140,8 @@ export class MockClient implements FileSystemOperations { `Deleting file: ${path} with:\n content ${new TextDecoder().decode(this.localFiles.get(path))}` ); this.localFiles.delete(path); - // we aren't the best client and it takes some time to notice changes - setImmediate(() => { + + this.runCallback(() => { void this.client.syncer.syncLocallyDeletedFile(path); }); } @@ -163,12 +163,20 @@ export class MockClient implements FileSystemOperations { `Renamed file: ${oldPath} -> ${newPath} with:\n content ${new TextDecoder().decode(file)}` ); - // we aren't the best client and it takes some time to notice changes - setImmediate(() => { + this.runCallback(() => { void this.client.syncer.syncLocallyUpdatedFile({ oldPath, relativePath: newPath }); }); } + + private runCallback(callback: () => void): void { + if (this.useSlowFileEvents) { + // we aren't the best client and it takes some time to notice changes + setTimeout(callback, 100); + } else { + callback(); + } + } } diff --git a/frontend/test-client/src/cli.ts b/frontend/test-client/src/cli.ts index 50543d94..02e86237 100644 --- a/frontend/test-client/src/cli.ts +++ b/frontend/test-client/src/cli.ts @@ -3,20 +3,26 @@ import { MockAgent } from "./agent/mock-agent"; import { sleep } from "./utils/sleep"; import { v4 as uuidv4 } from "uuid"; +let slowFileEvents = false; + async function runTest({ agentCount, concurrency, iterations, doDeletes, + useSlowFileEvents, jitterScaleInSeconds }: { agentCount: number; concurrency: number; iterations: number; doDeletes: boolean; + useSlowFileEvents: boolean; jitterScaleInSeconds: number; }): Promise { - const settings = `with ${agentCount} agents, concurrency ${concurrency}, iterations ${iterations}, doDeletes ${doDeletes}, jitterScaleInSeconds ${jitterScaleInSeconds}`; + slowFileEvents = useSlowFileEvents; + + const settings = `with ${agentCount} agents, concurrency ${concurrency}, iterations ${iterations}, doDeletes ${doDeletes}, jitterScaleInSeconds ${jitterScaleInSeconds}, useSlowFileEvents ${useSlowFileEvents}`; console.info(`Running test ${settings}`); const initialSettings: Partial = { @@ -34,6 +40,7 @@ async function runTest({ initialSettings, `agent-${i}`, doDeletes, + useSlowFileEvents, jitterScaleInSeconds ) ); @@ -56,12 +63,24 @@ async function runTest({ // Each agent can have unpushed changes which might conflict with eachother so each has to resolve the conflicts & push, and for (const client of clients) { - await client.finish(); + try { + await client.finish(); + } catch (err) { + if (!slowFileEvents) { + throw err; + } + } } // then we need a second pass to ensure that all agents pull the same state. for (const client of clients) { - await client.finish(); + try { + await client.finish(); + } catch (err) { + if (!slowFileEvents) { + throw err; + } + } } console.info("Agents finished successfully"); @@ -96,19 +115,21 @@ async function runTests(): Promise { 16, 1 // test with concurrency 1 to check for deadlocks ]; - const doDeletes = [true, false]; for (const agentCount of agentCounts) { for (const concurrency of concurrencies) { for (const jitter of networkJitterScaleInSeconds) { - for (const deleteFiles of doDeletes) { - await runTest({ - agentCount, - concurrency, - iterations: 200, - doDeletes: deleteFiles, - jitterScaleInSeconds: jitter - }); + for (const doDeletes of [true, false]) { + for (const useSlowFileEvents of [true, false]) { + await runTest({ + agentCount, + concurrency, + iterations: 200, + doDeletes, + useSlowFileEvents, + jitterScaleInSeconds: jitter + }); + } } } } @@ -116,11 +137,17 @@ async function runTests(): Promise { } process.on("uncaughtException", (error) => { + if (slowFileEvents) { + return; + } console.error("Uncaught Exception:", error); process.exit(1); }); process.on("unhandledRejection", (reason, _promise) => { + if (slowFileEvents) { + return; + } console.error("Unhandled Rejection:", reason); process.exit(1); });