andras/vault-link
andras/vault-link
1
0
Fork 0
Code Issues Pull requests 19 Projects Releases 55 Packages Wiki Activity Actions 28

Add useSlowFileEvents

Browse source
This commit is contained in:
Andras Schmelczer 2025-03-15 18:01:33 +00:00
commit 78e1372483
No known key found for this signature in database
GPG key ID: FC8F2C3D3D1A718C
4 changed files with 87 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

2
frontend/sync-client/src/sync-operations/unrestricted-syncer.ts
View file

@ -158,7 +158,7 @@ export class UnrestrictedSyncer {
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
if (document.metadata === undefined) { if (document.metadata === undefined) {
throw new Error( throw new Error(
`Document ${document.relativePath} no longer has metadata after updating it` `Document ${document.relativePath} no longer has metadata after updating it, this cannot happen`
); );
} }

19
frontend/test-client/src/agent/mock-agent.ts
View file

@ -18,9 +18,10 @@ export class MockAgent extends MockClient {
initialSettings: Partial<SyncSettings>, initialSettings: Partial<SyncSettings>,
public readonly name: string, public readonly name: string,
private readonly doDeletes: boolean, private readonly doDeletes: boolean,
useSlowFileEvents: boolean,
private readonly jitterScaleInSeconds: number private readonly jitterScaleInSeconds: number
) { ) {
super(initialSettings); super(initialSettings, useSlowFileEvents);
} }
public async init(): Promise<void> { public async init(): Promise<void> {
@ -62,9 +63,11 @@ export class MockAgent extends MockClient {
case LogLevel.ERROR: case LogLevel.ERROR:
console.error(formatted); console.error(formatted);
// Let's not ignore errors if (!this.useSlowFileEvents) {
// eslint-disable-next-line @typescript-eslint/no-floating-promises // Let's not ignore errors
sleep(100).then(() => process.exit(1)); // eslint-disable-next-line @typescript-eslint/no-floating-promises
sleep(100).then(() => process.exit(1));
}
break; break;
case LogLevel.WARNING: case LogLevel.WARNING:
@ -189,6 +192,14 @@ export class MockAgent extends MockClient {
} }
public assertAllContentIsPresentOnce(): void { public assertAllContentIsPresentOnce(): void {
if (this.useSlowFileEvents) {
this.client.logger.info(
// We can't ensure that we have seen every single update
`Skipping content check for ${this.name} because slow file events are enabled`
);
return;
}
for (const content of this.writtenContents) { for (const content of this.writtenContents) {
const found = Array.from(this.localFiles.keys()).filter((key) => { const found = Array.from(this.localFiles.keys()).filter((key) => {
return new TextDecoder() return new TextDecoder()

56
frontend/test-client/src/agent/mock-client.ts
View file

@ -12,7 +12,8 @@ export class MockClient implements FileSystemOperations {
protected data: object | undefined = undefined; protected data: object | undefined = undefined;
public constructor( public constructor(
private readonly initialSettings: Partial<SyncSettings> private readonly initialSettings: Partial<SyncSettings>,
protected readonly useSlowFileEvents: boolean
) {} ) {}
public async init(): Promise<void> { public async init(): Promise<void> {
@ -64,8 +65,7 @@ export class MockClient implements FileSystemOperations {
); );
this.localFiles.set(path, newContent); this.localFiles.set(path, newContent);
// we aren't the best client and it takes some time to notice changes this.runCallback(() => {
setImmediate(() => {
void this.client.syncer.syncLocallyCreatedFile(path); void this.client.syncer.syncLocallyCreatedFile(path);
}); });
} }
@ -87,26 +87,27 @@ export class MockClient implements FileSystemOperations {
const newContentUint8Array = new TextEncoder().encode(newContent); const newContentUint8Array = new TextEncoder().encode(newContent);
this.localFiles.set(path, newContentUint8Array); this.localFiles.set(path, newContentUint8Array);
const existingParts = currentContent if (!this.useSlowFileEvents) {
.split(" ") const existingParts = currentContent
.map((part) => part.trim()); .split(" ")
const newParts = newContent.split(" ").map((part) => part.trim()); .map((part) => part.trim());
existingParts.forEach((part) => const newParts = newContent.split(" ").map((part) => part.trim());
// all changes should be additive existingParts.forEach((part) =>
{ // all changes should be additive
assert( {
newParts.includes(part), assert(
`Part ${part} not found in new content` newParts.includes(part),
); `Part ${part} not found in new content`
} );
); }
);
}
this.client.logger.info( this.client.logger.info(
`Updated file ${path} with:\n current content: ${currentContent}\n new content: ${newContent}` `Updated file ${path} with:\n current content: ${currentContent}\n new content: ${newContent}`
); );
// we aren't the best client and it takes some time to notice changes this.runCallback(() => {
setImmediate(() => {
void this.client.syncer.syncLocallyUpdatedFile({ void this.client.syncer.syncLocallyUpdatedFile({
relativePath: path relativePath: path
}); });
@ -123,8 +124,7 @@ export class MockClient implements FileSystemOperations {
`Updated file ${path} with:\n new content: ${new TextDecoder().decode(content)}` `Updated file ${path} with:\n new content: ${new TextDecoder().decode(content)}`
); );
// we aren't the best client and it takes some time to notice changes this.runCallback(() => {
setImmediate(() => {
if (hasExisted) { if (hasExisted) {
void this.client.syncer.syncLocallyUpdatedFile({ void this.client.syncer.syncLocallyUpdatedFile({
relativePath: path relativePath: path
@ -140,8 +140,8 @@ export class MockClient implements FileSystemOperations {
`Deleting file: ${path} with:\n content ${new TextDecoder().decode(this.localFiles.get(path))}` `Deleting file: ${path} with:\n content ${new TextDecoder().decode(this.localFiles.get(path))}`
); );
this.localFiles.delete(path); this.localFiles.delete(path);
// we aren't the best client and it takes some time to notice changes
setImmediate(() => { this.runCallback(() => {
void this.client.syncer.syncLocallyDeletedFile(path); void this.client.syncer.syncLocallyDeletedFile(path);
}); });
} }
@ -163,12 +163,20 @@ export class MockClient implements FileSystemOperations {
`Renamed file: ${oldPath} -> ${newPath} with:\n content ${new TextDecoder().decode(file)}` `Renamed file: ${oldPath} -> ${newPath} with:\n content ${new TextDecoder().decode(file)}`
); );
// we aren't the best client and it takes some time to notice changes this.runCallback(() => {
setImmediate(() => {
void this.client.syncer.syncLocallyUpdatedFile({ void this.client.syncer.syncLocallyUpdatedFile({
oldPath, oldPath,
relativePath: newPath relativePath: newPath
}); });
}); });
} }
private runCallback(callback: () => void): void {
if (this.useSlowFileEvents) {
// we aren't the best client and it takes some time to notice changes
setTimeout(callback, 100);
} else {
callback();
}
}
} }

51
frontend/test-client/src/cli.ts
View file

@ -3,20 +3,26 @@ import { MockAgent } from "./agent/mock-agent";
import { sleep } from "./utils/sleep"; import { sleep } from "./utils/sleep";
import { v4 as uuidv4 } from "uuid"; import { v4 as uuidv4 } from "uuid";
let slowFileEvents = false;
async function runTest({ async function runTest({
agentCount, agentCount,
concurrency, concurrency,
iterations, iterations,
doDeletes, doDeletes,
useSlowFileEvents,
jitterScaleInSeconds jitterScaleInSeconds
}: { }: {
agentCount: number; agentCount: number;
concurrency: number; concurrency: number;
iterations: number; iterations: number;
doDeletes: boolean; doDeletes: boolean;
useSlowFileEvents: boolean;
jitterScaleInSeconds: number; jitterScaleInSeconds: number;
}): Promise<void> { }): Promise<void> {
const settings = `with ${agentCount} agents, concurrency ${concurrency}, iterations ${iterations}, doDeletes ${doDeletes}, jitterScaleInSeconds ${jitterScaleInSeconds}`; slowFileEvents = useSlowFileEvents;
const settings = `with ${agentCount} agents, concurrency ${concurrency}, iterations ${iterations}, doDeletes ${doDeletes}, jitterScaleInSeconds ${jitterScaleInSeconds}, useSlowFileEvents ${useSlowFileEvents}`;
console.info(`Running test ${settings}`); console.info(`Running test ${settings}`);
const initialSettings: Partial<SyncSettings> = { const initialSettings: Partial<SyncSettings> = {
@ -34,6 +40,7 @@ async function runTest({
initialSettings, initialSettings,
`agent-${i}`, `agent-${i}`,
doDeletes, doDeletes,
useSlowFileEvents,
jitterScaleInSeconds jitterScaleInSeconds
) )
); );
@ -56,12 +63,24 @@ async function runTest({
// Each agent can have unpushed changes which might conflict with eachother so each has to resolve the conflicts & push, and // Each agent can have unpushed changes which might conflict with eachother so each has to resolve the conflicts & push, and
for (const client of clients) { for (const client of clients) {
await client.finish(); try {
await client.finish();
} catch (err) {
if (!slowFileEvents) {
throw err;
}
}
} }
// then we need a second pass to ensure that all agents pull the same state. // then we need a second pass to ensure that all agents pull the same state.
for (const client of clients) { for (const client of clients) {
await client.finish(); try {
await client.finish();
} catch (err) {
if (!slowFileEvents) {
throw err;
}
}
} }
console.info("Agents finished successfully"); console.info("Agents finished successfully");
@ -96,19 +115,21 @@ async function runTests(): Promise<void> {
16, 16,
1 // test with concurrency 1 to check for deadlocks 1 // test with concurrency 1 to check for deadlocks
]; ];
const doDeletes = [true, false];
for (const agentCount of agentCounts) { for (const agentCount of agentCounts) {
for (const concurrency of concurrencies) { for (const concurrency of concurrencies) {
for (const jitter of networkJitterScaleInSeconds) { for (const jitter of networkJitterScaleInSeconds) {
for (const deleteFiles of doDeletes) { for (const doDeletes of [true, false]) {
await runTest({ for (const useSlowFileEvents of [true, false]) {
agentCount, await runTest({
concurrency, agentCount,
iterations: 200, concurrency,
doDeletes: deleteFiles, iterations: 200,
jitterScaleInSeconds: jitter doDeletes,
}); useSlowFileEvents,
jitterScaleInSeconds: jitter
});
}
} }
} }
} }
@ -116,11 +137,17 @@ async function runTests(): Promise<void> {
} }
process.on("uncaughtException", (error) => { process.on("uncaughtException", (error) => {
if (slowFileEvents) {
return;
}
console.error("Uncaught Exception:", error); console.error("Uncaught Exception:", error);
process.exit(1); process.exit(1);
}); });
process.on("unhandledRejection", (reason, _promise) => { process.on("unhandledRejection", (reason, _promise) => {
if (slowFileEvents) {
return;
}
console.error("Unhandled Rejection:", reason); console.error("Unhandled Rejection:", reason);
process.exit(1); process.exit(1);
}); });