Extend E2E assertions
This commit is contained in:
parent
904a2737d4
commit
e8c57b3a37
5 changed files with 537 additions and 244 deletions
|
|
@ -1,11 +1,14 @@
|
|||
import type { SyncSettings } from "sync-client";
|
||||
import { utils } from "sync-client";
|
||||
import { utils, debugging, Logger } from "sync-client";
|
||||
import { MockAgent } from "./agent/mock-agent";
|
||||
import { sleep } from "./utils/sleep";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import { randomCasing } from "./utils/random-casing";
|
||||
import { TimeoutError } from "./utils/with-timeout";
|
||||
import { TestErrorTracker } from "./utils/test-error-tracker";
|
||||
|
||||
const TEST_ITERATIONS = 5;
|
||||
const MAX_INITIAL_DOCS = 10;
|
||||
|
||||
// Simulate async file access by injecting waiting time before returning from file operations.
|
||||
let slowFileEvents = false;
|
||||
|
|
@ -13,9 +16,13 @@ let slowFileEvents = false;
|
|||
// Whether to do resets in the test runs
|
||||
let doResets = false;
|
||||
|
||||
const logger = new Logger();
|
||||
debugging.logToConsole(logger);
|
||||
|
||||
const errorTracker = new TestErrorTracker();
|
||||
|
||||
async function runTest({
|
||||
agentCount,
|
||||
concurrency,
|
||||
iterations,
|
||||
doDeletes,
|
||||
useResets,
|
||||
|
|
@ -23,7 +30,6 @@ async function runTest({
|
|||
jitterScaleInSeconds
|
||||
}: {
|
||||
agentCount: number;
|
||||
concurrency: number;
|
||||
iterations: number;
|
||||
doDeletes: boolean;
|
||||
useResets: boolean;
|
||||
|
|
@ -32,18 +38,18 @@ async function runTest({
|
|||
}): Promise<void> {
|
||||
slowFileEvents = useSlowFileEvents;
|
||||
doResets = useResets;
|
||||
errorTracker.reset();
|
||||
|
||||
const settings = `with ${agentCount} agents, concurrency ${concurrency}, iterations ${iterations}, doDeletes ${doDeletes}, doResets ${useResets}, jitterScaleInSeconds ${jitterScaleInSeconds}, useSlowFileEvents ${useSlowFileEvents}`;
|
||||
console.info(`Running test ${settings}`);
|
||||
const settings = `with ${agentCount} agents, iterations ${iterations}, doDeletes ${doDeletes}, doResets ${useResets}, jitterScaleInSeconds ${jitterScaleInSeconds}, useSlowFileEvents ${useSlowFileEvents}`;
|
||||
logger.info(`Running test ${settings}`);
|
||||
|
||||
const vaultName = uuidv4();
|
||||
console.info(`Using vault name: ${vaultName}`);
|
||||
logger.info(`Using vault name: ${vaultName}`);
|
||||
const initialSettings: Partial<SyncSettings> = {
|
||||
isSyncEnabled: true,
|
||||
token: " test-token-change-me ", // same as in sync-server/config-e2e.yml with spaces
|
||||
vaultName: randomCasing(vaultName) + (Math.random() > 0.5 ? " " : ""), // extra spaces shouldn't matter
|
||||
syncConcurrency: concurrency,
|
||||
remoteUri: "http://localhost:3000"
|
||||
remoteUri: "http://localhost:3010"
|
||||
};
|
||||
|
||||
const clients: MockAgent[] = [];
|
||||
|
|
@ -55,67 +61,108 @@ async function runTest({
|
|||
doDeletes,
|
||||
useResets,
|
||||
useSlowFileEvents,
|
||||
jitterScaleInSeconds
|
||||
jitterScaleInSeconds,
|
||||
errorTracker
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
for (const client of clients) {
|
||||
const initialDocCount = Math.floor(
|
||||
Math.random() * MAX_INITIAL_DOCS
|
||||
);
|
||||
if (initialDocCount > 0) {
|
||||
logger.info(
|
||||
`Creating ${initialDocCount} initial documents for ${client.name}`
|
||||
);
|
||||
await client.createInitialDocuments(initialDocCount);
|
||||
}
|
||||
}
|
||||
|
||||
await utils.awaitAll(clients.map(async (client) => client.init()));
|
||||
|
||||
for (let i = 0; i < iterations; i++) {
|
||||
console.info(`Iteration ${i + 1}/${iterations}`);
|
||||
logger.info(`Iteration ${i + 1}/${iterations}`);
|
||||
await utils.awaitAll(clients.map(async (client) => client.act()));
|
||||
await sleep(Math.random() * 200);
|
||||
}
|
||||
|
||||
console.info("Stopping agents");
|
||||
errorTracker.checkAndThrow();
|
||||
|
||||
// Each agent can have unpushed changes which might conflict with eachother so each has to resolve the conflicts & push, and
|
||||
logger.info("Stopping agents");
|
||||
|
||||
// Drain pending actions and enable sync for each client
|
||||
for (const client of clients) {
|
||||
try {
|
||||
console.info(`Finishing up ${client.name}`);
|
||||
logger.info(`Finishing up ${client.name}`);
|
||||
await client.finish();
|
||||
} catch (err) {
|
||||
if (!slowFileEvents) {
|
||||
if (err instanceof TimeoutError || !slowFileEvents) {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// then we need a second pass to ensure that all agents pull the same state.
|
||||
// Settling rounds to drain cascading broadcasts between agents.
|
||||
// Completing work on agent A can trigger broadcasts to agent B,
|
||||
// which can cascade further. With N agents the worst case is N
|
||||
// hops, so N+1 passes guarantees all cascades are drained.
|
||||
for (let round = 0; round <= clients.length; round++) {
|
||||
for (const client of clients) {
|
||||
try {
|
||||
await client.waitUntilSynced();
|
||||
} catch (err) {
|
||||
if (err instanceof TimeoutError || !slowFileEvents) {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const client of clients) {
|
||||
try {
|
||||
console.info(`Destroying ${client.name}`);
|
||||
logger.info(`Destroying ${client.name}`);
|
||||
await client.destroy();
|
||||
} catch (err) {
|
||||
if (!slowFileEvents) {
|
||||
if (err instanceof TimeoutError || !slowFileEvents) {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.info("Agents finished successfully");
|
||||
logger.info("Agents finished successfully");
|
||||
errorTracker.checkAndThrow();
|
||||
|
||||
clients.slice(0, -1).forEach((client, i) => {
|
||||
console.info(
|
||||
logger.info(
|
||||
`Checking consistency between ${client.name} and ${clients[i + 1].name}`
|
||||
);
|
||||
client.assertFileSystemsAreConsistent(clients[i]);
|
||||
console.info(`Consistency check for ${client.name} passed`);
|
||||
client.assertFileSystemsAreConsistent(clients[i + 1]);
|
||||
logger.info(`Consistency check for ${client.name} passed`);
|
||||
});
|
||||
|
||||
console.info("File systems found to be consistent");
|
||||
logger.info("File systems found to be consistent");
|
||||
|
||||
clients.forEach((client) => {
|
||||
console.info(`Checking content for ${client.name}`);
|
||||
logger.info(`Checking content for ${client.name}`);
|
||||
client.assertAllContentIsPresentOnce();
|
||||
console.info(`Content check for ${client.name} passed`);
|
||||
logger.info(`Content check for ${client.name} passed`);
|
||||
});
|
||||
|
||||
console.info(`Test passed ${settings}`);
|
||||
clients.forEach((client) => {
|
||||
logger.info(
|
||||
`Checking binary content duplication for ${client.name}`
|
||||
);
|
||||
client.assertBinaryContentNotDuplicated();
|
||||
logger.info(
|
||||
`Binary content duplication check for ${client.name} passed`
|
||||
);
|
||||
});
|
||||
|
||||
logger.info(`Test passed ${settings}`);
|
||||
} catch (err) {
|
||||
console.error(`Test failed ${settings}`);
|
||||
logger.error(`Test failed ${settings}`);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
|
@ -124,7 +171,6 @@ async function runTests(): Promise<void> {
|
|||
for (let i = 0; i < TEST_ITERATIONS; i++) {
|
||||
await runTest({
|
||||
agentCount: 2,
|
||||
concurrency: 16,
|
||||
iterations: 100,
|
||||
doDeletes: true,
|
||||
useResets: true,
|
||||
|
|
@ -133,24 +179,59 @@ async function runTests(): Promise<void> {
|
|||
});
|
||||
|
||||
for (const useSlowFileEvents of [true, false]) {
|
||||
for (const concurrency of [
|
||||
16,
|
||||
1 // test with concurrency 1 to check for deadlocks
|
||||
]) {
|
||||
for (const doDeletes of [false, true]) {
|
||||
await runTest({
|
||||
agentCount: 2,
|
||||
concurrency,
|
||||
iterations: 100,
|
||||
doDeletes,
|
||||
useResets: false,
|
||||
useSlowFileEvents,
|
||||
jitterScaleInSeconds: 0.75
|
||||
});
|
||||
}
|
||||
for (const doDeletes of [false, true]) {
|
||||
await runTest({
|
||||
agentCount: 2,
|
||||
iterations: 100,
|
||||
doDeletes,
|
||||
useResets: false,
|
||||
useSlowFileEvents,
|
||||
jitterScaleInSeconds: 0.75
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await runTest({
|
||||
agentCount: 3,
|
||||
iterations: 75,
|
||||
doDeletes: true,
|
||||
useResets: false,
|
||||
useSlowFileEvents: false,
|
||||
jitterScaleInSeconds: 0.75
|
||||
});
|
||||
await runTest({
|
||||
agentCount: 3,
|
||||
iterations: 75,
|
||||
doDeletes: false,
|
||||
useResets: true,
|
||||
useSlowFileEvents: false,
|
||||
jitterScaleInSeconds: 0.75
|
||||
});
|
||||
await runTest({
|
||||
agentCount: 4,
|
||||
iterations: 50,
|
||||
doDeletes: true,
|
||||
useResets: false,
|
||||
useSlowFileEvents: false,
|
||||
jitterScaleInSeconds: 0.75
|
||||
});
|
||||
await runTest({
|
||||
agentCount: 2,
|
||||
iterations: 100,
|
||||
doDeletes: true,
|
||||
useResets: false,
|
||||
useSlowFileEvents: false,
|
||||
jitterScaleInSeconds: 0.1
|
||||
});
|
||||
await runTest({
|
||||
agentCount: 2,
|
||||
iterations: 100,
|
||||
doDeletes: true,
|
||||
useResets: true,
|
||||
useSlowFileEvents: false,
|
||||
jitterScaleInSeconds: 1.5
|
||||
});
|
||||
}
|
||||
|
||||
process.on("uncaughtException", (error) => {
|
||||
|
|
@ -163,12 +244,19 @@ process.on("uncaughtException", (error) => {
|
|||
return;
|
||||
}
|
||||
|
||||
console.error("Uncaught exception:", error);
|
||||
logger.error(`Error: uncaught exception: ${error}`);
|
||||
if (error instanceof Error && error.stack != null) {
|
||||
logger.error(error.stack);
|
||||
}
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
process.on("unhandledRejection", (error, _promise) => {
|
||||
if (error instanceof Error && error.message === "Sync was reset") {
|
||||
if (
|
||||
error instanceof Error &&
|
||||
(
|
||||
error.name === "SyncResetError")
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -191,7 +279,10 @@ process.on("unhandledRejection", (error, _promise) => {
|
|||
return;
|
||||
}
|
||||
|
||||
console.error("Unhandled rejection:", error);
|
||||
logger.error(`Error - unhandled rejection: ${error}`);
|
||||
if (error instanceof Error && error.stack != null) {
|
||||
logger.error(error.stack);
|
||||
}
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
|
|
@ -199,7 +290,10 @@ runTests()
|
|||
.then(() => {
|
||||
process.exit(0);
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
console.error(err);
|
||||
.catch((error: unknown) => {
|
||||
logger.error(`Error - tests failed with ${error}`);
|
||||
if (error instanceof Error && error.stack != null) {
|
||||
logger.error(error.stack);
|
||||
}
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue