Add deterministic tests

frontend/deterministic-tests/src/cli.ts

@@ -0,0 +1,228 @@
+import { TestRunner } from "./test-runner";
+import { ServerControl } from "./server-control";
+import { ServerManager } from "./server-manager";
+import { PrefixedLogger } from "./prefixed-logger";
+import { TESTS } from "./test-registry";
+import type { TestDefinition, TestResult } from "./test-definition";
+import { parseConcurrency } from "./parse-concurrency";
+import { runWithConcurrency } from "./run-with-concurrency";
+import { TOKEN, SERVER_BINARY_PATH, CONFIG_PATH } from "./consts";
+import * as path from "node:path";
+import * as fs from "node:fs";
+import { debugging, Logger } from "sync-client";
+
+const logger = new Logger();
+debugging.logToConsole(logger, { useColors: true });
+
+process.on("unhandledRejection", (reason) => {
+    logger.error(`Unhandled Rejection: ${reason}`);
+    process.exit(1);
+});
+
+process.on("uncaughtException", (error) => {
+    logger.error(`Uncaught Exception: ${error}`);
+    process.exit(1);
+});
+
+const serverManager = new ServerManager(logger);
+serverManager.installSignalHandlers();
+
+function testUsesPauseServer(test: TestDefinition): boolean {
+    return test.steps.some(
+        (step) => step.type === "pause-server" || step.type === "resume-server"
+    );
+}
+
+interface NamedTestResult {
+    test: TestDefinition;
+    result: TestResult;
+}
+
+
+async function main(): Promise<void> {
+    const cwd = process.cwd();
+    let projectRoot = cwd;
+
+    if (cwd.endsWith("frontend/deterministic-tests")) {
+        projectRoot = path.resolve(cwd, "../..");
+    } else if (cwd.endsWith("frontend")) {
+        projectRoot = path.resolve(cwd, "..");
+    }
+
+    const serverPath = path.join(projectRoot, SERVER_BINARY_PATH);
+    if (!fs.existsSync(serverPath)) {
+        logger.error(`Server binary not found at: ${serverPath}`);
+        process.exit(1);
+    }
+
+    const configPath = path.join(projectRoot, CONFIG_PATH);
+    if (!fs.existsSync(configPath)) {
+        logger.error(`Config file not found at: ${configPath}`);
+        process.exit(1);
+    }
+
+    const filterArg = process.argv.find((a) => a.startsWith("--filter="));
+    const filter = filterArg?.slice("--filter=".length);
+
+    const testsToRun: TestDefinition[] = [];
+    for (const [key, test] of Object.entries(TESTS)) {
+        if (test) {
+            if (filter && !key.includes(filter) && !test.name.toLowerCase().includes(filter.toLowerCase())) {
+                continue;
+            }
+            testsToRun.push(test);
+        }
+    }
+
+    if (testsToRun.length === 0) {
+        logger.error(
+            filter
+                ? `No tests matched filter "${filter}"`
+                : "No tests found"
+        );
+        process.exit(1);
+    }
+
+    const concurrency = parseConcurrency();
+    const regularTests = testsToRun.filter((t) => !testUsesPauseServer(t));
+    const pauseTests = testsToRun.filter((t) => testUsesPauseServer(t));
+
+    logger.info(`Server: ${serverPath}`);
+    logger.info(`Config: ${configPath}`);
+    logger.info(
+        `Tests: ${testsToRun.length} total (${regularTests.length} regular, ${pauseTests.length} server-pause)`
+    );
+    logger.info(`Concurrency: ${concurrency}`);
+
+    const allResults: NamedTestResult[] = [];
+
+    if (regularTests.length > 0) {
+        logger.info(
+            `\n--- Running ${regularTests.length} regular tests (shared server, concurrency ${concurrency}) ---`
+        );
+        const sharedServer = new ServerControl(
+            serverPath,
+            configPath,
+            logger
+        );
+        serverManager.track(sharedServer);
+
+        try {
+            await sharedServer.start();
+
+            const results = await runWithConcurrency(
+                regularTests,
+                concurrency,
+                async (test) => runSharedServerTest(test, sharedServer)
+            );
+
+            allResults.push(...results);
+        } finally {
+            try {
+                await sharedServer.stop();
+            } catch (error) {
+                logger.warn(
+                    `Error stopping shared server: ${error instanceof Error ? error.message : String(error)}`
+                );
+            }
+            serverManager.untrack(sharedServer);
+        }
+    }
+
+    if (pauseTests.length > 0) {
+        logger.info(
+            `\n--- Running ${pauseTests.length} server-pause tests (dedicated servers, concurrency ${concurrency}) ---`
+        );
+
+        const results = await runWithConcurrency(
+            pauseTests,
+            concurrency,
+            async (test) => runDedicatedServerTest(test, serverPath, configPath)
+        );
+
+        allResults.push(...results);
+    }
+
+    const passed = allResults.filter((r) => r.result.success);
+    const failed = allResults.filter((r) => !r.result.success);
+
+    logger.info(`\n--- Results: ${passed.length}/${allResults.length} passed ---`);
+
+    if (failed.length > 0) {
+        for (const { test, result } of failed) {
+            logger.error(`  FAILED: ${test.name}: ${result.error}`);
+        }
+        process.exit(1);
+    } else {
+        logger.info("All tests passed!");
+        process.exit(0);
+    }
+}
+
+main().catch((err: unknown) => {
+    logger.error(`Unexpected error: ${err}`);
+    process.exit(1);
+});
+
+
+/**
+ * Run a test on a shared server (for tests that don't use pause-server).
+ */
+async function runSharedServerTest(
+    test: TestDefinition,
+    sharedServer: ServerControl
+): Promise<NamedTestResult> {
+    const testLogger = new PrefixedLogger(logger, test.name);
+    const runner = new TestRunner(
+        sharedServer,
+        testLogger,
+        TOKEN,
+        sharedServer.remoteUri
+    );
+    const result = await runner.runTest(test);
+    if (result.success) {
+        logger.info(`PASSED: ${test.name} (${result.duration}ms)`);
+    } else {
+        logger.error(`FAILED: ${test.name} - ${result.error}`);
+    }
+    return { test, result };
+}
+
+/**
+ * Run a test with its own dedicated server (for tests that use pause-server).
+ * SIGSTOP/SIGCONT affects the entire server process, so these tests need
+ * isolated servers to avoid interfering with other tests.
+ */
+async function runDedicatedServerTest(
+    test: TestDefinition,
+    serverPath: string,
+    configPath: string
+): Promise<NamedTestResult> {
+    const testLogger = new PrefixedLogger(logger, test.name);
+    const server = new ServerControl(serverPath, configPath, testLogger);
+    serverManager.track(server);
+
+    try {
+        await server.start();
+        const runner = new TestRunner(
+            server,
+            testLogger,
+            TOKEN,
+            server.remoteUri
+        );
+        const result = await runner.runTest(test);
+        if (result.success) {
+            logger.info(`PASSED: ${test.name} (${result.duration}ms)`);
+        } else {
+            logger.error(`FAILED: ${test.name} - ${result.error}`);
+        }
+        return { test, result };
+    } finally {
+        try {
+            await server.stop();
+        } catch {
+            // best-effort cleanup
+        }
+        serverManager.untrack(server);
+    }
+}