Investigate dead-lock

.github/workflows/e2e.yml

@@ -49,3 +49,7 @@ jobs:
           cd ..
 
           scripts/e2e.sh 16
+
+      - name: Cleanup
+        if: always()
+        run: scripts/clean-up.sh

frontend/test-client/src/agent/mock-agent.ts

@@ -5,7 +5,10 @@ import type { RelativePath, SyncSettings } from "sync-client";
 import { debugging, Logger, LogLevel } from "sync-client";
 import { MockClient } from "./mock-client";
 import { sleep } from "../utils/sleep";
-import type { LogLine } from "sync-client/dist/types/tracing/logger";
+import type { LogLine } from "sync-client";
+import { withTimeout } from "../utils/with-timeout";
+
+const TIMEOUT_MS = 10 * 60 * 1000;
 
 export class MockAgent extends MockClient {
 	private readonly writtenContents: string[] = [];
@@ -134,15 +137,27 @@ export class MockAgent extends MockClient {
 	}
 
 	public async finish(): Promise<void> {
-		await this.client.setSetting("isSyncEnabled", true);
-		// eslint-disable-next-line no-restricted-properties
-		await Promise.all(this.pendingActions);
-		await this.client.waitUntilFinished();
+		await withTimeout(
+			(async (): Promise<void> => {
+				await this.client.setSetting("isSyncEnabled", true);
+				// eslint-disable-next-line no-restricted-properties
+				await Promise.all(this.pendingActions);
+				await this.client.waitUntilFinished();
+			})(),
+			TIMEOUT_MS,
+			"finish()"
+		);
 	}
 
 	public async destroy(): Promise<void> {
-		await this.client.waitUntilFinished();
-		await this.client.destroy();
+		await withTimeout(
+			(async (): Promise<void> => {
+				await this.client.waitUntilFinished();
+				await this.client.destroy();
+			})(),
+			TIMEOUT_MS,
+			"destroy()"
+		);
 	}
 
 	public assertFileSystemsAreConsistent(otherAgent: MockAgent): void {
@@ -184,14 +199,14 @@ export class MockAgent extends MockClient {
 			);
 			this.client.logger.info(
 				"Local files: " +
-					Array.from(otherAgent.localFiles.keys()).join(", ")
+				Array.from(otherAgent.localFiles.keys()).join(", ")
 			);
 			otherAgent.client.logger.info(
 				"Local data: " + JSON.stringify(otherAgent.data, null, 2)
 			);
 			otherAgent.client.logger.info(
 				"Local files: " +
-					Array.from(otherAgent.localFiles.keys()).join(", ")
+				Array.from(otherAgent.localFiles.keys()).join(", ")
 			);
 
 			throw e;

frontend/test-client/src/cli.ts

@@ -71,6 +71,7 @@ async function runTest({
 		// Each agent can have unpushed changes which might conflict with eachother so each has to resolve the conflicts & push, and
 		for (const client of clients) {
 			try {
+				console.info(`Finishing up ${client.name}`);
 				await client.finish();
 			} catch (err) {
 				if (!slowFileEvents) {
@@ -82,6 +83,7 @@ async function runTest({
 		// then we need a second pass to ensure that all agents pull the same state.
 		for (const client of clients) {
 			try {
+				console.info(`Destroying ${client.name}`);
 				await client.destroy();
 			} catch (err) {
 				if (!slowFileEvents) {

frontend/test-client/src/utils/with-timeout.ts

@@ -0,0 +1,23 @@
+
+export async function withTimeout<T>(
+	promise: Promise<T>,
+	timeoutMs: number,
+	operationName: string
+): Promise<T> {
+	return Promise.race([
+		promise,
+		new Promise<T>((_, reject) =>
+			setTimeout(
+				() =>
+					{ reject(
+						new Error(
+							`${operationName} timed out after ${timeoutMs}ms`
+						)
+					); },
+				timeoutMs
+			)
+		)
+	]);
+}
+
+

scripts/e2e.sh

@@ -3,6 +3,9 @@
 set -e
 set -o pipefail
 
+NO_COLOR=1
+FORCE_COLOR=0
+
 node_version=$(node -v | sed 's/^v\([0-9]*\).*/\1/')
 if [ "$node_version" != "22" ]; then
     echo "Error: This script requires Node.js version 22, found: $node_version"
@@ -37,8 +40,18 @@ cd frontend
 
 pids=()
 for i in $(seq 1 $process_count); do
-    node test-client/dist/cli.js > "../logs/log_${i}.log" 2>&1 &
-    pids+=($!)
+    # Create a named pipe for this process
+    pipe="/tmp/vaultlink_pipe_$$_$i"
+    mkfifo "$pipe"
+
+    # Start the node process writing to the pipe
+    node test-client/dist/cli.js > "$pipe" 2>&1 &
+    pid=$!
+    pids+=($pid)
+    echo "Started process $i with PID: $pid"
+
+    # Read from pipe, prefix with PID, and write to log file
+    (sed "s/^/[PID $pid] /" < "$pipe" > "../logs/log_${i}.log"; rm "$pipe") &
 done
 
 cd ..
@@ -52,6 +65,7 @@ print_failed_log() {
 
             # Only consider non-zero exit codes as failures
             if [ $exit_code -ne 0 ]; then
+                echo "----- Log for process ${pids[$i-1]} (log_${i}.log) -----"
                 cat "$(pwd)/logs/log_${i}.log"
                 echo "Process ${pids[$i-1]} failed with exit code $exit_code. Log file: $(pwd)/logs/log_${i}.log"
                 return 0