From 63867be48a100f171353656ba2d8e8c88328fe7e Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 14 Dec 2025 14:39:16 +0000 Subject: [PATCH] Remove ws dep --- CLAUDE.md | 167 +++++++++++++----- frontend/local-client-cli/Dockerfile | 4 +- frontend/local-client-cli/package.json | 6 +- frontend/local-client-cli/webpack.config.js | 54 +++--- frontend/package-lock.json | 40 ++--- frontend/sync-client/package.json | 3 +- .../safe-filesystem-operations.ts | 2 +- .../src/services/websocket-manager.test.ts | 10 -- .../src/services/websocket-manager.ts | 20 +-- frontend/sync-client/src/sync-client.ts | 1 - frontend/sync-client/webpack.config.js | 9 - 11 files changed, 172 insertions(+), 144 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 75324418..323681d9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Project Overview -VaultLink is a self-hosted Obsidian plugin for real-time collaborative file syncing. The project consists of a Rust-based sync server and a TypeScript frontend with three main components: an Obsidian plugin, a sync client library, and a test client. +VaultLink is a self-hosted Obsidian plugin for real-time collaborative file syncing. The project consists of a Rust-based sync server and a TypeScript frontend with four main components: an Obsidian plugin, a sync client library, a test client, and a standalone CLI client. ## Architecture @@ -13,22 +13,75 @@ VaultLink is a self-hosted Obsidian plugin for real-time collaborative file sync - **sync-server/**: Rust-based WebSocket server with SQLite database for document versioning and real-time synchronization - **frontend/sync-client/**: TypeScript library providing core sync functionality, WebSocket management, and file operations - **frontend/obsidian-plugin/**: Obsidian plugin that integrates the sync client with Obsidian's API -- **frontend/test-client/**: CLI testing tool for the sync functionality +- **frontend/test-client/**: CLI testing tool for simulating multiple concurrent users +- **frontend/local-client-cli/**: Standalone CLI for VaultLink sync client ### Key Technologies - **Backend**: Rust with Axum framework, SQLite with SQLx, WebSockets for real-time sync -- **Frontend**: TypeScript, Webpack for bundling, Jest for testing +- **Frontend**: TypeScript, Webpack for bundling, Node.js native test runner - **Sync Algorithm**: Uses reconcile-text library for operational transformation +### Architectural Patterns + +**Server Architecture:** + +- `AppState`: Central state container holding `Database`, `Cursors`, and `Broadcasts` +- `Database`: SQLite-backed document versioning with SQLx for compile-time query verification +- `Broadcasts`: WebSocket broadcast system for real-time updates to connected clients +- `Cursors`: Tracks user cursor positions across documents with background cleanup task + +**Client Architecture:** + +- `SyncClient`: Main entry point, orchestrates all sync operations +- `SyncService`: HTTP API client for CRUD operations on documents +- `WebSocketManager`: Manages WebSocket connection and real-time updates +- `Syncer`: Coordinates file synchronization between local filesystem and server +- `CursorTracker`: Manages local and remote cursor positions +- `Database`: Client-side document metadata cache +- `FileOperations`: Abstraction layer for filesystem operations + +**Dual-Bundle Strategy:** +The sync-client builds two separate bundles: + +- `sync-client.web.js`: Browser-compatible UMD bundle (excludes `ws` package) +- `sync-client.node.js`: Node.js CommonJS bundle with WebSocket support + ## Development Commands +### Initial Setup + +**Node.js (requires version 25):** + +```bash +curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash +nvm install 25 +nvm use 25 +nvm alias default 25 # Optional: set as system default +``` + +**Rust:** + +```bash +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh +cargo install sqlx-cli cargo-machete cargo-edit cargo-insta +``` + +**Frontend:** + +```bash +cd frontend +npm install +``` + ### Server Development ```bash cd sync-server cargo run config-e2e.yml # Start development server -cargo test --verbose # Run Rust tests +cargo test --verbose # Run all Rust tests +cargo test # Run specific test cargo clippy --all-targets --all-features # Lint Rust code cargo clippy --all-targets --all-features --fix --allow-dirty --allow-staged # Auto-fix clippy warnings cargo fmt --all -- --check # Check Rust formatting @@ -42,34 +95,35 @@ cargo machete --with-metadata # Detect unused dependencies cd frontend npm run dev # Start development mode (watches sync-client and obsidian-plugin) npm run build # Build all workspaces -npm run test # Run all tests -npm run lint # Lint and format TypeScript code +npm run build -w sync-client # Build specific workspace +npm run test # Run all tests across all workspaces +npm run test -w sync-client # Run tests for specific workspace +npm run lint # Lint and format TypeScript code with ESLint + Prettier ``` -### Database Setup (Development) +### Database Operations ```bash cd sync-server +# Create/reset database for development +rm -rf db.sqlite* sqlx database create --database-url sqlite://db.sqlite3 sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3 cargo sqlx prepare --workspace + +# Add new migration +sqlx migrate add --source src/app_state/database/migrations +sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3 ``` -### Initial Setup +### Project Scripts -```bash -# Install required cargo tools -cargo install sqlx-cli cargo-machete cargo-edit -``` - -### Scripts - -- `scripts/check.sh`: Full CI check (builds, lints, tests both server and frontend) +- `scripts/check.sh`: Full CI check (builds, lints, tests both server and frontend). **Run before pushing.** - `scripts/check.sh --fix`: Same as above but auto-fixes linting and formatting issues -- `scripts/e2e.sh`: End-to-end testing +- `scripts/e2e.sh`: End-to-end testing (e.g., `scripts/e2e.sh 8` for 8 concurrent clients) - `scripts/clean-up.sh`: Clean logs and database files -- `scripts/bump-version.sh patch`: Publish new version -- `scripts/update-api-types.sh`: Update TypeScript bindings from Rust types +- `scripts/bump-version.sh patch`: Publish new version (options: patch, minor, major) +- `scripts/update-api-types.sh`: Update TypeScript bindings from Rust types (uses ts-rs) ## Code Structure @@ -77,47 +131,78 @@ cargo install sqlx-cli cargo-machete cargo-edit The frontend uses npm workspaces with four packages: -- `sync-client`: Core synchronization logic +- `sync-client`: Core synchronization logic (builds dual bundles for web and Node.js) - `obsidian-plugin`: Obsidian-specific integration -- `test-client`: Testing utilities +- `test-client`: Testing utilities for E2E tests - `local-client-cli`: Standalone CLI for VaultLink sync client -### Type Generation +### Type Generation and API Updates -Rust structs generate TypeScript types via ts-rs crate, stored in `sync-server/bindings/` and used by frontend packages. +Rust structs generate TypeScript types via ts-rs crate: -### Key Files +1. Rust structs annotated with `#[derive(TS)]` export to `sync-server/bindings/` +2. Run `scripts/update-api-types.sh` to copy bindings to `frontend/sync-client/src/services/types/` +3. Frontend imports these types for type-safe API communication -- `sync-server/src/`: Rust server implementation with WebSocket handlers -- `frontend/sync-client/src/sync-client.ts`: Main sync client entry point -- `frontend/obsidian-plugin/src/vault-link-plugin.ts`: Main Obsidian plugin class -- `frontend/sync-client/src/services/sync-service.ts`: Core synchronization logic +### Important Implementation Details + +**SQLx Compile-Time Verification:** + +- SQLx verifies SQL queries at compile time against the database schema +- Run `cargo sqlx prepare --workspace` after schema changes to update `.sqlx/` directory +- CI builds require prepared query metadata to avoid needing a live database ## Testing ### Running Tests -- Server: `cargo test --verbose` -- Frontend: `npm run test` (runs Jest across all workspaces) -- E2E: `scripts/e2e.sh` +**Server:** + +```bash +cargo test --verbose # All tests +cargo test # Specific test +``` + +**Frontend:** + +```bash +npm run test # All workspaces +npm run test -w sync-client # Specific workspace +``` + +**E2E:** + +```bash +scripts/e2e.sh 8 # 8 concurrent clients +scripts/clean-up.sh # Clean up after tests +``` ### Test Structure -- Rust: Unit tests alongside source files -- TypeScript: `.test.ts` files using Jest -- E2E: Uses test-client to simulate multiple concurrent users +- **Rust**: Unit tests alongside source files, uses `cargo-insta` for snapshot testing +- **TypeScript**: `.test.ts` files using Node.js native test runner (not Jest) +- **E2E**: Uses `test-client` to simulate multiple concurrent users with random operations -## Code Style +## Code Style and Formatting ### Rust -- Uses extensive Clippy lints (see Cargo.toml) -- Follows pedantic linting rules +- Extensive Clippy lints (see `Cargo.toml`) +- Pedantic linting rules enabled - Forbids unsafe code -- Uses cargo fmt with default settings +- Uses `rustfmt.toml` for formatting configuration (4 spaces, Unix line endings) +- Run `cargo fmt --all` to format ### TypeScript -- Prettier configuration: 4-space tabs, trailing commas removed, LF line endings -- ESLint with unused imports plugin -- Consistent across all three frontend packages +- **Prettier**: 4-space indentation, no trailing commas, LF line endings +- **YAML/Markdown override**: 2-space indentation (via prettier config) +- **ESLint**: Strict rules with unused imports detection +- Configuration in `frontend/package.json` +- Run `npm run lint` to format and fix issues + +### EditorConfig + +- `.editorconfig` at project root defines baseline formatting rules +- `rustfmt.toml` and Prettier config explicitly mirror these settings +- Both formatters enforce: 4-space indent (2 for YAML/MD), LF endings, final newline, trim trailing whitespace diff --git a/frontend/local-client-cli/Dockerfile b/frontend/local-client-cli/Dockerfile index 695ab587..0dfa7055 100644 --- a/frontend/local-client-cli/Dockerfile +++ b/frontend/local-client-cli/Dockerfile @@ -1,4 +1,4 @@ -FROM node:22-slim AS builder +FROM node:25-slim AS builder WORKDIR /build @@ -7,7 +7,7 @@ COPY . . RUN npm ci RUN npm run build -FROM node:22-alpine +FROM node:25-alpine LABEL org.opencontainers.image.title="VaultLink Local CLI" LABEL org.opencontainers.image.description="Standalone CLI for VaultLink sync client" diff --git a/frontend/local-client-cli/package.json b/frontend/local-client-cli/package.json index 0cd7da7b..cc14eaa4 100644 --- a/frontend/local-client-cli/package.json +++ b/frontend/local-client-cli/package.json @@ -11,11 +11,9 @@ "build": "webpack --mode production", "test": "tsx --test 'src/**/*.test.ts'" }, - "dependencies": { - "commander": "^14.0.2", - "watcher": "^2.3.1" - }, "devDependencies": { + "commander": "^14.0.2", + "watcher": "^2.3.1", "@types/node": "^25.0.2", "sync-client": "file:../sync-client", "ts-loader": "^9.5.4", diff --git a/frontend/local-client-cli/webpack.config.js b/frontend/local-client-cli/webpack.config.js index f8f48534..9226b9dc 100644 --- a/frontend/local-client-cli/webpack.config.js +++ b/frontend/local-client-cli/webpack.config.js @@ -2,32 +2,32 @@ const path = require("path"); const webpack = require("webpack"); module.exports = { - entry: { - cli: "./src/cli.ts", - healthcheck: "./src/healthcheck.ts" - }, - target: "node", - mode: "production", - optimization: { - minimize: false - }, - module: { - rules: [ - { - test: /\.ts$/, - use: "ts-loader" - } - ] - }, - resolve: { - extensions: [".ts", ".js"] - }, - output: { - globalObject: "this", - filename: "[name].js", - path: path.resolve(__dirname, "dist") - }, - plugins: [ - new webpack.BannerPlugin({ banner: "#!/usr/bin/env node", raw: true }) + entry: { + cli: "./src/cli.ts", + healthcheck: "./src/healthcheck.ts" + }, + target: "node", + mode: "production", + optimization: { + minimize: false + }, + module: { + rules: [ + { + test: /\.ts$/, + use: "ts-loader" + } ] + }, + resolve: { + extensions: [".ts", ".js"] + }, + output: { + globalObject: "this", + filename: "[name].js", + path: path.resolve(__dirname, "dist") + }, + plugins: [ + new webpack.BannerPlugin({ banner: "#!/usr/bin/env node", raw: true }) + ] }; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 0fc0ce40..52d4d304 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -22,20 +22,18 @@ }, "local-client-cli": { "version": "0.13.1", - "dependencies": { - "commander": "^14.0.2", - "watcher": "^2.3.1" - }, "bin": { "vaultlink": "dist/cli.js" }, "devDependencies": { "@types/node": "^25.0.2", + "commander": "^14.0.2", "sync-client": "file:../sync-client", "ts-loader": "^9.5.4", "tslib": "2.8.1", "tsx": "^4.21.0", "typescript": "5.9.3", + "watcher": "^2.3.1", "webpack": "^5.103.0", "webpack-cli": "^6.0.1" } @@ -1735,6 +1733,7 @@ "version": "14.0.2", "resolved": "https://registry.npmjs.org/commander/-/commander-14.0.2.tgz", "integrity": "sha512-TywoWNNRbhoD0BXs1P3ZEScW8W5iKrnbithIl0YH+uCmBd0QpPOA8yc82DS3BIE5Ma6FnBVUsJ7wVUDz4dvOWQ==", + "dev": true, "license": "MIT", "engines": { "node": ">=20" @@ -1888,6 +1887,7 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/dettle/-/dettle-1.0.5.tgz", "integrity": "sha512-ZVyjhAJ7sCe1PNXEGveObOH9AC8QvMga3HJIghHawtG7mE4K5pW9nz/vDGAr/U7a3LWgdOzEE7ac9MURnyfaTA==", + "dev": true, "license": "MIT" }, "node_modules/dunder-proto": { @@ -3291,6 +3291,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/promise-make-counter/-/promise-make-counter-1.0.2.tgz", "integrity": "sha512-FJAxTBWQuQoAs4ZOYuKX1FHXxEgKLEzBxUvwr4RoOglkTpOjWuM+RXsK3M9q5lMa8kjqctUrhwYeZFT4ygsnag==", + "dev": true, "license": "MIT", "dependencies": { "promise-make-naked": "^3.0.2" @@ -3300,6 +3301,7 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/promise-make-naked/-/promise-make-naked-3.0.2.tgz", "integrity": "sha512-B+b+kQ1YrYS7zO7P7bQcoqqMUizP06BOyNSBEnB5VJKDSWo8fsVuDkfSmwdjF0JsRtaNh83so5MMFJ95soH5jg==", + "dev": true, "license": "MIT" }, "node_modules/punycode": { @@ -3744,7 +3746,8 @@ "node_modules/stubborn-fs": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/stubborn-fs/-/stubborn-fs-1.2.5.tgz", - "integrity": "sha512-H2N9c26eXjzL/S/K+i/RHHcFanE74dptvvjM8iwzwbVcWY/zjBbgRqF3K0DY4+OD+uTTASTBvDoxPDaPN02D7g==" + "integrity": "sha512-H2N9c26eXjzL/S/K+i/RHHcFanE74dptvvjM8iwzwbVcWY/zjBbgRqF3K0DY4+OD+uTTASTBvDoxPDaPN02D7g==", + "dev": true }, "node_modules/style-mod": { "version": "4.1.3", @@ -3933,6 +3936,7 @@ "version": "2.7.4", "resolved": "https://registry.npmjs.org/tiny-readdir/-/tiny-readdir-2.7.4.tgz", "integrity": "sha512-721U+zsYwDirjr8IM6jqpesD/McpZooeFi3Zc6mcjy1pse2C+v19eHPFRqz4chGXZFw7C3KITDjAtHETc2wj7Q==", + "dev": true, "license": "MIT", "dependencies": { "promise-make-counter": "^1.0.2" @@ -4218,6 +4222,7 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/watcher/-/watcher-2.3.1.tgz", "integrity": "sha512-d3yl+ey35h05r5EFP0TafE2jsmQUJ9cc2aernRVyAkZiu0J3+3TbNugNcqdUJDoWOfL2p+bNsN427stsBC/HnA==", + "dev": true, "dependencies": { "dettle": "^1.0.2", "stubborn-fs": "^1.2.5", @@ -4480,28 +4485,6 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, - "node_modules/ws": { - "version": "8.18.3", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz", - "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, "node_modules/y18n": { "version": "5.0.8", "dev": true, @@ -4616,8 +4599,7 @@ "uuid": "^13.0.0", "webpack": "^5.103.0", "webpack-cli": "^6.0.1", - "webpack-merge": "^6.0.1", - "ws": "^8.18.3" + "webpack-merge": "^6.0.1" } }, "sync-client/node_modules/minimatch": { diff --git a/frontend/sync-client/package.json b/frontend/sync-client/package.json index a106d870..c3d86efb 100644 --- a/frontend/sync-client/package.json +++ b/frontend/sync-client/package.json @@ -26,7 +26,6 @@ "webpack": "^5.103.0", "webpack-cli": "^6.0.1", "webpack-merge": "^6.0.1", - "@sentry/browser": "^10.30.0", - "ws": "^8.18.3" + "@sentry/browser": "^10.30.0" } } diff --git a/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts b/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts index a3df4ea5..fc0a1ed5 100644 --- a/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts +++ b/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts @@ -2,7 +2,7 @@ import type { RelativePath } from "../persistence/database"; import type { FileSystemOperations } from "./filesystem-operations"; import type { Logger } from "../tracing/logger"; import { Locks } from "../utils/data-structures/locks"; -import { FileNotFoundError } from "./file-not-found-error"; +import { FileNotFoundError } from "../errors/file-not-found-error"; import type { TextWithCursors } from "reconcile-text"; /** diff --git a/frontend/sync-client/src/services/websocket-manager.test.ts b/frontend/sync-client/src/services/websocket-manager.test.ts index fef901e7..3b61b5a1 100644 --- a/frontend/sync-client/src/services/websocket-manager.test.ts +++ b/frontend/sync-client/src/services/websocket-manager.test.ts @@ -4,8 +4,6 @@ import assert from "node:assert"; import { WebSocketManager } from "./websocket-manager"; import type { Logger } from "../tracing/logger"; import type { Settings } from "../persistence/settings"; -// eslint-disable-next-line @typescript-eslint/no-require-imports -const WebSocket = require("ws") as typeof globalThis.WebSocket; class MockCloseEvent extends Event { public code: number; @@ -91,10 +89,8 @@ function createMockFn unknown>( describe("WebSocketManager", () => { let mockLogger: Logger = undefined as unknown as Logger; let mockSettings: Settings = undefined as unknown as Settings; - let deviceId = "test-device-123"; beforeEach(() => { - deviceId = "test-device-123"; const noop = (): void => { // Intentionally empty for mock }; @@ -116,7 +112,6 @@ describe("WebSocketManager", () => { it("cleans up promises after message handling", async () => { const manager = new WebSocketManager( - deviceId, mockLogger, mockSettings, MockWebSocket as unknown as typeof WebSocket @@ -146,7 +141,6 @@ describe("WebSocketManager", () => { it("cleans up cursor position promises", async () => { const manager = new WebSocketManager( - deviceId, mockLogger, mockSettings, MockWebSocket as unknown as typeof WebSocket @@ -176,7 +170,6 @@ describe("WebSocketManager", () => { it("logs handshake send errors", async () => { const manager = new WebSocketManager( - deviceId, mockLogger, mockSettings, MockWebSocket as unknown as typeof WebSocket @@ -205,7 +198,6 @@ describe("WebSocketManager", () => { it("completes stop with timeout protection", async () => { const manager = new WebSocketManager( - deviceId, mockLogger, mockSettings, MockWebSocket as unknown as typeof WebSocket @@ -220,7 +212,6 @@ describe("WebSocketManager", () => { it("clears old handlers on reconnection", async () => { const manager = new WebSocketManager( - deviceId, mockLogger, mockSettings, MockWebSocket as unknown as typeof WebSocket @@ -257,7 +248,6 @@ describe("WebSocketManager", () => { it("tracks message handling promises", async () => { const manager = new WebSocketManager( - deviceId, mockLogger, mockSettings, MockWebSocket as unknown as typeof WebSocket diff --git a/frontend/sync-client/src/services/websocket-manager.ts b/frontend/sync-client/src/services/websocket-manager.ts index 09787bce..4f47fcbe 100644 --- a/frontend/sync-client/src/services/websocket-manager.ts +++ b/frontend/sync-client/src/services/websocket-manager.ts @@ -31,28 +31,12 @@ export class WebSocketManager { private readonly outstandingPromises: Promise[] = []; private webSocket: WebSocket | undefined; - private readonly webSocketFactoryImplementation: typeof globalThis.WebSocket; public constructor( - private readonly deviceId: string, private readonly logger: Logger, private readonly settings: Settings, - webSocketImplementation?: typeof globalThis.WebSocket - ) { - if (webSocketImplementation) { - this.webSocketFactoryImplementation = webSocketImplementation; - } else { - if ( - typeof globalThis !== "undefined" && - typeof globalThis.WebSocket === "undefined" - ) { - // eslint-disable-next-line - this.webSocketFactoryImplementation = require("ws"); // polyfill for WebSocket in Node.js - } else { - this.webSocketFactoryImplementation = WebSocket; - } - } - } + private readonly webSocketFactoryImplementation: typeof globalThis.WebSocket = WebSocket + ) {} public get isWebSocketConnected(): boolean { return ( diff --git a/frontend/sync-client/src/sync-client.ts b/frontend/sync-client/src/sync-client.ts index 42d70cac..5c427d7e 100644 --- a/frontend/sync-client/src/sync-client.ts +++ b/frontend/sync-client/src/sync-client.ts @@ -195,7 +195,6 @@ export class SyncClient { ); const webSocketManager = new WebSocketManager( - deviceId, logger, settings, webSocket diff --git a/frontend/sync-client/webpack.config.js b/frontend/sync-client/webpack.config.js index b7c3a3fd..413bfeba 100644 --- a/frontend/sync-client/webpack.config.js +++ b/frontend/sync-client/webpack.config.js @@ -49,11 +49,6 @@ module.exports = [ type: "umd" }, globalObject: "this" - }, - resolve: { - fallback: { - ws: false // Exclude `ws` from the browser bundle - } } }), merge(common, { @@ -62,10 +57,6 @@ module.exports = [ path: path.resolve(__dirname, "dist"), filename: "sync-client.node.js", libraryTarget: "commonjs2" - }, - externals: { - bufferutil: "bufferutil", - "utf-8-validate": "utf-8-validate" // required for ws: https://github.com/websockets/ws/issues/2245#issuecomment-2250318733 } }) ];