Compare commits

..

165 commits
0.9.2 ... main

Author SHA1 Message Date
d99e249fa5 Durable rename
Some checks failed
Check / build (pull_request) Has been cancelled
E2E tests / build (pull_request) Has been cancelled
Publish CLI / publish-docker (pull_request) Has been cancelled
Publish server Docker image / publish-docker (pull_request) Has been cancelled
Check / build (push) Has been cancelled
E2E tests / build (push) Has been cancelled
Publish CLI / publish-docker (push) Has been cancelled
Publish server Docker image / publish-docker (push) Has been cancelled
2026-05-09 14:20:36 +01:00
6647a4e632 Improvements 2026-05-09 14:17:52 +01:00
201f9aeaee Remove clutter 2026-05-09 13:46:48 +01:00
682dc74497 Update local-client-cli and obsidian-plugin
Some checks failed
Check / build (pull_request) Has been cancelled
E2E tests / build (pull_request) Has been cancelled
Publish CLI / publish-docker (pull_request) Has been cancelled
Publish server Docker image / publish-docker (pull_request) Has been cancelled
Pulls the local-client-cli and obsidian-plugin changes from
asch/fix-everything onto a fresh branch off main.
2026-05-09 13:41:51 +01:00
40fbd42b92 Remove GH actions (#192)
Some checks are pending
Check / build (push) Waiting to run
E2E tests / build (push) Waiting to run
Publish CLI / publish-docker (push) Waiting to run
Publish server Docker image / publish-docker (push) Waiting to run
Reviewed-on: https://home.schmelczer.dev/git/git/andras/vault-link/pulls/192
Co-authored-by: Andras Schmelczer <andras@schmelczer.dev>
Co-committed-by: Andras Schmelczer <andras@schmelczer.dev>
2026-05-09 11:17:21 +01:00
0e3132f96c Add deterministic-tests (#190)
Some checks are pending
Check / build (push) Waiting to run
E2E tests / build (push) Waiting to run
Publish CLI / publish-docker (push) Waiting to run
Publish server Docker image / publish-docker (push) Waiting to run
Reviewed-on: https://home.schmelczer.dev/git/git/andras/vault-link/pulls/190
Co-authored-by: Andras Schmelczer <andras@schmelczer.dev>
Co-committed-by: Andras Schmelczer <andras@schmelczer.dev>
2026-05-09 10:15:21 +01:00
4482e0155f Migrate to forgejo & reformat (#189)
Some checks failed
Check / build (push) Waiting to run
E2E tests / build (push) Waiting to run
Publish CLI / publish-docker (push) Waiting to run
Publish server Docker image / publish-docker (push) Waiting to run
Deploy Documentation / build (push) Has been cancelled
- Migrate to forgejo
- Bump Rust & Node
- Reformat project
- Small script cleanup

Reviewed-on: https://home.schmelczer.dev/git/git/andras/vault-link/pulls/189
Co-authored-by: Andras Schmelczer <andras@schmelczer.dev>
Co-committed-by: Andras Schmelczer <andras@schmelczer.dev>
2026-05-08 21:53:33 +01:00
9a75569e83 Bump versions to 0.14.0 2025-12-14 23:31:40 +00:00
5efe30d9d6 Format & lint 2025-12-14 17:19:25 +00:00
0e0a85df82 Check node version 2025-12-14 17:19:25 +00:00
42a77a5cd5 Upload logs instead of printing them 2025-12-14 17:19:25 +00:00
4fb3839b3e Add lock tests 2025-12-14 17:19:25 +00:00
7daa363723 Unsubscribe in SyncClient 2025-12-14 17:19:25 +00:00
47f24e168b Wait for idle instead 2025-12-14 17:19:25 +00:00
b6ab01d56a Handle websocket race condition 2025-12-14 17:19:25 +00:00
580c993071 Reject pending locks on reset 2025-12-14 17:19:25 +00:00
299c3baea9 Don't publish PRs 2025-12-14 17:19:25 +00:00
1b71f3e780 Always kill server 2025-12-14 17:19:25 +00:00
8aba8ee44a Extract const 2025-12-14 17:19:25 +00:00
079cd26faa Bump versions to 0.13.1 2025-12-11 22:10:21 +00:00
f6dccc4492 Try fixing E2E tests more 2025-12-11 22:08:48 +00:00
387e7afd58 Allow-list error type 2025-12-10 23:14:50 +00:00
056fb96ce8 chmod +x 2025-12-10 22:35:44 +00:00
9ac7fdbeb7
Improve CI (#181) 2025-12-10 22:03:13 +00:00
8e4ac3a26a Fix manifests 2025-12-08 20:11:56 +00:00
e2b24725ef Bump versions to 0.13.0 2025-12-07 19:29:15 +00:00
2db49da654 Fix cron 2025-12-07 16:42:23 +00:00
dbc63fcecd Once an hour 2025-12-07 16:42:23 +00:00
ce6d44f26b Add log line 2025-12-07 16:42:23 +00:00
6608804d34 Refactor & lint 2025-12-07 16:42:23 +00:00
e47d8a8179 Fix file watching 2025-12-07 16:42:23 +00:00
e9252955b4 Align prettier & editorconfig 2025-12-07 16:42:23 +00:00
570c41299b Create vault dir if doesn't exist 2025-12-07 16:42:23 +00:00
78a706ab8d Move log level to config file 2025-12-07 16:42:23 +00:00
8439bd8b92 Delete temp folder before test 2025-12-07 16:42:23 +00:00
504ddb6ff6 Pick up new events API 2025-12-07 16:42:23 +00:00
0a5bbbf20e Fix and apply editorconfig 2025-12-07 16:42:23 +00:00
b05e415acf Apply editorconfig 2025-12-07 16:42:23 +00:00
ad3191957a Add event handler class 2025-12-07 16:42:23 +00:00
1ed22c72d7 Enforce editorconfig 2025-12-07 16:42:23 +00:00
e6bfefd2d5 Fix file creation deduplication 2025-12-07 16:42:23 +00:00
9e06d99512 Run all tests 2025-12-07 16:42:23 +00:00
3f2ecfb0b6 Use efficient filters 2025-12-07 16:42:23 +00:00
07cb8491e2 Bump versions to 0.12.0 2025-12-06 22:21:55 +00:00
aca1ca50a4 Update reconcile to 0.8.0 2025-12-06 22:20:31 +00:00
2885026d2f Remove serde_with and use human serde instead 2025-12-06 22:14:20 +00:00
e6f7543114 Fix broken endpoint 2025-12-06 22:01:01 +00:00
d979963f86 Fix http error handling in the client service 2025-12-06 22:00:54 +00:00
ea603f83fd Fix HTTP method of the server 2025-12-06 21:25:30 +00:00
66e2fb3768 Fix docs publishing 2025-12-06 21:16:12 +00:00
a1bda41646 Always fetch the right document version content 2025-12-06 11:44:57 +00:00
bfe3e9aeeb Merge branch 'asch/fix-tests' 2025-12-06 10:53:20 +00:00
5238d85181 Print more details 2025-12-06 10:52:46 +00:00
1646f74633 More frequent tests 2025-12-06 10:49:30 +00:00
7a13cb57ce
Investigate deadlock (#178) 2025-12-05 22:34:14 +00:00
77e0bb4caf Await all 2025-12-05 22:33:33 +00:00
e8d86c737b More logs 2025-12-05 22:29:46 +00:00
a5e128efcd Lint 2025-12-05 21:48:35 +00:00
8adb8841ef Investigate dead-lock 2025-12-05 21:42:34 +00:00
564d4a6c37 Lint 2025-12-03 23:24:53 +00:00
2607bc5213 Run E2E more often 2025-12-03 23:18:16 +00:00
8ef2f8c132 Escape vault name 2025-12-03 23:18:16 +00:00
dependabot[bot]
d39a91b447
Bump tsx from 4.20.5 to 4.20.6 in /frontend (#154)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-12-03 22:32:58 +00:00
dependabot[bot]
da2237fa68
Bump sass-loader from 16.0.5 to 16.0.6 in /frontend (#159)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-12-03 22:32:49 +00:00
dependabot[bot]
e98f7acefa
Bump log from 0.4.27 to 0.4.28 in /sync-server (#170)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-12-03 22:32:39 +00:00
8e336cb0f3 Bump versions to 0.11.1 2025-12-02 20:46:41 +00:00
f8d62f4416 Force install 2025-12-02 20:44:50 +00:00
89e3b61766 Fix release 2025-12-02 20:44:20 +00:00
215c024876 Bump versions to 0.11.0 2025-11-30 15:26:40 +00:00
bbf81d3111 Install set-version 2025-11-30 15:26:22 +00:00
9349afc00f Run lint & fmt 2025-11-30 15:24:52 +00:00
c7c96b787a Add log lines 2025-11-30 15:24:52 +00:00
7beda491e9 Rename method 2025-11-30 15:24:52 +00:00
515a8f2bf4 Install cargo machete 2025-11-30 15:24:52 +00:00
39860f7f04 Add lock on settings 2025-11-30 15:24:52 +00:00
3517af1461 Disallow changing settings while applying previous changes 2025-11-30 15:24:52 +00:00
89565e23f3 Log deduping 2025-11-30 15:24:52 +00:00
d07fa32ba3 Format 2025-11-30 15:24:52 +00:00
10bde4bc3a Fix race condition of client-side path deconflicting 2025-11-30 15:24:52 +00:00
952e89343a Don't broadcast without clients 2025-11-30 15:24:52 +00:00
2ce5faea92 Ignore ds store 2025-11-30 15:24:52 +00:00
b595a060a7 Await settings event handlers 2025-11-30 15:24:52 +00:00
5905aa37b9 Add copy to clipboard button 2025-11-30 15:24:52 +00:00
5417c1ddd0 Small clean up 2025-11-30 15:24:52 +00:00
84f077f36b Improve logging 2025-11-30 15:24:52 +00:00
4456767ec4 Clean up 2025-11-30 15:24:52 +00:00
e635e84aa4 Close unsued databases 2025-11-30 15:24:52 +00:00
10fdc938c5 Add error on duplicate plugin load 2025-11-30 15:24:52 +00:00
91f49d6997 Decrease parallelism 2025-11-30 15:24:52 +00:00
7a95d9f0a8 Use named group 2025-11-30 15:24:52 +00:00
e53482ced8 Make skipped file a warning 2025-11-30 15:24:52 +00:00
67c912ae4c Format 2025-11-30 15:24:52 +00:00
b0b5da7d37 Remove frequent popups 2025-11-30 15:24:52 +00:00
13f5456b39 Fix race condition 2025-11-30 15:24:52 +00:00
c10b6435d4 Don't download all documents when initial sync gets interrupted 2025-11-30 15:24:52 +00:00
476588a63b Don't print success twice 2025-11-30 15:24:52 +00:00
9d60ec14dd Improve API 2025-11-30 15:24:52 +00:00
d45d2c0be3 Fix E2E testing 2025-11-30 15:24:52 +00:00
82f11d8c86 Fix testing logic 2025-11-30 15:24:52 +00:00
c3cc678446 Stop leaking promises in ws manager 2025-11-30 15:24:52 +00:00
3ed2e4f666 Add api version check to client 2025-11-30 15:24:52 +00:00
b1826907e7 Add resetting tests 2025-11-30 15:24:52 +00:00
c3cbde052a Add server config for mergable extensions 2025-11-30 15:24:52 +00:00
7008c54e2e Run check.sh 2025-11-30 15:24:52 +00:00
18be9f4dd8 Fix lint 2025-11-30 15:24:52 +00:00
4b195b070d Expose new advanced settings 2025-11-30 15:24:52 +00:00
c4da1426b1 Fix compile 2025-11-30 15:24:52 +00:00
340c347841 Don't leak promises 2025-11-30 15:24:52 +00:00
c94d732f24 Fix resetting 2025-11-30 15:24:52 +00:00
d8058d396c Add awaitAll 2025-11-30 15:24:52 +00:00
ef4444afc2 Lint 2025-11-30 15:24:52 +00:00
fb2d82a06e Lint 2025-11-30 15:24:52 +00:00
5a0c64d39c Ban bad methods 2025-11-30 15:24:52 +00:00
17fa584ea1 use allSettled 2025-11-30 15:24:52 +00:00
83c15a77c3 Add 2 more settings from consts 2025-11-30 15:24:52 +00:00
3cdd2a4387 Use updated APIs 2025-11-30 15:24:52 +00:00
213a9e18fb Use new WS api 2025-11-30 15:24:52 +00:00
cb2a1c0df1 Fix reset logic for WS 2025-11-30 15:24:52 +00:00
9f1f4beae4 Renamce 2025-11-30 15:24:52 +00:00
12d8d15572 Add fetch controller tests 2025-11-30 15:24:52 +00:00
56c77dc3f6 Fix fetch controller 2025-11-30 15:24:52 +00:00
4186aa9e0c Formatting 2025-11-30 15:24:52 +00:00
71274d466c Extract function 2025-11-30 15:24:52 +00:00
9d645f43f8 Handle move on create 2025-11-30 15:24:52 +00:00
72ad82ab83 Fix dotfile handling 2025-11-30 15:24:52 +00:00
4fcd134e55 Extract consts 2025-11-30 15:24:52 +00:00
aa3c587002 Dedup paths on create document 2025-11-30 15:24:52 +00:00
10fd928459 Fix file operations 2025-11-30 15:24:52 +00:00
91675ea99c Add remove event listener methods 2025-11-30 15:24:52 +00:00
d4b68154df Export consts 2025-11-30 15:24:52 +00:00
c798d96009 Fix import 2025-11-30 15:24:52 +00:00
51baa4d8e0 Have the same error message for file not found 2025-11-30 15:24:52 +00:00
a57ed5c4ae Fix edge cases 2025-11-30 15:24:52 +00:00
c3c2cafde5 Fix +1 2025-11-30 15:24:52 +00:00
f11c8db6d2 Replace all instead of just replace 2025-11-30 15:24:52 +00:00
9c3dedad76 Rename param 2025-11-30 15:24:52 +00:00
d590a2c9c8 Extend 2025-11-30 15:24:52 +00:00
511ac78e6d Don't kill CI with E2E tests 2025-11-30 15:24:52 +00:00
aaeca588fb Enforce british english 2025-11-30 15:24:52 +00:00
1b1b72cb92 Configure dependabot for docs 2025-11-30 15:24:52 +00:00
fbf03c41e0 Refactor plugin setup and avoid dangling resources 2025-11-30 15:24:52 +00:00
a1a4610109 Simplify docs 2025-11-30 15:24:52 +00:00
00d2061627 Update docs 2025-11-30 15:24:52 +00:00
38810579ec Update type imports 2025-11-30 15:24:52 +00:00
ea189f3d09 All sync-client deps are devDeps 2025-11-30 15:24:52 +00:00
fccc66aaea Re-export type 2025-11-30 15:24:52 +00:00
50a95b114d Add docs 2025-11-30 15:24:52 +00:00
56c1f4d58b Restructure packages 2025-11-30 15:24:52 +00:00
dependabot[bot]
812eb7a644
Bump tracing-subscriber from 0.3.19 to 0.3.20 in /sync-server (#146)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-11-22 12:39:48 +00:00
72bae2d93e Bump versions to 0.10.1 2025-11-19 22:40:36 +00:00
e2189d4dbe Use stderr for logging 2025-11-19 22:39:06 +00:00
c08feba0ad
Improve settings (#168)
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-11-19 19:53:10 +00:00
e75298c4f1 Bump versions to 0.10.0 2025-11-16 22:10:42 +00:00
be1635c26e
Improve network usage for small text changes (#166) 2025-11-16 22:10:22 +00:00
dependabot[bot]
1da17c462e
Bump anyhow from 1.0.98 to 1.0.100 in /sync-server (#150)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-11-02 17:54:12 +00:00
dependabot[bot]
29747d0829
Bump commander from 12.1.0 to 14.0.2 in /frontend (#149)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-11-02 17:53:58 +00:00
dependabot[bot]
f1c2c8f846
Bump obsidian from 1.8.7 to 1.10.2 in /frontend (#155)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-11-02 17:53:14 +00:00
dependabot[bot]
04034b85da
Bump regex from 1.11.1 to 1.12.2 in /sync-server (#152)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-11-02 17:53:01 +00:00
dependabot[bot]
c3773a2a7a
Bump tokio from 1.47.1 to 1.48.0 in /sync-server (#151)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-11-02 17:52:52 +00:00
dependabot[bot]
97a4494085
Bump @plausible-analytics/tracker from 0.4.0 to 0.4.3 in /frontend (#145)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-11-02 17:52:36 +00:00
dependabot[bot]
33a24c3a77
Bump serde_with from 3.15.0 to 3.15.1 in /sync-server (#153)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-11-02 17:52:27 +00:00
dependabot[bot]
0a7b8568e8
Bump rust from 1.90-slim-trixie to 1.91-slim-trixie in /sync-server (#156)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-11-02 17:52:18 +00:00
cd57ea6682
Add log rotation to server & UI improvements (#157) 2025-11-02 17:52:04 +00:00
dependabot[bot]
2b9d77d165
Bump eslint from 9.28.0 to 9.38.0 in /frontend (#142)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-10-26 11:02:41 +00:00
dependabot[bot]
b4ff4cbf25
Bump uuid from 11.1.0 to 13.0.0 in /frontend (#143)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-10-26 11:02:30 +00:00
368 changed files with 36062 additions and 13386 deletions

View file

@ -9,6 +9,8 @@ trim_trailing_whitespace = true
charset = utf-8
indent_style = space
indent_size = 4
tab_width = 4
[*.{yml,yaml}]
[*.{yml,yaml,md}]
indent_size = 2
tab_width = 2

View file

@ -5,6 +5,7 @@ on:
branches: ["main"]
pull_request:
branches: ["main"]
workflow_dispatch:
env:
CARGO_TERM_COLOR: always
@ -12,29 +13,23 @@ env:
jobs:
build:
runs-on: self-hosted
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup Node.js environment
uses: actions/setup-node@v4.2.0
uses: actions/setup-node@v4
with:
node-version: "22.x"
check-latest: true
node-version: "25.x"
- name: Setup Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: "1.89.0"
toolchain: "1.92.0"
components: clippy, rustfmt
- name: Setup rust
run: |
cargo install sqlx-cli cargo-machete
cd sync-server
sqlx database create --database-url sqlite://db.sqlite3
sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3
- name: Lint & test
run: scripts/check.sh

View file

@ -0,0 +1,38 @@
name: Deploy Documentation
on:
push:
branches:
- main
paths:
- "docs/**"
- ".forgejo/workflows/deploy-docs.yml"
workflow_dispatch:
concurrency:
group: pages
cancel-in-progress: false
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup Node.js environment
uses: actions/setup-node@v4
with:
node-version: "25.x"
- name: Build docs
run: scripts/build-docs.sh
- name: Upload artifact
uses: actions/upload-artifact@v4
with:
name: docs
path: docs/.vitepress/dist

View file

@ -0,0 +1,71 @@
name: E2E tests
on:
push:
branches: ["main"]
pull_request:
branches: ["main"]
schedule:
- cron: "0 * * * *"
workflow_dispatch:
concurrency:
group: e2e-tests
cancel-in-progress: false
env:
RUSTFLAGS: "-Dwarnings"
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup Node.js environment
uses: actions/setup-node@v4
with:
node-version: "25.x"
- name: Setup Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: "1.92.0"
components: clippy, rustfmt
- name: Setup rust
run: |
which sqlx || cargo install sqlx-cli
cd sync-server
sqlx database create --database-url sqlite://db.sqlite3
sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3
- name: E2E tests
run: |
cd sync-server
cargo run config-e2e.yml --color never &
SERVER_PID=$!
cd ..
scripts/e2e.sh 8
EXIT_CODE=$?
kill $SERVER_PID 2>/dev/null || true
wait $SERVER_PID 2>/dev/null || true
exit $EXIT_CODE
- name: Upload e2e logs
if: always()
uses: actions/upload-artifact@v4
with:
name: e2e-logs
path: logs/
retention-days: 30
- name: Cleanup
if: always()
run: scripts/clean-up.sh

View file

@ -0,0 +1,51 @@
name: Publish CLI
on:
push:
branches: ["main"]
tags: ["*"]
pull_request:
branches: ["main"]
jobs:
publish-docker:
runs-on: ubuntu-docker
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Extract registry hostname
id: registry
run: echo "host=$(echo '${{ github.server_url }}' | sed 's|https\?://||')" >> "$GITHUB_OUTPUT"
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log into container registry
uses: docker/login-action@v3
with:
registry: ${{ steps.registry.outputs.host }}
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract Docker metadata
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ steps.registry.outputs.host }}/${{ github.repository }}-cli
- name: Build and push Docker image
id: build-and-push
uses: docker/build-push-action@v5
with:
context: frontend
file: frontend/local-client-cli/Dockerfile
platforms: linux/amd64
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=registry,ref=${{ steps.registry.outputs.host }}/${{ github.repository }}-cli:buildcache
cache-to: type=registry,ref=${{ steps.registry.outputs.host }}/${{ github.repository }}-cli:buildcache,mode=max

View file

@ -0,0 +1,71 @@
name: Publish Obsidian plugin
on:
push:
tags: ["*"]
env:
CARGO_TERM_COLOR: always
jobs:
publish-plugin:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup Node.js environment
uses: actions/setup-node@v4
with:
node-version: "25.x"
- name: Build plugin
run: |
cd frontend
npm ci
npm run build
- name: Setup Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: "1.92.0"
components: clippy, rustfmt
- name: Install cross-compilation tools
run: |
apt update
apt install -y gcc-aarch64-linux-gnu musl-tools gcc-mingw-w64-x86-64 jq
- name: Build Linux and Windows binaries
run: ./scripts/build-sync-server-binaries.sh
- name: Create release
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SERVER_URL: ${{ github.server_url }}
REPO: ${{ github.repository }}
run: |
tag="${GITHUB_REF#refs/tags/}"
mkdir -p release
cp frontend/obsidian-plugin/dist/* release/
cp sync-server/artifacts/sync-server-* release/
# Create draft release via Forgejo API
RELEASE_ID=$(curl -s -X POST \
"${SERVER_URL}/api/v1/repos/${REPO}/releases" \
-H "Authorization: token ${GITHUB_TOKEN}" \
-H "Content-Type: application/json" \
-d "{\"tag_name\": \"${tag}\", \"name\": \"${tag}\", \"draft\": true}" \
| jq -r '.id')
# Upload release assets
for file in release/*; do
filename=$(basename "$file")
curl -s -X POST \
"${SERVER_URL}/api/v1/repos/${REPO}/releases/${RELEASE_ID}/assets?name=${filename}" \
-H "Authorization: token ${GITHUB_TOKEN}" \
-F "attachment=@${file}"
done

View file

@ -0,0 +1,51 @@
name: Publish server Docker image
on:
push:
branches: ["main"]
tags: ["*"]
pull_request:
branches: ["main"]
jobs:
publish-docker:
runs-on: ubuntu-docker
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Extract registry hostname
id: registry
run: echo "host=$(echo '${{ github.server_url }}' | sed 's|https\?://||')" >> "$GITHUB_OUTPUT"
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log into container registry
if: github.ref_type == 'tag'
uses: docker/login-action@v3
with:
registry: ${{ steps.registry.outputs.host }}
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract Docker metadata
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ steps.registry.outputs.host }}/${{ github.repository }}
- name: Build and push Docker image
id: build-and-push
uses: docker/build-push-action@v5
with:
context: sync-server
platforms: linux/amd64,linux/arm64
push: ${{ github.ref_type == 'tag' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=registry,ref=${{ steps.registry.outputs.host }}/${{ github.repository }}:buildcache
cache-to: type=registry,ref=${{ steps.registry.outputs.host }}/${{ github.repository }}:buildcache,mode=max

View file

@ -1,27 +0,0 @@
# To get started with Dependabot version updates, you'll need to specify which
# package ecosystems to update and where the package manifests are located.
# Please see the documentation for all configuration options:
# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
version: 2
updates:
- package-ecosystem: "npm"
directories: ["/frontend"]
schedule:
interval: "daily"
- package-ecosystem: "docker"
directories: ["**"]
schedule:
interval: "daily"
- package-ecosystem: "cargo"
directories: ["**"]
schedule:
interval: "daily"
# Disable this for security reasons
# - package-ecosystem: "github-actions"
# directories: ["**"]
# schedule:
# interval: "daily"

View file

@ -1,45 +0,0 @@
name: E2E tests
on:
push:
branches: ["main"]
pull_request:
branches: ["main"]
env:
CARGO_TERM_COLOR: always
RUSTFLAGS: "-Dwarnings"
jobs:
build:
runs-on: self-hosted
steps:
- uses: actions/checkout@v4
- name: Setup Node.js environment
uses: actions/setup-node@v4.2.0
with:
node-version: "22.x"
check-latest: true
- name: Setup Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: "1.89.0"
components: clippy, rustfmt
- name: Setup rust
run: |
cargo install sqlx-cli
cd sync-server
sqlx database create --database-url sqlite://db.sqlite3
sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3
- name: E2E tests
run: |
cd sync-server
cargo run config-e2e.yml --color never &
cd ..
scripts/e2e.sh 32

View file

@ -1,64 +0,0 @@
name: Publish CLI
on:
push:
tags: ["*"]
pull_request:
branches: ["main"]
env:
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}-cli
jobs:
publish-docker:
runs-on: self-hosted
permissions:
contents: read
packages: write
id-token: write
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Install cosign
uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 #v3.5.0
with:
cosign-release: "v2.2.4"
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
- name: Log into registry ${{ env.REGISTRY }}
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract Docker metadata
id: meta
uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
- name: Build and push Docker image
id: build-and-push
uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
with:
context: frontend
file: frontend/local-client-cli/Dockerfile
platforms: linux/amd64
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Sign the published Docker image
env:
TAGS: ${{ steps.meta.outputs.tags }}
DIGEST: ${{ steps.build-and-push.outputs.digest }}
run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}

View file

@ -1,57 +0,0 @@
name: Publish Obsidian plugin
on:
push:
tags: ["*"]
env:
CARGO_TERM_COLOR: always
jobs:
publish-plugin:
runs-on: self-hosted
steps:
- uses: actions/checkout@v4
- name: Setup Node.js environment
uses: actions/setup-node@v4.2.0
with:
node-version: "22.x"
check-latest: true
- name: Build plugin
run: |
cd frontend
npm ci
npm run build
- name: Setup Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: "1.89.0"
components: clippy, rustfmt
- name: Install cross-compilation tools
run: |
apt update
apt install -y gcc-aarch64-linux-gnu musl-tools gcc-mingw-w64-x86-64
- name: Build Linux and Windows binaries
run: ./scripts/build-sync-server-binaries.sh
- name: Create release
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
tag="${GITHUB_REF#refs/tags/}"
mkdir -p release
cp frontend/obsidian-plugin/dist/* release/
cp sync-server/artifacts/sync-server-* release/
cd release
gh release create "$tag" \
--title="$tag" \
--draft \
*

View file

@ -1,90 +0,0 @@
name: Publish server Docker image
on:
push:
branches: ["main"]
tags: ["*"]
pull_request:
branches: ["main"]
env:
# Use docker.io for Docker Hub if empty
REGISTRY: ghcr.io
# github.repository as <account>/<repo>
IMAGE_NAME: ${{ github.repository }}
jobs:
publish-docker:
runs-on: self-hosted
permissions:
contents: read
packages: write
# This is used to complete the identity challenge
# with sigstore/fulcio.
id-token: write
steps:
- name: Checkout repository
uses: actions/checkout@v4
# Install the cosign tool
# https://github.com/sigstore/cosign-installer
- name: Install cosign
if: github.ref_type == 'tag'
uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 #v3.5.0
with:
cosign-release: "v2.2.4"
# Set up BuildKit Docker container builder to be able to build
# multi-platform images and export cache
# https://github.com/docker/setup-buildx-action
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
# Login against a Docker registry
# https://github.com/docker/login-action
- name: Log into registry ${{ env.REGISTRY }}
if: github.ref_type == 'tag'
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
# Extract metadata (tags, labels) for Docker
# https://github.com/docker/metadata-action
- name: Extract Docker metadata
id: meta
uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
# Build and push Docker image with Buildx
# https://github.com/docker/build-push-action
- name: Build and push Docker image
id: build-and-push
uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
with:
context: sync-server
platforms: linux/amd64,linux/arm64
push: ${{ github.ref_type == 'tag' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
# Sign the resulting Docker image digest.
# This will only write to the public Rekor transparency log when the Docker
# repository is public to avoid leaking data. If you would like to publish
# transparency data even for private images, pass --force to cosign below.
# https://github.com/sigstore/cosign
- name: Sign the published Docker image
if: ${{ github.ref_type == 'tag' }}
env:
# https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
TAGS: ${{ steps.meta.outputs.tags }}
DIGEST: ${{ steps.build-and-push.outputs.digest }}
# This step uses the identity token to provision an ephemeral certificate
# against the sigstore community Fulcio instance.
run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}

13
.gitignore vendored
View file

@ -4,18 +4,21 @@ node_modules
# Exclude macOS Finder (System Explorer) View States
.DS_Store
# Frontend build folders
frontend/*/dist
sync-server/db.sqlite3*
sync-server/databases
# Rust build folders
sync-server/target
sync-server/artifacts
sync-server/bindings/*.ts
# build folders
sync-server/db.sqlite3*
**/databases
*.log
*.sqlx
target
.task

View file

@ -5,6 +5,6 @@
"**/dist": true,
"**/node_modules": true,
"**/.sqlx": true,
"**/target": true,
},
"**/target": true
}
}

188
CLAUDE.md
View file

@ -2,98 +2,154 @@
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
## Project Overview
## Project shape
VaultLink is a self-hosted Obsidian plugin for real-time collaborative file syncing. The project consists of a Rust-based sync server and a TypeScript frontend with three main components: an Obsidian plugin, a sync client library, and a test client.
VaultLink is a self-hosted Obsidian file-sync system. Two halves of one repo:
## Architecture
- `sync-server/` — Rust (axum + sqlx/SQLite). Source of truth for vault state, broadcasts changes via WebSocket.
- `frontend/` — npm workspaces. The sync engine (`sync-client`) is consumed by an Obsidian plugin, a standalone CLI, a fuzz E2E harness, a scripted determinism harness, and a history UI.
### Core Components
The HTTP/WS API types are generated from Rust (`ts-rs`) and mirrored into the TS workspaces. **Never hand-edit files in `frontend/sync-client/src/services/types/` or `frontend/history-ui/src/lib/types/`** — run `scripts/update-api-types.sh` after changing anything Serde-derived in the server.
- **sync-server/**: Rust-based WebSocket server with SQLite database for document versioning and real-time synchronization
- **frontend/sync-client/**: TypeScript library providing core sync functionality, WebSocket management, and file operations
- **frontend/obsidian-plugin/**: Obsidian plugin that integrates the sync client with Obsidian's API
- **frontend/test-client/**: CLI testing tool for the sync functionality
### Frontend workspaces
### Key Technologies
- `sync-client` — the sync engine; published to consumers via `dist/`. All other TS workspaces depend on it via `file:../sync-client`.
- `obsidian-plugin` — Obsidian plugin built from `sync-client`.
- `local-client-cli` — same engine wrapped as a standalone CLI.
- `history-ui` — vault-history web UI.
- `test-client` — fuzz E2E harness (random ops across N processes).
- `deterministic-tests` — scripted multi-client tests with an in-memory FS, run against a real server.
- **Backend**: Rust with Axum framework, SQLite with SQLx, WebSockets for real-time sync
- **Frontend**: TypeScript, Webpack for bundling, Jest for testing
- **Sync Algorithm**: Uses reconcile-text library for operational transformation
## Common commands
## Development Commands
Pre-push hygiene (formats, lints, runs tests, requires clean git state):
### Server Development
```bash
cd sync-server
cargo run config-e2e.yml # Start development server
cargo test --verbose # Run Rust tests
cargo clippy --all-targets --all-features # Lint Rust code
cargo fmt --all -- --check # Check Rust formatting
```sh
scripts/check.sh --fix
```
### Frontend Development
```bash
Run the fuzz E2E (N parallel processes):
```sh
scripts/e2e.sh 12
# Logs land in logs/log_<i>.log. Clean with scripts/clean-up.sh
```
Run deterministic tests (require a release-built server in `sync-server/target/release/sync_server` — they spawn it themselves):
```sh
cd sync-server && cargo build --release && cd ..
cd frontend
npm run dev # Start development mode (watches sync-client and obsidian-plugin)
npm run build # Build all workspaces
npm run test # Run all tests
npm run lint # Lint and format TypeScript code
npm run build -w sync-client -w deterministic-tests
node deterministic-tests/dist/cli.js # all
node deterministic-tests/dist/cli.js --filter=rename # subset
node deterministic-tests/dist/cli.js --filter=… -j 4 # cap parallelism
```
### Database Setup (Development)
```bash
Run a single sync-client unit test by file:
```sh
cd frontend/sync-client && npx tsx --test 'src/**/sync-event-queue.test.ts'
```
Server: dev runs from `sync-server/` against `config-e2e.yml`:
```sh
cd sync-server
cargo run config-e2e.yml # dev
cargo build --release # used by both e2e harnesses
cargo test # unit + ts-rs binding export tests
```
Frontend dev (sync-client + obsidian-plugin watch in parallel):
```sh
cd frontend && npm install && npm run dev
```
Regenerate TS bindings from Rust types (touches `frontend/{sync-client,history-ui}/src/.../types/`):
```sh
scripts/update-api-types.sh
```
## SQLite / sqlx
The server uses `sqlx::query!` macros that need a prepared `.sqlx` cache to compile offline. Touching any SQL means regenerating it:
```sh
cd sync-server
sqlx database create --database-url sqlite://db.sqlite3
sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3
cargo sqlx prepare --workspace
```
### Scripts
- `scripts/check.sh`: Full CI check (builds, lints, tests both server and frontend)
- `scripts/e2e.sh`: End-to-end testing
- `scripts/clean-up.sh`: Clean logs and database files
- `scripts/bump-version.sh patch`: Publish new version
- `scripts/update-api-types.sh`: Update TypeScript bindings from Rust types
New migrations: `sqlx migrate add --source src/app_state/database/migrations <name>`.
## Code Structure
## Sync engine architecture
### Workspace Configuration
The frontend uses npm workspaces with three packages:
- `sync-client`: Core synchronization logic
- `obsidian-plugin`: Obsidian-specific integration
- `test-client`: Testing utilities
Read `frontend/sync-client/src/sync-operations/` to follow the sync engine; the rest of `sync-client` is plumbing (filesystem ops, persistence, services, telemetry).
### Type Generation
Rust structs generate TypeScript types via ts-rs crate, stored in `sync-server/bindings/` and used by frontend packages.
The engine is **two independent loops with separate invariants**:
### Key Files
- `sync-server/src/`: Rust server implementation with WebSocket handlers
- `frontend/sync-client/src/sync-client.ts`: Main sync client entry point
- `frontend/obsidian-plugin/src/vault-link-plugin.ts`: Main Obsidian plugin class
- `frontend/sync-client/src/services/sync-service.ts`: Core synchronization logic
- **Wire loop** (`syncer.ts`) — drains the single-consumer FIFO queue. HTTP and WS handlers update record fields (`remoteRelativePath`, `parentVersionId`, `remoteHash`) and write content to the file at `record.localPath`. They never move files for path placement.
- **Path reconciler** (`reconciler.ts`) — runs after every drained event. Best-effort pass that moves files to make `localPath === remoteRelativePath`. The move graph is topologically sorted; cycles are resolved by reading every file in the cycle into memory and writing each back to its new slot (no tmp files). Records with pending local events are skipped on each pass — the reconciler operates only on settled records. Failures (slot occupied by an untracked file, etc.) are silent skips; the next pass retries.
## Testing
**`SyncEventQueue`** (`sync-event-queue.ts`) holds:
### Running Tests
- Server: `cargo test --verbose`
- Frontend: `npm run test` (runs Jest across all workspaces)
- E2E: `scripts/e2e.sh`
- `byDocId: Map<DocumentId, DocumentRecord>` — primary record store.
- `byLocalPath: Map<RelativePath, DocumentRecord>` — derived index for path lookups, maintained at every mutation point.
- `events: SyncEvent[]` — pending wire ops in FIFO drain order.
### Test Structure
- Rust: Unit tests alongside source files
- TypeScript: `.test.ts` files using Jest
- E2E: Uses test-client to simulate multiple concurrent users
```ts
DocumentRecord = {
documentId,
parentVersionId,
remoteHash?,
remoteRelativePath,
localPath: RelativePath | undefined
}
```
## Code Style
`localPath === undefined` means the doc has no local file yet — typically a remote create whose target slot was occupied at receive time; the reconciler will fetch and place when the slot frees (the bytes wait in `pendingPlacementContent`).
### Rust
- Uses extensive Clippy lints (see Cargo.toml)
- Follows pedantic linting rules
- Forbids unsafe code
- Uses cargo fmt with default settings
Local FS events from the watcher update `localPath` synchronously at enqueue time via `setLocalPath` / `upsertRecord`. The wire loop never updates it for path placement; only the reconciler does. A user rename onto a tracked slot enqueues a `LocalDelete` for the displaced doc (the OS rename clobbered its content) and clears that doc's `localPath`.
### TypeScript
- Prettier configuration: 4-space tabs, trailing commas removed, LF line endings
- ESLint with unused imports plugin
- Consistent across all three frontend packages
**Pending creates** use a `Promise<DocumentId>` chain to serialize dependent ops (`LocalUpdate`, `LocalDelete`) behind the still-in-flight `LocalCreate`. `resolveCreate` resolves the promise once the server returns a docId, and `replacePendingDocumentId` swaps the resolved id across already-queued events. `findLatestCreateForPath` is the lookup the watcher uses to attach dependents; `updatePendingCreatePath` rewrites a pending create's `event.path` in place when the user renames the file before its create has acked.
**Watermark.** `lastSeenUpdateId` uses a `MinCovered` (a contiguous-prefix tracker over a stream of integers): we only advance the published min when the next consecutive id has been processed, so out-of-order RemoteChange ids don't fool the WebSocket handshake into requesting a too-recent catch-up.
**Server catch-up.** The server's WS handshake replays events newer than the client's `last_seen_vault_update_id` from the `latest_document_versions` view (one row per doc, the latest). On those replayed rows `is_new_file` means _new to this client_ (`creation_vault_update_id > last_seen_vault_update_id`), not "this row is the doc's first version" — necessary because the catch-up only carries the latest version; if a doc was created and updated past the watermark, the client never sees its create otherwise.
## Edge-case patterns the sync engine has to survive
The two-loop split defuses most of the old race catalogue (slot-collision stashes, conflict-uuid divergence, `MoveOnConflict.NEW`/`EXISTING` policy choices) by separating wire transport from path placement. What's left:
**Pending-create docId is a `Promise`, not a string, until the create acks.** Any `LocalUpdate` / `LocalDelete` queued behind a still-in-flight `LocalCreate` carries the create's `resolvers.promise` as its `documentId`. `replacePendingDocumentId` swaps the resolved id across queued events when the create resolves; `===` comparisons against the resolved string elsewhere will silently fail until that swap runs. Anything that walks `events[]` looking for a docId match must either run after the swap or be tolerant of `Promise`-typed ids.
**`processCreate` reads `event.path` live, not `event.originalPath`.** The watcher rewrites `event.path` in place via `updatePendingCreatePath` when the user renames a pending-create file. `originalPath` was removed from `LocalCreate` events specifically because reading it would send the stale pre-rename path to the server.
**`record.localPath` mutates in place across awaits.** When the watcher renames a doc while a drain handler is awaiting an HTTP roundtrip, the queue mutates the in-flight event's record so subsequent reads see the new path. Snapshotting `record.localPath` into a local at function entry and using it after an `await` reads/writes a now-vacated slot. Read `record.localPath` live; only snapshot for the deliberate "did it change while I was awaiting" comparison.
**Reconciler-defer is the wire-loop's contract with the reconciler.** The reconciler skips records where `hasPendingLocalEventsForDocumentId` returns true. Wire-loop handlers can therefore freely write `remoteRelativePath` to whatever the server returned — even if it disagrees with `localPath` — knowing the reconciler won't move the file out from under a queued user rename.
**Watermark advancement is load-bearing both ways.** Branches that _skip_ a remote event without advancing `lastSeenUpdateId` create permanent gaps that re-deliver forever. Branches that _advance_ without applying the content lose data: the server has no further event to re-deliver, the catch-up only carries the latest version, and any state in between is gone. Don't advance unless the event was actually applied (or deliberately discarded after weighing both halves).
**`isNewFile` semantics differ between catch-up and real-time.** On WS handshake replay it means _new to this client_ (`creation_vault_update_id > last_seen_vault_update_id`); on real-time broadcasts it means _this version is the create_ (`creation_vault_update_id == vault_update_id`). A handler that decides based on one interpretation will be wrong on the other channel; reasoning about fetch-and-treat-as-new vs. ignore needs to know which channel delivered the event.
**Pause / disable-sync mid-flight** is the one race the new model doesn't structurally fix. An HTTP that committed server-side but whose response was discarded leaves the server holding a doc the client has no record of. Resume → offline scan → server-side dedupe handles it (the server merges the duplicate create into the existing doc), but if the merge produces a deconflict, the client picks up an extra file. Out of scope for the two-loop split.
**Cycle reconciliation uses in-memory content swap.** When the move graph contains a cycle, the reconciler reads every file in the cycle into memory and writes each back to its new slot, with no tmp files. A write-ahead marker at `.vaultlink/swap-<uuid>.json` lists each leg; on startup the reconciler reads the marker, hashes each `from` to determine which legs ran, and replays the rest. The `.vaultlink/**` glob is hard-coded as an internal ignore pattern so swap markers don't get sync'd.
## Two complementary E2E harnesses
- **`test-client` (fuzz):** random ops across N parallel processes for many minutes. Used by `scripts/e2e.sh`. Catches bugs nobody thought to write a test for, but reproductions are noisy.
- **`deterministic-tests`:** scripted scenarios with an in-memory FS pinned to a real server. Used to _capture_ a fuzz-discovered bug as a minimal repro before fixing it. See `frontend/deterministic-tests/README.md` for the step grammar (`pause-server`, `pause-websocket`, `barrier`, `assert-consistent`, etc.).
When a fuzz failure surfaces, the workflow is: root-cause from logs → write a deterministic test that fails on the bug → fix → confirm both the deterministic test and `e2e.sh` pass.
## Style
- TS: 4-space indent, no tabs, LF, prettier (`trailingComma: "none"`). YAML/MD use 2-space indent.
- Rust: `rustfmt.toml` enforces 4-space spaces, LF.
- Lint: ESLint for TS, Clippy for Rust, `cargo machete` for unused deps. All wired into `scripts/check.sh`.

View file

@ -2,23 +2,24 @@
[![Check](https://github.com/schmelczer/vault-link/actions/workflows/check.yml/badge.svg)](https://github.com/schmelczer/vault-link/actions/workflows/check.yml)
[![E2E tests](https://github.com/schmelczer/vault-link/actions/workflows/e2e.yml/badge.svg)](https://github.com/schmelczer/vault-link/actions/workflows/e2e.yml)
[![Publish server Docker image](https://github.com/schmelczer/vault-link/actions/workflows/publish-docker.yml/badge.svg)](https://github.com/schmelczer/vault-link/actions/workflows/publish-docker.yml)
[![Publish server Docker image](https://github.com/schmelczer/vault-link/actions/workflows/publish-server-docker.yml/badge.svg)](https://github.com/schmelczer/vault-link/actions/workflows/publish-server-docker.yml)
[![Publish CLI](https://github.com/schmelczer/vault-link/actions/workflows/publish-cli-docker.yml/badge.svg)](https://github.com/schmelczer/vault-link/actions/workflows/publish-cli-docker.yml)
[![Publish Obsidian plugin](https://github.com/schmelczer/vault-link/actions/workflows/publish-plugin.yml/badge.svg)](https://github.com/schmelczer/vault-link/actions/workflows/publish-plugin.yml)
## Develop
### Install [nvm](https://github.com/nvm-sh/nvm)
### Set up Node.JS 25 with [nvm](https://github.com/nvm-sh/nvm)
- `curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash`
- `nvm install 22`
- `nvm use 22`
- Optionally set the system-wide default: `nvm alias default 22`
- `nvm install 25`
- `nvm use 25`
- Optionally, set the system-wide default: `nvm alias default 25`
### Set up Rust
- Install [`rustup`](https://rustup.rs): `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh`
- Install [`wasm-pack`](https://rustwasm.github.io/wasm-pack/installer): `curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh`
- `cargo install cargo-insta sqlx-cli cargo-edit`
- `cargo install cargo-insta sqlx-cli`
### Install Obsidian on Linux
@ -34,7 +35,7 @@ flatpak run md.obsidian.Obsidian
Start the server:
```sh
cargo install sqlx-cli cargo-machete cargo-edit
cargo install sqlx-cli
cd sync-server
cargo run config-e2e.yml
```
@ -68,7 +69,7 @@ scripts/bump-version.sh patch
#### Run E2E tests
```sh
scripts/e2e.sh
scripts/e2e.sh 8
```
And to clean up the logs & database files, run `scripts/clean-up.sh`

87
docs/.cspell.json Normal file
View file

@ -0,0 +1,87 @@
{
"version": "0.2",
"language": "en-GB",
"dictionaries": ["en-gb"],
"ignorePaths": ["node_modules", ".vitepress/dist", ".vitepress/cache", "package-lock.json"],
"words": [
"VaultLink",
"Obsidian",
"WebSocket",
"SQLite",
"codebase",
"CRDT",
"CRDTs",
"YAML",
"nginx",
"Caddy",
"Traefik",
"systemd",
"localhost",
"vaultlink",
"Axum",
"Tokio",
"SQLx",
"reconcile",
"postgresql",
"VitePress",
"markdownlint",
"filesystem",
"backend",
"frontend",
"macOS",
"CLI",
"API",
"JSON",
"HTTP",
"HTTPS",
"SSL",
"TLS",
"WSS",
"TCP",
"VPS",
"Docker",
"Github",
"Dockerfile",
"dockerignore",
"Rustup",
"PostgreSQL",
"UUID",
"CORS",
"HSTS",
"CI",
"CD",
"OpenSSL",
"README",
"config",
"submodule",
"repo",
"autocomplete",
"autoformat",
"dedupe",
"diff",
"grep",
"stdout",
"stderr",
"chmod",
"mkdir",
"rclone",
"uuidgen",
"letsencrypt",
"fullchain",
"privkey",
"schmelczer",
"Schmelczer",
"ghcr",
"keepalive",
"healthcheck",
"writable",
"Cloudant",
"Syncthing",
"cadvisor",
"Caddyfile",
"nodelay",
"websecure",
"certresolver",
"rootfs"
]
}

2
docs/.gitignore vendored Normal file
View file

@ -0,0 +1,2 @@
.vitepress/dist/
.vitepress/cache/

4
docs/.prettierignore Normal file
View file

@ -0,0 +1,4 @@
node_modules/
.vitepress/dist/
.vitepress/cache/
package-lock.json

19
docs/.prettierrc Normal file
View file

@ -0,0 +1,19 @@
{
"printWidth": 120,
"tabWidth": 4,
"useTabs": false,
"semi": false,
"singleQuote": false,
"trailingComma": "none",
"endOfLine": "lf",
"proseWrap": "preserve",
"overrides": [
{
"files": "*.md",
"options": {
"proseWrap": "preserve",
"printWidth": 120
}
}
]
}

View file

@ -0,0 +1,60 @@
import { defineConfig } from "vitepress"
export default defineConfig({
title: "VaultLink",
description: "Self-hosted real-time synchronisation for Obsidian",
base: "/vault-link/",
themeConfig: {
logo: "/logo.svg",
nav: [
{ text: "Home", link: "/" },
{ text: "Guide", link: "/guide/getting-started" },
{ text: "Architecture", link: "/architecture/" },
{ text: "GitHub", link: "https://github.com/schmelczer/vault-link" }
],
sidebar: [
{
text: "Introduction",
items: [
{ text: "What is VaultLink?", link: "/guide/what-is-vaultlink" },
{ text: "Getting Started", link: "/guide/getting-started" },
{ text: "Limitations", link: "/guide/limitations" },
{ text: "Comparison with Alternatives", link: "/guide/alternatives" }
]
},
{
text: "Setup",
items: [
{ text: "Server Setup", link: "/guide/server-setup" },
{ text: "Obsidian Plugin", link: "/guide/obsidian-plugin" },
{ text: "CLI Client", link: "/guide/cli-client" }
]
},
{
text: "Configuration",
items: [
{ text: "Server Configuration", link: "/config/server" },
{ text: "Authentication", link: "/config/authentication" },
{ text: "Advanced Options", link: "/config/advanced" }
]
},
{
text: "Architecture",
items: [
{ text: "Overview", link: "/architecture/" },
{ text: "Sync Algorithm", link: "/architecture/sync-algorithm" },
{ text: "Data Flow", link: "/architecture/data-flow" }
]
}
],
socialLinks: [{ icon: "github", link: "https://github.com/schmelczer/vault-link" }],
footer: {
message: "Released under the MIT License.",
copyright: "Copyright © 2024-present Andras Schmelczer"
},
search: {
provider: "local"
}
},
head: [["link", { rel: "icon", type: "image/svg+xml", href: "/vault-link/logo.svg" }]]
})

159
docs/README.md Normal file
View file

@ -0,0 +1,159 @@
# VaultLink Documentation
This directory contains the VaultLink documentation site built with [VitePress](https://vitepress.dev/).
## Development
### Prerequisites
- Node.js 18+
- npm
### Setup
```bash
cd docs
npm install
```
### Local Development
Start the development server with hot reload:
```bash
npm run dev
```
The site will be available at `http://localhost:5173/vault-link/`
### Build
Build the static site:
```bash
npm run build
```
Output will be in `.vitepress/dist/`
### Preview
Preview the built site:
```bash
npm run preview
```
### Format
Format all markdown and TypeScript files:
```bash
npm run format
```
Check formatting without making changes:
```bash
npm run format:check
```
### Spell Check
Check spelling (British English):
```bash
npm run spell
```
The spell checker enforces British English spellings (e.g., "synchronisation", "optimise", "behaviour").
## Deployment
The documentation is automatically deployed to GitHub Pages when changes are pushed to the `main` branch.
The deployment workflow is configured in `.github/workflows/deploy-docs.yml`.
## Structure
```
docs/
├── .vitepress/
│ └── config.ts # VitePress configuration
├── public/ # Static assets
│ └── logo.svg # VaultLink logo
├── guide/ # User guides
│ ├── what-is-vaultlink.md
│ ├── getting-started.md
│ ├── server-setup.md
│ ├── obsidian-plugin.md
│ └── cli-client.md
├── architecture/ # Architecture documentation
│ ├── index.md
│ ├── sync-algorithm.md
│ └── data-flow.md
├── config/ # Configuration reference
│ ├── server.md
│ ├── authentication.md
│ └── advanced.md
└── index.md # Home page
```
## Writing Documentation
### Language
All documentation uses **British English**. The spell checker enforces this in CI.
### Markdown Features
VitePress supports:
- GitHub Flavoured Markdown
- Custom containers (tip, warning, danger)
- Code syntax highlighting
- Mermaid diagrams
- Emoji :rocket:
### Custom Containers
```markdown
::: tip
This is a tip
:::
::: warning
This is a warning
:::
::: danger
This is a danger message
:::
```
### Code Blocks
````markdown
```bash
npm install
```
```yaml
server:
port: 3000
```
````
## Contributing
When adding new pages:
1. Create the markdown file in the appropriate directory
2. Add it to the sidebar in `.vitepress/config.ts`
3. Test locally with `npm run dev`
4. Submit a pull request
## License
MIT - Same as VaultLink

View file

@ -0,0 +1,553 @@
# Data Flow
How data flows through VaultLink, from client to server and back.
## Connection Lifecycle
### 1. Initial Connection
```mermaid
sequenceDiagram
participant C as Client
participant S as Server
participant DB as Database
C->>S: WebSocket connect
S->>S: Accept connection
C->>S: Auth message (token + vault)
S->>S: Validate token
S->>S: Check vault access
S-->>C: Auth success
Note over C,S: Connection established
```
**Steps**:
1. Client initiates WebSocket connection to server
2. Server accepts connection
3. Client sends authentication message with token and vault name
4. Server validates token against `config.yml`
5. Server checks if user has access to requested vault
6. Server responds with success or error
7. Connection is ready for syncing
### 2. Initial Sync
After authentication, the client performs initial synchronisation:
```mermaid
sequenceDiagram
participant C as Client
participant S as Server
participant DB as SQLite
C->>C: Scan local filesystem
C->>S: Request file list
S->>DB: Query all files
DB-->>S: File metadata
S-->>C: File list with versions
loop For each local file
C->>C: Check if file on server
alt File not on server
C->>S: Upload file
S->>DB: Store file + metadata
else File on server (different version)
C->>C: Compare versions
C->>S: Upload newer or merge
end
end
loop For each server file
C->>C: Check if file local
alt File not local
C->>S: Download file
S->>DB: Retrieve file
DB-->>S: File content
S-->>C: File content
C->>C: Write to disk
end
end
S-->>C: Sync complete message
```
**Process**:
1. Client scans local filesystem
2. Client requests file list from server
3. Server queries database and returns metadata
4. Client uploads missing or changed local files
5. Client downloads missing files from server
6. Server sends sync complete notification
### 3. Real-Time Synchronization
After initial sync, changes are pushed in real-time:
```mermaid
sequenceDiagram
participant FS as Filesystem
participant C1 as Client 1
participant S as Server
participant DB as Database
participant C2 as Client 2
FS->>C1: File changed (fs.watch)
C1->>C1: Read file content
C1->>S: Upload file
S->>DB: Store new version
S->>S: Apply OT if needed
S-->>C1: Upload ACK
S->>C2: File update notification
C2->>S: Download file
S->>DB: Retrieve file
DB-->>S: File content
S-->>C2: File content
C2->>FS: Write to disk
```
**Flow**:
1. Filesystem watcher detects local change
2. Client reads file content
3. Client uploads file via WebSocket
4. Server stores in database
5. Server applies operational transformation if concurrent edits
6. Server acknowledges upload to sender
7. Server broadcasts update to other clients
8. Other clients download and apply changes
## File Operations
### Upload
```
┌─────────┐
│ Client │
└───┬─-───┘
│ 1. Detect file change
├─► 2. Read file content
├─► 3. Create upload message
│ {
│ type: "upload_file",
│ path: "notes/daily.md",
│ content: "...",
│ version: 42,
│ timestamp: "2024-01-01T12:00:00Z"
│ }
┌─────────┐
│ Server │
└───┬────-┘
│ 4. Validate message
├─► 5. Check permissions
├─► 6. Apply OT (if conflicts)
├─► 7. Store in database
├─► 8. Update version
├─► 9. Broadcast to clients
└─► 10. Send ACK to uploader
```
### Download
```
┌─────────┐
│ Server │
└───┬─-───┘
│ 1. File updated by another client
├─► 2. Broadcast notification
│ {
│ type: "file_updated",
│ path: "notes/daily.md",
│ version: 43
│ }
┌─────────┐
│ Client │
└───┬─-───┘
│ 3. Receive notification
├─► 4. Request file download
│ {
│ type: "download_file",
│ path: "notes/daily.md",
│ version: 43
│ }
┌─────────┐
│ Server │
└───┬─=───┘
│ 5. Retrieve from database
└─► 6. Send file content
{
type: "file_content",
path: "notes/daily.md",
content: "...",
version: 43
}
┌─────────┐
│ Client │
└───-─┬───┘
│ 7. Write to filesystem
└─► 8. Update local metadata
```
### Delete
```
┌─────────┐
│ Client │
└────┬────┘
│ 1. File deleted locally
├─► 2. Send delete message
│ {
│ type: "delete_file",
│ path: "notes/old.md"
│ }
┌─────────┐
│ Server │
└────┬────┘
│ 3. Mark as deleted in DB
│ (soft delete for history)
├─► 4. Broadcast deletion
└─► 5. ACK to sender
┌─────────┐
│ Other │
│ Clients │
└────┬────┘
│ 6. Delete local file
└─► 7. Update metadata
```
## Conflict Resolution Flow
### Concurrent Edits Scenario
```
Time →
Client A Server Client B
│ │ │
│ Edit file v10 │ │
│ "Add line A" │ │ Edit file v10
│ │ │ "Add line B"
│ │ │
├─── Upload @ t1 ─────────►│ │
│ │◄────── Upload @ t2 ────────┤
│ │ │
│ │ 1. Receive both edits │
│ │ (based on v10) │
│ │ │
│ │ 2. Apply first edit │
│ │ → v11 (line A added) │
│ │ │
│ │ 3. Transform second edit │
│ │ against first │
│ │ │
│ │ 4. Apply transformed edit │
│ │ → v12 (both lines) │
│ │ │
│◄──── v12 content ────────┤ │
│ ├───── v12 content ─────────►│
│ │ │
│ Apply v12 │ │ Apply v12
│ (has both lines) │ │ (has both lines)
│ │ │
```
### Conflict Resolution Steps
1. **Detection**: Server receives two edits based on the same version
2. **Ordering**: Determine which edit to apply first (by timestamp or client ID)
3. **First edit**: Apply directly to database
4. **Transformation**: Transform second edit against first using OT
5. **Second edit**: Apply transformed edit to database
6. **Broadcast**: Send merged result to all clients
7. **Application**: Clients apply merged version locally
## Database Schema
### Core Tables
```sql
-- Document metadata
CREATE TABLE documents (
id INTEGER PRIMARY KEY,
path TEXT NOT NULL,
version INTEGER NOT NULL,
content_hash TEXT,
size INTEGER,
created_at TIMESTAMP,
updated_at TIMESTAMP,
deleted BOOLEAN DEFAULT FALSE
);
-- Version history
CREATE TABLE versions (
id INTEGER PRIMARY KEY,
document_id INTEGER,
version INTEGER,
content BLOB,
created_at TIMESTAMP,
FOREIGN KEY (document_id) REFERENCES documents(id)
);
-- Client sync cursors
CREATE TABLE cursors (
client_id TEXT PRIMARY KEY,
last_version INTEGER,
last_updated TIMESTAMP
);
```
### Queries
**Get files since version**:
```sql
SELECT * FROM documents
WHERE version > ? AND deleted = FALSE
ORDER BY version ASC;
```
**Store new version**:
```sql
INSERT INTO versions (document_id, version, content, created_at)
VALUES (?, ?, ?, ?);
UPDATE documents
SET version = ?, updated_at = ?
WHERE id = ?;
```
**Update cursor**:
```sql
INSERT OR REPLACE INTO cursors (client_id, last_version, last_updated)
VALUES (?, ?, ?);
```
## Message Protocol
### Client → Server Messages
**Upload File**:
```json
{
"type": "upload_file",
"path": "notes/example.md",
"content": "File content here...",
"base_version": 10,
"timestamp": "2024-01-01T12:00:00Z"
}
```
**Download File**:
```json
{
"type": "download_file",
"path": "notes/example.md"
}
```
**Delete File**:
```json
{
"type": "delete_file",
"path": "notes/old.md"
}
```
**List Files**:
```json
{
"type": "list_files",
"since_version": 0
}
```
### Server → Client Messages
**File Updated**:
```json
{
"type": "file_updated",
"path": "notes/example.md",
"version": 11,
"size": 1024,
"hash": "abc123..."
}
```
**File Content**:
```json
{
"type": "file_content",
"path": "notes/example.md",
"content": "Updated content...",
"version": 11
}
```
**File Deleted**:
```json
{
"type": "file_deleted",
"path": "notes/old.md",
"version": 12
}
```
**Sync Complete**:
```json
{
"type": "sync_complete",
"total_files": 150,
"current_version": 200
}
```
**Error**:
```json
{
"type": "error",
"message": "File too large",
"code": "FILE_TOO_LARGE"
}
```
## Error Handling
### Client-Side Errors
**Network failure**:
1. Detect WebSocket disconnect
2. Queue pending operations
3. Retry connection with exponential backoff
4. Replay queued operations on reconnect
**File read error**:
1. Log error
2. Skip file
3. Continue with other files
4. Report to user
**Write conflict**:
1. Receive updated version from server
2. Apply OT merge locally
3. Overwrite local file
4. Continue syncing
### Server-Side Errors
**Database error**:
1. Log error
2. Return error to client
3. Client retries operation
**Invalid operation**:
1. Validate message format
2. Return specific error code
3. Client handles error appropriately
**Authentication failure**:
1. Reject connection
2. Send auth error
3. Client prompts for new credentials
## Performance Optimizations
### Batching
- Small, rapid changes are batched together
- Reduces message overhead
- Applied as single atomic update
### Compression
- Large files compressed before transmission
- Reduces bandwidth usage
- Transparent to application layer
### Incremental Sync
- Only changed portions of files sent
- Uses content-based diffing
- Significantly reduces data transfer
### Caching
- Server caches recent file versions
- Reduces database queries
- Improves response time
## Monitoring Data Flow
### Server Logs
```
2024-01-01 12:00:00 INFO WebSocket connection from 192.168.1.100
2024-01-01 12:00:01 INFO User 'alice' authenticated for vault 'personal'
2024-01-01 12:00:05 INFO Upload: notes/daily.md (v10 -> v11)
2024-01-01 12:00:06 INFO Broadcast to 3 clients
2024-01-01 12:00:10 INFO Conflict resolved: notes/shared.md (v12)
```
### Client Logs
```
2024-01-01 12:00:00 INFO Connecting to ws://sync.example.com
2024-01-01 12:00:01 INFO Connected, authenticating...
2024-01-01 12:00:01 INFO Authentication successful
2024-01-01 12:00:02 INFO Starting initial sync
2024-01-01 12:00:10 INFO Sync complete: 150 files, 200 MB
2024-01-01 12:00:15 INFO Uploaded: notes/daily.md
2024-01-01 12:00:20 INFO Downloaded: notes/shared.md (merged)
```
## Next Steps
- [Understand the sync algorithm →](/architecture/sync-algorithm)
- [Configure the server →](/config/server)
- [Deploy VaultLink →](/guide/getting-started)

334
docs/architecture/index.md Normal file
View file

@ -0,0 +1,334 @@
# Architecture Overview
Central sync server with multiple clients. High-level architecture and design decisions.
## System Components
```
┌─────────────────────────────────────────────────────────────┐
│ Clients │
├─────────────────────┬───────────────────┬───────────────────┤
│ Obsidian Plugin │ Obsidian Plugin │ CLI Client │
│ (User A - Device1) │ (User A - Device2│ (Server/Backup) │
└──────────┬──────────┴─────────┬─────────┴──────────┬────────┘
│ │ │
│ WebSocket │ WebSocket │ WebSocket
│ │ │
└────────────────────┼────────────────────┘
┌───────────▼───────────┐
│ Sync Server │
│ (Rust + Axum) │
│ │
│ ┌─────────────────┐ │
│ │ WebSocket Hub │ │
│ └────────┬────────┘ │
│ │ │
│ ┌────────▼────────┐ │
│ │ Sync Engine │ │
│ │ (OT Algorithm) │ │
│ └────────┬────────┘ │
│ │ │
│ ┌────────▼────────┐ │
│ │ SQLite Database │ │
│ │ (Per Vault) │ │
│ └─────────────────┘ │
└───────────────────────┘
```
## Core Components
### Sync Server
Central authority for synchronisation. Rust + Axum framework.
**Responsibilities**:
- Accept WebSocket connections from clients
- Authenticate users via token-based auth
- Store document versions in SQLite
- Coordinate real-time updates between clients
- Apply operational transformation for conflict resolution
- Manage vault access control
**Technology**:
- **Language**: Rust 1.92+
- **Framework**: Axum (async web framework)
- **Database**: SQLite with SQLx
- **Protocol**: WebSockets for real-time communication
- **Sync Algorithm**: reconcile-text (operational transformation)
### Sync Client Library
TypeScript library with core sync logic. Used by Obsidian plugin and CLI client.
**Responsibilities**:
- Manage WebSocket connection to server
- Watch local filesystem for changes
- Upload and download files
- Apply remote changes locally
- Handle conflict resolution
- Maintain sync metadata
**Technology**:
- **Language**: TypeScript
- **Build**: Webpack
- **Protocol**: WebSocket client
- **File System**: Node.js `fs` API / Obsidian API
### Obsidian Plugin
Integration layer between sync client and Obsidian.
**Responsibilities**:
- Provide UI for configuration
- Bridge sync client with Obsidian's file system API
- Handle Obsidian lifecycle events
- Display sync status to users
**Technology**:
- **Platform**: Obsidian Plugin API
- **Core**: sync-client library
- **UI**: Obsidian settings UI
### CLI Client
Standalone executable for syncing vaults without Obsidian.
**Responsibilities**:
- Command-line interface
- File system access via Node.js
- Daemon mode for continuous sync
- Health check endpoint for monitoring
**Technology**:
- **Language**: TypeScript
- **Runtime**: Node.js
- **CLI**: Commander.js
- **Core**: sync-client library
## Data Flow
### Initial Connection
1. Client connects via WebSocket to server
2. Server authenticates using provided token
3. Server verifies user has access to requested vault
4. Connection established, sync begins
### File Upload Flow
```
Client Server
│ │
│ 1. File changed locally │
│ │
│ 2. Read file content │
│ │
│ 3. WebSocket: Upload file │
├──────────────────────────────►│
│ │ 4. Store in SQLite
│ │
│ │ 5. Broadcast to other clients
│ ├───────────────────────►
│ 6. Ack upload │
│◄──────────────────────────────┤
```
### File Download Flow
```
Client A Server Client B
│ │ │
│ │ 1. File uploaded │
│ │◄────────────────────────┤
│ │ │
│ │ 2. Store in DB │
│ │ │
│ 3. Push notification │ │
│◄────────────────────────┤ │
│ │ │
│ 4. Download file │ │
├────────────────────────►│ │
│ │ │
│ 5. Write locally │ │
│ │ │
```
### Conflict Resolution
When two clients edit the same file simultaneously:
```
Client A Server Client B
│ │ │
│ 1. Edit file │ │ 1. Edit same file
│ │ │
│ 2. Upload changes │ │ 2. Upload changes
├────────────────────────►│◄────────────────────────┤
│ │ │
│ │ 3. Apply OT algorithm │
│ │ - Merge both edits │
│ │ - Preserve all changes│
│ │ │
│ 4. Receive merged ver. │ 5. Receive merged ver. │
│◄────────────────────────┤────────────────────────►│
│ │ │
│ 6. Apply locally │ │ 6. Apply locally
```
## Storage Architecture
### Server Storage
Each vault has its own SQLite database:
```
databases/
├── vault-1.db
├── vault-2.db
└── shared-team.db
```
**Database Schema** (simplified):
- **documents**: File metadata (path, size, modified time)
- **versions**: Document content with version history
- **cursors**: Client sync state
### Client Storage
Clients maintain sync metadata:
```
.vaultlink/
├── metadata.json # Sync state
└── cache/ # Optional local cache
```
The `.vaultlink` directory tracks which files have been synced and their versions to enable efficient synchronisation.
## Communication Protocol
### WebSocket Messages
Client-server communication uses JSON messages over WebSocket.
**Message Types**:
- `upload_file`: Client → Server (file upload)
- `download_file`: Client → Server (request file)
- `file_updated`: Server → Client (file changed notification)
- `file_deleted`: Server → Client (file deleted notification)
- `sync_complete`: Server → Client (initial sync finished)
### Authentication
Token-based authentication on connection:
```typescript
// Client sends token on connect
{
type: "auth",
token: "user-auth-token",
vault: "vault-name"
}
// Server responds
{
type: "auth_success"
}
// or
{
type: "auth_error",
message: "Invalid token"
}
```
## Scalability Considerations
### Current Architecture
- **SQLite per vault**: Simple, performant, limited to single server
- **WebSocket connections**: Stateful, requires sticky sessions for load balancing
- **Operational transformation**: Centralized on server
### Scaling Approaches
**Vertical Scaling**:
- Increase server resources (CPU, RAM, storage)
- Optimize database queries and indexing
- Tune connection limits
**Horizontal Scaling** (future):
- Separate vault servers (vault sharding)
- Load balancer with sticky sessions
- Shared storage layer for SQLite databases
- Consider alternative databases (PostgreSQL) for multi-server setups
### Performance Characteristics
- **Small vaults** (< 1000 files): Excellent performance
- **Medium vaults** (1000-10000 files): Good performance with tuning
- **Large vaults** (> 10000 files): May require optimisation
- **Concurrent users**: Tested with dozens of simultaneous clients per vault
## Security Model
### Authentication
- Token-based authentication
- Tokens configured in server `config.yml`
- No password hashing (tokens are secrets)
### Authorization
- Per-user vault access control
- Allow-list or deny-list patterns
- Global access or vault-specific access
### Network Security
- WebSocket over TLS (WSS) for encrypted transport
- No built-in SSL (use reverse proxy)
- CORS configured for web clients
### Data Security
- No encryption at rest (use encrypted filesystems if needed)
- No end-to-end encryption (server sees all content)
- Self-hosted model: you control the data
## Technology Choices
**Rust**: Low latency, memory safe, excellent async with Tokio, compile-time SQL verification
**SQLite**: No separate database server, fast for reads, single file per vault, backups are file copies
**WebSocket**: Bidirectional push, no polling overhead, built-in browser/Node.js support
**Operational Transformation**: Automatic conflict resolution, preserves all edits, real-time collaboration
## Design Principles
1. **Self-hosted first**: Users control their data and infrastructure
2. **Simplicity**: Easy to deploy and operate
3. **Real-time**: Changes appear immediately
4. **Reliability**: Handle network failures gracefully
5. **Performance**: Fast sync for typical vault sizes
6. **Privacy**: No third-party services or telemetry
## Next Steps
- [Learn about the sync algorithm →](/architecture/sync-algorithm)
- [Understand data flow in detail →](/architecture/data-flow)
- [Deploy the server →](/guide/server-setup)

View file

@ -0,0 +1,438 @@
# Sync Algorithm
VaultLink uses operational transformation (OT) to handle concurrent edits and maintain consistency across clients.
## Operational Transformation
Operational transformation is a technique for managing concurrent edits to the same document. It transforms operations (edits) so they can be applied in different orders while preserving user intent.
### Why OT?
Traditional conflict resolution approaches:
- **Last write wins**: Loses data, frustrating for users
- **Manual merging**: Interrupts workflow, requires user intervention
- **Version branching**: Complex, not suitable for real-time sync
Operational transformation:
- **Automatic**: No user intervention required
- **Preserves all edits**: No data loss
- **Real-time**: Changes appear immediately
- **Intuitive**: Behaviour matches user expectations
## The reconcile-text Library
VaultLink uses the [`reconcile-text`](https://crates.io/crates/reconcile-text) Rust library for operational transformation on text documents.
### Why reconcile-text over CRDTs?
VaultLink faces a **differential synchronisation** challenge: users edit Obsidian vaults with various editors (Obsidian desktop, Obsidian mobile, Vim, VS Code, or any text editor), often while offline. This means we only observe the **final state** of each document after editing, not the individual keystrokes or operations that produced it.
**The fundamental problem**:
- **CRDTs and traditional OT** require capturing every individual operation (each character insertion, deletion, cursor movement)
- **VaultLink's reality**: Users edit files with arbitrary tools, sync happens after the fact
- **What we know**: Parent version and two modified versions
- **What we don't know**: The sequence of operations that created those modifications
**Why reconcile-text wins for this use case**:
1. **Works with end states only**: reconcile-text performs conflict-free 3-way merging given just parent, left, and right versions—no operation history needed
2. **Editor-agnostic**: Users can edit with any tool without requiring VaultLink-specific plugins or operation tracking
3. **Offline-first**: Edits made while disconnected are merged cleanly when sync resumes, because we're diffing final states rather than replaying operations
4. **No conflict markers**: Unlike Git merge, produces clean merged output without `<<<<<<<` markers that interrupt note-taking flow
5. **Human text forgiveness**: For knowledge bases and documentation, a slightly imperfect merge (e.g., minor word order issues) is vastly preferable to manual conflict resolution
6. **Simpler infrastructure**: No need for complex operation capture, transformation logs, or tombstone management that CRDTs require
**The trade-off**:
CRDTs excel when you control the entire editing infrastructure and can capture every operation. reconcile-text excels when you're synchronising independently-edited files—exactly VaultLink's scenario. The merge quality depends on Myers' diff algorithm rather than operation history, which is the correct trade-off for differential sync.
For note-taking workflows where users value editor freedom and offline editing, this approach provides superior user experience compared to either CRDTs (which would require operation tracking) or Git-style merging (which requires manual conflict resolution).
[Learn more about reconcile-text →](https://schmelczer.dev/reconcile)
### How It Works
Given three versions (parent, left, right), reconcile-text produces a merged result.
**How reconcile-text works**:
1. **Tokenisation**: Split text into words (using `BuiltinTokenizer::Word`)
2. **Three-way diff**: Compare parent→left and parent→right changes
3. **Merge**: Combine non-conflicting changes, prefer content preservation for conflicts
4. **Result**: Merged text with both edits applied
**Example**:
```
Parent: "The quick brown fox"
User A: "The quick red fox" (changes "brown" → "red")
User B: "The very quick brown fox" (inserts "very ")
Merged: "The very quick red fox" (both changes applied)
```
**Merge conditions**: Only `.md` and `.txt` files with valid UTF-8 get merged. Binary files or other extensions use last-write-wins.
### Operation Types
The algorithm handles these operations:
- **Insert**: Add text at position
- **Delete**: Remove text from position
- **Retain**: Keep existing text unchanged
### Transformation Process
1. **Client A** makes edit and sends to server
2. **Client B** makes concurrent edit and sends to server
3. **Server** receives both edits
4. **Server** transforms operations to account for concurrent changes
5. **Server** applies merged result to database
6. **Server** sends transformed operations to both clients
7. **Clients** apply transformed operations locally
## Sync State Management
VaultLink maintains sync state to track which changes have been applied.
### Version Vectors
Each document has a version tracked by:
- **Server version**: Incremented on each change
- **Client cursors**: Track which version each client has seen
This enables:
- Efficient syncing (only send changes since last sync)
- Conflict detection (concurrent edits to same version)
- Ordering of operations
### Cursor Management
Clients maintain a cursor position:
```rust
struct Cursor {
vault_id: String,
client_id: String,
last_version: u64,
last_updated: DateTime,
}
```
On sync:
1. Client sends cursor (last seen version)
2. Server returns all changes since that version
3. Client applies changes and updates cursor
## Conflict Resolution Flow
### Scenario: Concurrent Edits
Two users edit the same paragraph simultaneously.
**Initial state**:
```
Version 10: "The quick brown fox jumps over the lazy dog."
```
**User A's edit** (version 11):
```
"The quick brown fox jumps over the very lazy dog."
```
_Inserts "very " at position 40_
**User B's edit** (also from version 10):
```
"The quick red fox jumps over the lazy dog."
```
_Replaces "brown" with "red" at position 10_
### Server Processing
1. **Receive User A's operation**:
- Base: version 10
- Operation: Insert("very ", position=40)
- Apply to database → version 11
2. **Receive User B's operation**:
- Base: version 10
- Operation: Replace("brown"→"red", position=10)
- **Conflict detected**: Base is version 10, but current is version 11
3. **Transform User B's operation**:
- Transform against User A's operation
- Adjust positions/content as needed
- Apply transformed operation → version 12
4. **Broadcast updates**:
- Send User A's operation to User B
- Send transformed User B's operation to User A
### Final Result
```
Version 12: "The quick red fox jumps over the very lazy dog."
```
Both edits are preserved in the final document.
## Edge Cases
### 1. Delete vs Insert Conflict
**Scenario**: User A deletes a paragraph while User B edits it.
**Resolution**:
- OT algorithm prioritizes preservation of content
- Insert operation is transformed to account for deletion
- Typically results in inserted content appearing nearby
**Example**:
```
Base: "Line 1\nLine 2\nLine 3"
User A: Delete Line 2 → "Line 1\nLine 3"
User B: Edit Line 2 → "Line 1\nLine 2 modified\nLine 3"
Result: "Line 1\nLine 2 modified\nLine 3"
```
(Insert takes precedence, preserving user content)
### 2. Overlapping Edits
**Scenario**: Two users edit overlapping regions.
**Resolution**:
- OT splits operations into non-overlapping segments
- Applies each segment independently
- Merges results
### 3. Delete vs Delete
**Scenario**: Two users delete overlapping text.
**Resolution**:
- Deletes are merged
- Final result has the union of deleted ranges removed
### 4. Network Partitions
**Scenario**: Client loses connection, makes edits offline, reconnects.
**Resolution**:
1. Client queues edits locally
2. On reconnect, sends all queued operations
3. Server applies OT against all operations that happened during partition
4. Client receives transformed operations and applies
## Performance Characteristics
### Time Complexity
- **Single operation**: O(1) for most operations
- **Transformation**: O(n) where n is operation size
- **Conflict resolution**: O(m × n) where m is number of concurrent operations
### Space Complexity
- **Version history**: Grows with number of changes
- **Cursors**: O(clients × vaults)
- **Active operations**: Minimal (processed in real-time)
### Optimisation
VaultLink optimises for:
- Small, frequent edits (typical typing patterns)
- Text documents (not binary files)
- Real-time processing (no batching delay)
## Limitations
### Binary and Non-Mergeable Files
Only **`.md`** and **`.txt`** files get automatic merging. Everything else uses last-write-wins.
**Binary detection**:
- Files with NUL bytes (`0x00`)
- Files failing UTF-8 validation
Even `.md` files are treated as binary if they fail UTF-8 checks.
**Last-write-wins behaviour**:
```
User A uploads image.png → Server version 1
User B uploads image.png → Server version 2 (A's upload lost)
```
**Workaround**: Avoid concurrent edits to non-text files. [See all limitations →](/guide/limitations)
### Large Documents
Very large documents (> 1MB) may have:
- Higher transformation costs
- Slower sync times
- Increased memory usage
**Workaround**: Split large documents or increase timeout settings.
### Complex Formatting
Markdown with complex structures may occasionally produce unexpected results:
- Nested lists
- Tables
- Code blocks
**Workaround**: Manual cleanup if needed, or minimize concurrent edits to complex structures.
## Consistency Guarantees
### Strong Consistency
VaultLink provides **strong eventual consistency**:
- All clients eventually converge to the same state
- Operations applied in causal order
- No data loss under normal operation
### Ordering Guarantees
- Operations from the same client are applied in order
- Concurrent operations may be applied in any order
- Final result is independent of operation order (commutative)
### Durability
- Operations are written to SQLite before acknowledgment
- SQLite ACID guarantees protect against data loss
- Clients retry failed uploads
## Comparison with Other Approaches
### Git-style Merging
| Aspect | Git Merge | VaultLink OT |
| -------------------------- | ------------ | ----------------------- |
| Real-time | No | Yes |
| Manual conflict resolution | Yes | No |
| Branching | Yes | No |
| Automatic merge | Limited | Always |
| Use case | Code changes | Collaborative documents |
### CRDTs (Conflict-free Replicated Data Types)
| Aspect | CRDTs | VaultLink (reconcile-text) |
| ----------------------------- | ------------------------------------ | ------------------------------------------------- |
| **Operation tracking** | Required (every keystroke) | Not required (end states only) |
| **Editor freedom** | Limited (must use CRDT-aware editor) | Unlimited (any text editor works) |
| **Offline editing** | Requires operation log | Works with file comparison |
| **Server required** | No | Yes |
| **Memory overhead** | Higher (tombstones, metadata) | Lower (versions only) |
| **Infrastructure complexity** | Higher | Lower |
| **Best for** | Controlled editing environments | Independent file editing (Obsidian, Vim, VS Code) |
**Key insight**: CRDTs are superior when you can capture every operation. reconcile-text is superior when users edit files independently with arbitrary tools—exactly VaultLink's scenario.
### Last Write Wins
| Aspect | LWW | VaultLink OT |
| --------------- | ---- | ------------ |
| Data loss | Yes | No |
| Simplicity | High | Medium |
| User experience | Poor | Excellent |
| Performance | Best | Good |
## Algorithm Details
### Transformation Rules
When transforming operation `A` against operation `B`:
1. **Insert vs Insert**:
- If positions equal: Order by client ID
- If different positions: Adjust positions
2. **Insert vs Delete**:
- If insert in deleted range: Shift insert position
- If insert after delete: Adjust position by deleted length
3. **Delete vs Delete**:
- If ranges overlap: Merge delete ranges
- If ranges disjoint: Adjust positions
4. **Retain vs Any**:
- Retain operations don't conflict
- Simply adjust positions
### Transformation Example
```rust
// Pseudo-code for transformation
fn transform(op_a: Operation, op_b: Operation) -> (Operation, Operation) {
match (op_a, op_b) {
(Insert(pos_a, text_a), Insert(pos_b, text_b)) => {
if pos_a < pos_b {
(op_a, Insert(pos_b + text_a.len(), text_b))
} else if pos_a > pos_b {
(Insert(pos_a + text_b.len(), text_a), op_b)
} else {
// Same position, use client ID to break tie
if client_id_a < client_id_b {
(op_a, Insert(pos_b + text_a.len(), text_b))
} else {
(Insert(pos_a + text_b.len(), text_a), op_b)
}
}
}
// ... other cases
}
}
```
## Best Practices
### For Smooth Collaboration
1. **Small edits**: Make small, focused changes for easier merging
2. **Coordinate major changes**: Discuss large refactors with team
3. **Monitor sync status**: Ensure changes are uploaded before signing off
4. **Test conflict resolution**: Verify behaviour matches expectations
### For Developers
1. **Text files preferred**: OT works best on text
2. **Limit file sizes**: Keep documents reasonably sized
3. **Binary files**: Use versioning or avoid concurrent edits
4. **Testing**: Test concurrent edit scenarios thoroughly
## Further Reading
- [reconcile-text library](https://crates.io/crates/reconcile-text)
- [Operational Transformation FAQ](https://en.wikipedia.org/wiki/Operational_transformation)
- [Data flow architecture →](/architecture/data-flow)

603
docs/config/advanced.md Normal file
View file

@ -0,0 +1,603 @@
# Advanced Configuration
Advanced topics for optimising and customising your VaultLink deployment.
## Database Optimisation
### SQLite Tuning
While VaultLink handles most SQLite configuration automatically, you can optimise for specific workloads.
#### WAL Mode
VaultLink uses Write-Ahead Logging (WAL) mode by default for better concurrency.
**Benefits**:
- Readers don't block writers
- Writers don't block readers
- Better performance for concurrent access
**Maintenance**:
```bash
# Checkpoint WAL to main database (run periodically)
sqlite3 databases/vault.db "PRAGMA wal_checkpoint(TRUNCATE);"
```
#### Database Size Management
Over time, databases can grow with version history:
```bash
# Check database size
du -h databases/*.db
# Vacuum to reclaim space (offline only)
sqlite3 databases/vault.db "VACUUM;"
# Analyse for query optimisation
sqlite3 databases/vault.db "ANALYZE;"
```
**Schedule maintenance**:
```bash
#!/bin/bash
# monthly-maintenance.sh
for db in databases/*.db; do
echo "Optimising $db"
sqlite3 "$db" "PRAGMA optimize;"
sqlite3 "$db" "PRAGMA wal_checkpoint(TRUNCATE);"
done
```
### Version History Cleanup
VaultLink stores **all versions indefinitely** by default. Database grows with every change.
**Database schema**: Each version stored in `documents` table with `vault_update_id` (sequential).
Manual cleanup (keep last 100 versions per document):
```bash
#!/bin/bash
# prune-old-versions.sh
for db in databases/*.db; do
sqlite3 "$db" <<EOF
DELETE FROM documents
WHERE vault_update_id NOT IN (
SELECT vault_update_id FROM documents d2
WHERE d2.document_id = documents.document_id
ORDER BY vault_update_id DESC
LIMIT 100
);
EOF
done
```
**Warning**: This deletes old versions permanently. No undo.
Run monthly via cron:
```bash
0 3 1 * * /opt/vaultlink/prune-old-versions.sh
```
## Performance Tuning
### Connection Pool Sizing
Calculate optimal `max_connections_per_vault`:
```
max_connections = (concurrent_users × avg_operations_per_user) + buffer
```
**Example**:
- 20 concurrent users
- 2 operations per user on average
- 25% buffer
```
max_connections = (20 × 2) × 1.25 = 50
```
### Timeout Configuration
Adjust timeouts based on network characteristics:
**Fast local network**:
```yaml
database:
cursor_timeout_seconds: 30
server:
response_timeout_seconds: 30
```
**Slow or unreliable network**:
```yaml
database:
cursor_timeout_seconds: 180
server:
response_timeout_seconds: 120
```
**Mobile clients**:
```yaml
database:
cursor_timeout_seconds: 300 # Longer for intermittent connections
server:
response_timeout_seconds: 180
```
## Reverse Proxy Configuration
### Nginx with SSL
Complete Nginx configuration for production:
```nginx
# Rate limiting
limit_req_zone $binary_remote_addr zone=vaultlink:10m rate=10r/s;
upstream vaultlink {
server localhost:3000;
keepalive 32;
}
server {
listen 443 ssl http2;
server_name sync.example.com;
ssl_certificate /etc/letsencrypt/live/sync.example.com/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/sync.example.com/privkey.pem;
# SSL security settings
ssl_protocols TLSv1.2 TLSv1.3;
ssl_ciphers HIGH:!aNULL:!MD5;
ssl_prefer_server_ciphers on;
# HSTS
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
# Rate limiting
limit_req zone=vaultlink burst=20 nodelay;
# Client body size (match server config)
client_max_body_size 512M;
# Timeouts
proxy_connect_timeout 90s;
proxy_send_timeout 90s;
proxy_read_timeout 3600s; # WebSocket long-lived connections
# WebSocket headers
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
# Disable buffering for WebSocket
proxy_buffering off;
location / {
proxy_pass http://vaultlink;
}
# Health check endpoint (use any vault name)
location /health {
proxy_pass http://vaultlink/vaults/test/ping;
access_log off;
}
}
# Redirect HTTP to HTTPS
server {
listen 80;
server_name sync.example.com;
return 301 https://$server_name$request_uri;
}
```
### Caddy with Auto SSL
Caddy handles SSL automatically:
```caddy
sync.example.com {
reverse_proxy localhost:3000 {
# WebSocket support
header_up X-Real-IP {remote_host}
header_up X-Forwarded-For {remote_host}
header_up X-Forwarded-Proto {scheme}
# Timeouts
transport http {
read_timeout 3600s
write_timeout 90s
}
}
# Rate limiting (requires caddy-rate-limit plugin)
rate_limit {
zone dynamic {
match {
remote_ip
}
rate 10r/s
burst 20
}
}
}
```
### Traefik Configuration
Using Docker labels:
```yaml
services:
vaultlink-server:
image: ghcr.io/schmelczer/vault-link-server:latest
labels:
- "traefik.enable=true"
- "traefik.http.routers.vaultlink.rule=Host(`sync.example.com`)"
- "traefik.http.routers.vaultlink.entrypoints=websecure"
- "traefik.http.routers.vaultlink.tls.certresolver=letsencrypt"
- "traefik.http.services.vaultlink.loadbalancer.server.port=3000"
# Middleware for timeouts
- "traefik.http.middlewares.vaultlink-timeout.timeout.request=3600s"
```
## Docker Optimizations
### Resource Limits
Limit container resources:
```yaml
services:
vaultlink-server:
image: ghcr.io/schmelczer/vault-link-server:latest
deploy:
resources:
limits:
cpus: "2.0"
memory: 4G
reservations:
cpus: "1.0"
memory: 2G
```
### Logging Configuration
Optimize Docker logging:
```yaml
services:
vaultlink-server:
image: ghcr.io/schmelczer/vault-link-server:latest
logging:
driver: "json-file"
options:
max-size: "50m"
max-file: "5"
```
### Volume Optimization
Use named volumes for better performance:
```yaml
services:
vaultlink-server:
image: ghcr.io/schmelczer/vault-link-server:latest
volumes:
- vaultlink-data:/data
- vaultlink-logs:/data/logs
volumes:
vaultlink-data:
driver: local
driver_opts:
type: none
o: bind
device: /mnt/fast-ssd/vaultlink
vaultlink-logs:
driver: local
```
## High Availability
### Health Checks
Comprehensive health monitoring:
```yaml
services:
vaultlink-server:
image: ghcr.io/schmelczer/vault-link-server:latest
healthcheck:
test: ["CMD-SHELL", "curl -f http://localhost:3000/vaults/test/ping || exit 1"]
interval: 10s
timeout: 5s
retries: 3
start_period: 30s
```
Monitor health in production:
```bash
#!/bin/bash
# health-monitor.sh
while true; do
if ! curl -sf http://localhost:3000/vaults/test/ping > /dev/null; then
echo "Health check failed at $(date)" | mail -s "VaultLink Down" admin@example.com
# Optionally restart
# docker restart vaultlink-server
fi
sleep 30
done
```
### Backup Automation
Automated backup script:
```bash
#!/bin/bash
# backup-vaultlink.sh
BACKUP_DIR="/backup/vaultlink"
DATA_DIR="/data"
DATE=$(date +%Y%m%d-%H%M%S)
RETENTION_DAYS=30
# Create backup directory
mkdir -p "$BACKUP_DIR/$DATE"
# Backup databases (with WAL checkpoint)
for db in "$DATA_DIR"/databases/*.db; do
sqlite3 "$db" "PRAGMA wal_checkpoint(TRUNCATE);"
cp "$db" "$BACKUP_DIR/$DATE/"
[ -f "${db}-wal" ] && cp "${db}-wal" "$BACKUP_DIR/$DATE/"
[ -f "${db}-shm" ] && cp "${db}-shm" "$BACKUP_DIR/$DATE/"
done
# Backup configuration
cp "$DATA_DIR/config.yml" "$BACKUP_DIR/$DATE/"
# Compress backup
tar -czf "$BACKUP_DIR/vaultlink-$DATE.tar.gz" -C "$BACKUP_DIR" "$DATE"
rm -rf "$BACKUP_DIR/$DATE"
# Clean old backups
find "$BACKUP_DIR" -name "vaultlink-*.tar.gz" -mtime +$RETENTION_DAYS -delete
# Upload to remote storage (optional)
# rclone copy "$BACKUP_DIR/vaultlink-$DATE.tar.gz" remote:backups/
```
Schedule with cron:
```cron
0 2 * * * /opt/vaultlink/backup-vaultlink.sh
```
### Restore from Backup
```bash
#!/bin/bash
# restore-vaultlink.sh
BACKUP_FILE="$1"
DATA_DIR="/data"
if [ -z "$BACKUP_FILE" ]; then
echo "Usage: $0 <backup-file.tar.gz>"
exit 1
fi
# Stop server
docker stop vaultlink-server
# Extract backup
tar -xzf "$BACKUP_FILE" -C /tmp/
BACKUP_DATE=$(basename "$BACKUP_FILE" .tar.gz | cut -d- -f2-)
# Restore databases
cp /tmp/"$BACKUP_DATE"/databases/*.db "$DATA_DIR/databases/"
# Restore config (careful!)
# cp /tmp/$BACKUP_DATE/config.yml "$DATA_DIR/"
# Cleanup
rm -rf /tmp/"$BACKUP_DATE"
# Start server
docker start vaultlink-server
echo "Restore complete. Check server logs."
```
## Monitoring and Metrics
### Prometheus Metrics
While VaultLink doesn't expose metrics natively, monitor Docker:
```yaml
# docker-compose.yml
services:
vaultlink-server:
image: ghcr.io/schmelczer/vault-link-server:latest
labels:
- "prometheus.io/scrape=true"
- "prometheus.io/port=3000"
cadvisor:
image: gcr.io/cadvisor/cadvisor:latest
volumes:
- /:/rootfs:ro
- /var/run:/var/run:ro
- /sys:/sys:ro
- /var/lib/docker/:/var/lib/docker:ro
ports:
- 8080:8080
```
### Log Analysis
Analyze logs for insights:
```bash
# Most active users
grep "authenticated" logs/*.log | cut -d"'" -f2 | sort | uniq -c | sort -rn
# Failed authentications by IP
grep "Authentication failed" logs/*.log | grep -oP '\d+\.\d+\.\d+\.\d+' | sort | uniq -c | sort -rn
# Upload activity
grep "Upload:" logs/*.log | wc -l
# Average files per vault
grep "Sync complete" logs/*.log | grep -oP '\d+ files' | cut -d' ' -f1 | awk '{sum+=$1; count++} END {print sum/count}'
```
### Alerting
Simple alerting with cron:
```bash
#!/bin/bash
# alert-errors.sh
ERROR_THRESHOLD=10
ERROR_COUNT=$(grep -c "ERROR" logs/latest.log)
if [ "$ERROR_COUNT" -gt "$ERROR_THRESHOLD" ]; then
echo "VaultLink has $ERROR_COUNT errors in the last hour" | \
mail -s "VaultLink Alert" admin@example.com
fi
```
## Security Hardening
### Network Isolation
Run VaultLink in isolated network:
```yaml
services:
vaultlink-server:
image: ghcr.io/schmelczer/vault-link-server:latest
networks:
- vaultlink-internal
- proxy-external
networks:
vaultlink-internal:
internal: true
proxy-external:
driver: bridge
```
### Read-Only Root Filesystem
Run with read-only root (mount writable volumes for data):
```yaml
services:
vaultlink-server:
image: ghcr.io/schmelczer/vault-link-server:latest
read_only: true
volumes:
- ./data:/data
- /tmp
```
### Drop Capabilities
Run with minimal privileges:
```yaml
services:
vaultlink-server:
image: ghcr.io/schmelczer/vault-link-server:latest
security_opt:
- no-new-privileges:true
cap_drop:
- ALL
```
## Migration
### Moving to New Server
1. **Backup on old server**:
```bash
./backup-vaultlink.sh
```
2. **Transfer backup**:
```bash
scp vaultlink-backup.tar.gz new-server:/tmp/
```
3. **Restore on new server**:
```bash
./restore-vaultlink.sh /tmp/vaultlink-backup.tar.gz
```
4. **Update DNS/clients** to point to new server
5. **Verify sync** on all clients
### Version Upgrades
```bash
# Pull latest image
docker pull ghcr.io/schmelczer/vault-link-server:latest
# Backup first
./backup-vaultlink.sh
# Stop old container
docker stop vaultlink-server
docker rm vaultlink-server
# Start with new image
docker run -d \
--name vaultlink-server \
--restart unless-stopped \
-p 3000:3000 \
-v ./data:/data \
ghcr.io/schmelczer/vault-link-server:latest \
/app/sync_server /data/config.yml
# Check logs
docker logs -f vaultlink-server
```
## Next Steps
- [Understand the architecture →](/architecture/)
- [Deploy the server →](/guide/server-setup)
- [Configure clients →](/guide/obsidian-plugin)

View file

@ -0,0 +1,558 @@
# Authentication Configuration
VaultLink uses token-based authentication with per-user vault access control. This guide covers all authentication and authorization options.
## Overview
Authentication in VaultLink:
- **Token-based**: Users authenticate with secure tokens
- **Configured in YAML**: All users defined in `config.yml`
- **Vault-level access**: Control which vaults each user can access
- **No password hashing**: Tokens are treated as secrets
## Basic Configuration
```yaml
users:
user_configs:
- name: alice
token: alice-secure-token-here
vault_access:
type: allow_access_to_all
```
## User Configuration Fields
### `name`
**Type**: String
**Required**: Yes
Human-readable identifier for the user. Used in logs and auditing.
```yaml
- name: alice
```
**Notes**:
- Must be unique across all users
- Used for identification only, not authentication
- Appears in server logs
- Can be any string (e.g., email, username)
### `token`
**Type**: String
**Required**: Yes
Authentication token for the user. Must be kept secret.
```yaml
- token: 1a2b3c4d5e6f7g8h9i0j...
```
**Best practices**:
- Generate with: `openssl rand -hex 32`
- Minimum length: 32 characters
- Use different token per user
- Never commit to version control
- Rotate periodically
**Example token generation**:
```bash
# Generate a secure token
openssl rand -hex 32
# Output: a7f3c9d1e8b2f4a6c3d9e1f7b8a4c2d6e9f1a3b7c5d8e2f4a6b9c3d1e8f7a4b2
```
### `vault_access`
**Type**: Object
**Required**: Yes
Defines which vaults the user can access.
**Three modes**:
1. `allow_access_to_all`: Access to all vaults
2. `allow_list`: Access to specific vaults only
3. `deny_list`: Access to all vaults except specific ones
## Access Control Modes
### Allow Access to All
Grant access to every vault:
```yaml
users:
user_configs:
- name: admin
token: admin-token
vault_access:
type: allow_access_to_all
```
**Use cases**:
- Administrator accounts
- Personal single-user deployments
- Development/testing
### Allow List
Grant access only to specific vaults:
```yaml
users:
user_configs:
- name: alice
token: alice-token
vault_access:
type: allow_list
allowed:
- personal
- shared-team
- project-alpha
```
**Use cases**:
- Multi-user deployments
- Restricted access scenarios
- Separation of concerns
**Notes**:
- User can only access listed vaults
- Attempting to access other vaults returns authentication error
- Empty list = no access to any vault
### Deny List
Grant access to all vaults except specific ones:
```yaml
users:
user_configs:
- name: bob
token: bob-token
vault_access:
type: deny_list
denied:
- restricted
- admin-only
```
**Use cases**:
- Users with broad access except sensitive vaults
- Simplify configuration when most vaults are accessible
**Notes**:
- User can access any vault not in the deny list
- Attempting to access denied vaults returns authentication error
## Multi-User Scenarios
### Personal Use (Single User)
```yaml
users:
user_configs:
- name: me
token: my-super-secret-token
vault_access:
type: allow_access_to_all
```
### Small Team (Shared Vaults)
```yaml
users:
user_configs:
- name: alice
token: alice-token
vault_access:
type: allow_list
allowed:
- personal-alice
- team-shared
- name: bob
token: bob-token
vault_access:
type: allow_list
allowed:
- personal-bob
- team-shared
- name: charlie
token: charlie-token
vault_access:
type: allow_list
allowed:
- personal-charlie
- team-shared
```
### Organization (Mixed Access)
```yaml
users:
user_configs:
- name: admin
token: admin-token
vault_access:
type: allow_access_to_all
- name: developer
token: dev-token
vault_access:
type: allow_list
allowed:
- engineering-docs
- api-specs
- shared
- name: designer
token: design-token
vault_access:
type: allow_list
allowed:
- design-docs
- brand-assets
- shared
- name: readonly
token: readonly-token
vault_access:
type: allow_list
allowed:
- public-wiki
```
## Authentication Flow
### Connection
1. Client connects via WebSocket
2. Client sends authentication message:
```json
{
"type": "auth",
"token": "user-token",
"vault": "vault-name"
}
```
3. Server validates:
- Token exists in config
- User has access to requested vault
4. Server responds:
- Success: Connection established
- Failure: Connection closed with error
### Validation
Server checks:
1. **Token match**: Token exists in `user_configs`
2. **Vault access**: User has permission for vault
3. **Connection limits**: Not exceeding `max_clients_per_vault`
### Errors
**Invalid token**:
```
Authentication failed: Invalid token
```
**No vault access**:
```
Authentication failed: User does not have access to vault 'restricted'
```
**Connection limit**:
```
Connection rejected: Maximum clients reached for vault
```
## Security Best Practices
### Token Generation
Generate strong tokens:
```bash
# 64 character hex token (256 bits)
openssl rand -hex 32
# Base64 encoded (256 bits)
openssl rand -base64 32
# UUID v4
uuidgen
```
### Token Storage
**In config file**:
```yaml
users:
user_configs:
- name: alice
token: !ENV ALICE_TOKEN # Read from environment variable
```
**Load from environment**:
```bash
export ALICE_TOKEN="$(openssl rand -hex 32)"
./sync_server config.yml
```
### Token Rotation
Periodically change tokens:
1. Generate new token
2. Update `config.yml`
3. Restart server
4. Update clients with new token
### Token Revocation
To revoke access:
1. Remove user from `config.yml`
2. Restart server
3. User's connections will be rejected
For immediate revocation:
- Remove user from config
- Restart server
- Existing connections are terminated
## Access Patterns
### Read-Only Users
VaultLink doesn't distinguish read-only vs read-write. Implement via client:
```yaml
# Server: Grant access
users:
user_configs:
- name: readonly
token: readonly-token
vault_access:
type: allow_list
allowed:
- public
# Client: Use CLI in read-only mode (mount vault read-only)
docker run -v /vault:/vault:ro ...
```
### Temporary Access
Grant temporary access:
1. Add user to config
2. Set reminder to remove later
3. Remove user when no longer needed
4. Restart server
For automation:
```bash
# Add user with expiry comment
echo " - name: temp-user # EXPIRES: 2024-12-31" >> config.yml
echo " token: temp-token" >> config.yml
```
### Shared Tokens (Not Recommended)
Multiple users sharing a token:
- All appear as same user in logs
- Can't revoke individual access
- Security risk if one person leaves
**Instead**: Create separate users with same vault access.
## Monitoring
### Server Logs
Authentication events are logged:
```
2024-01-01 12:00:00 INFO User 'alice' authenticated for vault 'personal'
2024-01-01 12:00:05 WARN Authentication failed: Invalid token from 192.168.1.100
2024-01-01 12:00:10 WARN User 'bob' denied access to vault 'restricted'
```
### Audit Trail
Monitor authentication:
```bash
# View authentication logs
grep "authenticated" logs/*.log
# View failed authentications
grep "Authentication failed" logs/*.log
# View access denials
grep "denied access" logs/*.log
```
## Advanced Scenarios
### Multiple Servers
Same user across multiple server instances:
```yaml
# Server 1 config.yml
users:
user_configs:
- name: alice
token: alice-global-token
vault_access:
type: allow_list
allowed:
- vault-1
- vault-2
# Server 2 config.yml
users:
user_configs:
- name: alice
token: alice-global-token # Same token
vault_access:
type: allow_list
allowed:
- vault-3
- vault-4
```
### Service Accounts
Tokens for automated systems:
```yaml
users:
user_configs:
- name: backup-service
token: backup-service-token
vault_access:
type: allow_access_to_all
- name: ci-pipeline
token: ci-token
vault_access:
type: allow_list
allowed:
- documentation
- name: monitoring
token: monitoring-token
vault_access:
type: allow_list
allowed:
- metrics
```
### Dynamic Vault Access
VaultLink doesn't support runtime user management. To change access:
1. Update `config.yml`
2. Restart server
3. Users reconnect automatically
For frequent changes, consider:
- Over-provision access (deny list)
- Use external authentication proxy
- Script config updates + reload
## Troubleshooting
### Can't connect
**Check token**:
```bash
# Verify token in config matches client
grep "token:" config.yml
```
**Check vault name**:
```bash
# Ensure vault is in allowed list
grep -A 5 "name: alice" config.yml
```
**Check server logs**:
```bash
tail -f logs/*.log | grep -i auth
```
### Access denied
**Verify vault access**:
```yaml
# Check user's vault_access configuration
users:
user_configs:
- name: alice
vault_access:
type: allow_list
allowed:
- vault-name # Must match exactly
```
**Case sensitivity**:
- Vault names are case-sensitive
- `Vault``vault`
- Ensure exact match in config and client
### Token not working
**Check for typos**:
- Extra spaces
- Hidden characters
- Wrong quotes in YAML
**Regenerate token**:
```bash
# Generate new token
openssl rand -hex 32
# Update config
# Restart server
# Update client
```
## Next Steps
- [Server configuration reference →](/config/server)
- [Advanced configuration →](/config/advanced)
- [Deploy the server →](/guide/server-setup)

489
docs/config/server.md Normal file
View file

@ -0,0 +1,489 @@
# Server Configuration
Complete reference for configuring the VaultLink sync server via `config.yml`.
## Configuration File Format
The server is configured using a YAML file passed as a command-line argument:
```bash
/app/sync_server /path/to/config.yml
```
## Complete Example
```yaml
database:
databases_directory_path: databases
max_connections_per_vault: 12
cursor_timeout_seconds: 60
server:
host: 0.0.0.0
port: 3000
max_body_size_mb: 512
max_clients_per_vault: 256
response_timeout_seconds: 60
users:
user_configs:
- name: admin
token: your-secure-random-token
vault_access:
type: allow_access_to_all
- name: alice
token: alice-token
vault_access:
type: allow_list
allowed:
- personal
- shared
- name: bob
token: bob-token
vault_access:
type: deny_list
denied:
- restricted
logging:
log_directory: logs
log_rotation: 7days
```
## Database Section
### `databases_directory_path`
**Type**: String
**Required**: Yes
**Default**: None
Directory where SQLite database files are stored. One database file per vault.
```yaml
database:
databases_directory_path: /data/databases
```
The directory structure:
```
databases/
├── vault-1.db
├── vault-2.db
└── personal.db
```
**Notes**:
- Path is relative to working directory or absolute
- Directory must be writable by the server process
- Ensure adequate disk space for vault data
- Back up this directory regularly
### `max_connections_per_vault`
**Type**: Integer
**Required**: Yes
**Default**: None
**Recommended**: 12
Maximum concurrent database connections per vault.
```yaml
database:
max_connections_per_vault: 12
```
**Tuning**:
- Higher values: Better performance under load
- Lower values: Less memory usage
- Typical range: 8-20
- Consider: Number of concurrent users × average operations per user
### `cursor_timeout_seconds`
**Type**: Integer
**Required**: Yes
**Default**: None
**Recommended**: 60
How long to keep database cursors alive for inactive clients.
```yaml
database:
cursor_timeout_seconds: 60
```
**Notes**:
- Cursors track client sync state
- Timeout too short: Clients may need to re-sync frequently
- Timeout too long: More memory usage
- Typical range: 30-300 seconds
## Server Section
### `host`
**Type**: String
**Required**: Yes
**Default**: None
Network interface to bind the server to.
```yaml
server:
host: 0.0.0.0 # All interfaces
# OR
host: 127.0.0.1 # Localhost only
# OR
host: 192.168.1.100 # Specific interface
```
**Common values**:
- `0.0.0.0`: Listen on all network interfaces (production)
- `127.0.0.1`: Listen on localhost only (development/testing)
- Specific IP: Listen on specific interface
### `port`
**Type**: Integer
**Required**: Yes
**Default**: None
**Recommended**: 3000
TCP port to listen on.
```yaml
server:
port: 3000
```
**Notes**:
- Must be available (not in use)
- Privileged ports (< 1024) require root
- Common ports: 3000, 8080, 8888
- Configure firewall to allow this port
### `max_body_size_mb`
**Type**: Integer
**Required**: Yes
**Default**: None
**Recommended**: 512
Maximum size of HTTP request body in megabytes.
```yaml
server:
max_body_size_mb: 512
```
**Usage**:
- Limits file upload size
- Prevents memory exhaustion attacks
- Must be larger than largest expected file
- Consider client `max_file_size_mb` settings
**Tuning**:
- Small vaults (mostly text): 100 MB
- Medium vaults (some images): 512 MB
- Large vaults (many images/PDFs): 1024+ MB
### `max_clients_per_vault`
**Type**: Integer
**Required**: Yes
**Default**: None
**Recommended**: 256
Maximum concurrent clients per vault.
```yaml
server:
max_clients_per_vault: 256
```
**Notes**:
- Limits concurrent WebSocket connections
- Prevents resource exhaustion
- Consider expected number of users
- Each client uses memory and file descriptors
**Scaling**:
- Personal use: 10-50
- Small team: 50-100
- Large team: 100-500
### `response_timeout_seconds`
**Type**: Integer
**Required**: Yes
**Default**: None
**Recommended**: 60
Maximum time to wait for client responses.
```yaml
server:
response_timeout_seconds: 60
```
**Usage**:
- Timeout for HTTP requests
- Timeout for WebSocket operations
- Clients disconnected if unresponsive
**Tuning**:
- Fast networks: 30 seconds
- Slow networks: 90-120 seconds
- Large file uploads: Increase proportionally
## Users Section
See [Authentication Configuration →](/config/authentication) for detailed user configuration.
## Logging Section
### `log_directory`
**Type**: String
**Required**: Yes
**Default**: None
Directory where log files are written.
```yaml
logging:
log_directory: /data/logs
# OR
log_directory: logs # Relative to working directory
```
**Notes**:
- Path is relative to working directory or absolute
- Directory must be writable
- Logs are rotated based on `log_rotation`
- Monitor disk usage
### `log_rotation`
**Type**: String
**Required**: Yes
**Default**: None
How often to rotate log files.
```yaml
logging:
log_rotation: 7days
# OR
log_rotation: 24hours
# OR
log_rotation: 30days
```
**Format**: `<number><unit>`
**Units**:
- `hours`: Hours (e.g., `12hours`, `24hours`)
- `days`: Days (e.g., `7days`, `30days`)
**Recommendations**:
- Development: `24hours` or `7days`
- Production: `7days` or `30days`
- High traffic: `24hours` (logs can be large)
## Environment-Specific Configurations
### Development
```yaml
database:
databases_directory_path: ./databases
max_connections_per_vault: 8
cursor_timeout_seconds: 30
server:
host: 127.0.0.1
port: 3000
max_body_size_mb: 100
max_clients_per_vault: 10
response_timeout_seconds: 30
users:
user_configs:
- name: dev
token: dev-token
vault_access:
type: allow_access_to_all
logging:
log_directory: logs
log_rotation: 24hours
```
### Production
```yaml
database:
databases_directory_path: /data/databases
max_connections_per_vault: 16
cursor_timeout_seconds: 120
server:
host: 0.0.0.0
port: 3000
max_body_size_mb: 512
max_clients_per_vault: 256
response_timeout_seconds: 90
users:
user_configs:
- name: admin
token: <strong-random-token>
vault_access:
type: allow_access_to_all
# Additional users...
logging:
log_directory: /data/logs
log_rotation: 7days
```
## Validation
The server validates configuration on startup:
```bash
# Start server
./sync_server config.yml
# Check for errors in logs
tail -f logs/latest.log
```
**Common errors**:
- Missing required fields
- Invalid YAML syntax
- Invalid values (negative numbers, etc.)
- Directory not writable
## Performance Tuning
### High Concurrency
For many concurrent users:
```yaml
database:
max_connections_per_vault: 20 # Increase
server:
max_clients_per_vault: 500 # Increase
response_timeout_seconds: 120 # Increase for slow clients
```
### Large Files
For vaults with large files:
```yaml
server:
max_body_size_mb: 1024 # Allow larger uploads
response_timeout_seconds: 180 # More time for uploads
```
### Resource-Constrained Systems
For limited CPU/memory:
```yaml
database:
max_connections_per_vault: 6 # Reduce
server:
max_clients_per_vault: 50 # Reduce
max_body_size_mb: 256 # Reduce
```
## Security Considerations
### Token Security
- Use strong random tokens: `openssl rand -hex 32`
- Never commit tokens to version control
- Rotate tokens periodically
- Use different tokens per user
### Network Security
- Bind to `127.0.0.1` if using reverse proxy on same host
- Use firewall to restrict access
- Enable SSL/TLS via reverse proxy
### Resource Limits
- Set `max_clients_per_vault` to prevent DoS
- Set `max_body_size_mb` to prevent memory exhaustion
- Configure `response_timeout_seconds` to prevent hanging connections
## Troubleshooting
### Server won't start
**Check YAML syntax**:
```bash
# Use a YAML validator
python -c 'import yaml, sys; yaml.safe_load(open("config.yml"))'
```
**Check file paths**:
```bash
# Ensure directories exist and are writable
mkdir -p databases logs
chmod 755 databases logs
```
**Check port availability**:
```bash
# Verify port is not in use
lsof -i :3000
```
### High memory usage
- Reduce `max_connections_per_vault`
- Reduce `max_clients_per_vault`
- Reduce `max_body_size_mb`
- Check for large vaults or many concurrent users
### Slow performance
- Increase `max_connections_per_vault`
- Increase database connection pool
- Use SSD for database storage
- Monitor database size (vacuum if needed)
## Next Steps
- [Configure authentication →](/config/authentication)
- [Advanced configuration options →](/config/advanced)
- [Deploy the server →](/guide/server-setup)

324
docs/guide/alternatives.md Normal file
View file

@ -0,0 +1,324 @@
# Comparison with Alternatives
VaultLink is one of several solutions for synchronising Obsidian vaults. This page compares VaultLink with popular alternatives to help you choose the right tool.
## Key Differentiator: Editor Agnostic
**VaultLink is not tied to Obsidian.** While it includes an Obsidian plugin for convenience, VaultLink synchronises plain text files and works with any editor:
- Edit with **Obsidian desktop** on your laptop
- Edit with **Vim** on your server
- Edit with **VS Code** on your workstation
- Edit with **Obsidian mobile** on your phone
- Use the **CLI client** for automated workflows
All changes merge automatically without conflict markers, regardless of which editor you use. This is possible because VaultLink uses [reconcile-text](/architecture/sync-algorithm#why-reconcile-text-over-crdts) for differential synchronisation rather than requiring operation-level tracking.
## VaultLink's Core Strengths
Before diving into comparisons:
1. **Fully self-hosted**: Server and all components are open source
2. **Collaborative editing**: Real-time sync with operational transformation
3. **Automatic conflict resolution**: No manual intervention or paid features required
4. **Cursor tracking**: See where other users are editing
5. **Extensively tested**: Comprehensive test suite for server and client
6. **Editor freedom**: Use any text editor, not just Obsidian
7. **Production-ready**: Docker images, health checks, monitoring
## Obsidian Sync Alternatives
### Self-hosted LiveSync
**Downloads**: ~300,000
**Repository**: https://github.com/vrtmrz/obsidian-livesync
**Overview**: CouchDB/IBM Cloudant-based sync with end-to-end encryption.
| Aspect | Self-hosted LiveSync | VaultLink |
| ------------------------- | --------------------------- | -------------------------------------- |
| **Self-hosted** | Yes (CouchDB required) | Yes (single binary or Docker) |
| **Conflict resolution** | Manual or automatic (basic) | Automatic (operational transformation) |
| **Collaborative editing** | No | Yes (real-time with cursors) |
| **Editor support** | Obsidian only | Any text editor |
| **Infrastructure** | CouchDB database | SQLite (bundled) |
| **Deployment complexity** | Medium (external DB) | Low (single container) |
| **End-to-end encryption** | Yes | No (transport encryption only) |
| **Out-of-band edits** | Limited support | Full support (edit with any tool) |
**When to use LiveSync**:
- Need end-to-end encryption
- Already running CouchDB
- Only use Obsidian (no external editors)
**When to use VaultLink**:
- Want collaborative editing with multiple users
- Edit files with various tools (Vim, VS Code, etc.)
- Need simpler deployment (no external database)
- Want operational transformation for better merges
---
### Remotely Save
**Downloads**: ~1.1M
**Repository**: https://github.com/remotely-save/remotely-save
**Overview**: Sync to cloud storage providers (S3, Dropbox, OneDrive, WebDAV).
| Aspect | Remotely Save | VaultLink |
| ------------------------- | ---------------------------- | ------------------------ |
| **Self-hosted** | Partial (uses cloud storage) | Fully self-hosted |
| **Conflict resolution** | Paid Pro feature | Free and automatic |
| **Collaborative editing** | No | Yes |
| **Editor support** | Obsidian only | Any text editor |
| **Storage backend** | Cloud providers | Self-hosted SQLite |
| **Cost** | Free (basic) / Paid (Pro) | Free (open source) |
| **Code quality** | No tests, complex codebase | Comprehensive test suite |
| **Real-time sync** | No (periodic polling) | Yes (WebSocket) |
**When to use Remotely Save**:
- Already use cloud storage (S3, Dropbox)
- Don't need real-time sync
- Single-user scenario
**When to use VaultLink**:
- Want full control over data
- Need automatic conflict resolution without paying
- Want real-time collaborative editing
- Value code quality and testing
**Note**: Remotely Save's conflict resolution is a paid feature. VaultLink provides superior automatic merging for free.
---
### Relay
**Downloads**: ~24,000
**Repository**: https://github.com/No-Instructions/Relay
**Overview**: CRDT-based sync with proprietary server component.
| Aspect | Relay | VaultLink |
| -------------------------- | ---------------------------- | ----------------------- |
| **Self-hosted** | No (proprietary server) | Yes (fully open source) |
| **Conflict resolution** | CRDT (automatic) | OT (automatic) |
| **Collaborative editing** | Yes | Yes |
| **Editor support** | Obsidian only | Any text editor |
| **Out-of-band edits** | No (breaks CRDT consistency) | Yes (differential sync) |
| **Server open source** | No | Yes |
| **Infrastructure control** | Limited | Full |
| **Per-file overhead** | High (CRDT metadata) | Low (version history) |
**When to use Relay**:
- Want hosted solution (don't self-host)
- Only edit within Obsidian
- Don't need out-of-band editing
**When to use VaultLink**:
- Need fully open source solution
- Want to self-host completely
- Edit files outside Obsidian (Vim, VS Code)
- Value infrastructure control
**Critical limitation**: Relay's CRDT approach requires tracking every operation within Obsidian. Editing files outside Obsidian breaks the CRDT state. VaultLink's differential sync works regardless of how files are edited.
---
### Obsidian Git
**Downloads**: ~1.4M
**Repository**: https://github.com/denolehov/obsidian-git
**Overview**: Uses Git for version control and synchronisation.
| Aspect | Obsidian Git | VaultLink |
| ------------------------- | ----------------------------- | ----------------------- |
| **Self-hosted** | Yes (Git server) | Yes (sync server) |
| **Conflict resolution** | Manual (conflict markers) | Automatic (no markers) |
| **Collaborative editing** | No | Yes (real-time) |
| **Editor support** | Any (it's Git) | Any (differential sync) |
| **Version history** | Full Git history | Document versions |
| **Real-time sync** | No (commit-based) | Yes (instant) |
| **Merge conflicts** | Manual resolution | Automatic |
| **Learning curve** | High (Git knowledge required) | Low |
| **Workflow interruption** | Yes (resolve conflicts) | No |
**When to use Obsidian Git**:
- Need full version control (branches, tags, etc.)
- Already familiar with Git workflows
- Want integration with existing Git repos
- Don't mind manual conflict resolution
**When to use VaultLink**:
- Want automatic conflict-free merging
- Need real-time collaborative editing
- Don't want workflow interruptions from merge conflicts
- Prefer simpler mental model (sync, not commits)
**Key difference**: Git requires manual conflict resolution with `<<<<<<<` markers. VaultLink automatically merges all changes using operational transformation, never interrupting your workflow.
---
### Syncthing Integration
**Downloads**: ~22,600
**Repository**: https://github.com/LBF38/obsidian-syncthing-integration
**Overview**: Wrapper around Syncthing for file synchronisation.
| Aspect | Syncthing Integration | VaultLink |
| ------------------------- | ------------------------------ | ----------------- |
| **Self-hosted** | Yes (Syncthing) | Yes (sync server) |
| **Conflict resolution** | Manual | Automatic |
| **Collaborative editing** | No | Yes |
| **Editor support** | Any | Any |
| **Status** | Unfinished | Production-ready |
| **Conflict files** | Creates `.sync-conflict` files | No conflict files |
| **Real-time sync** | Yes | Yes |
| **Automatic merging** | No | Yes |
**When to use Syncthing Integration**:
- Already use Syncthing for other files
- Don't need automatic conflict resolution
- Single-user with multiple devices
**When to use VaultLink**:
- Want automatic conflict resolution
- Need collaborative editing
- Want production-ready solution
- Don't want to manage conflict files
**Status note**: Syncthing Integration is marked as unfinished. VaultLink is production-ready with comprehensive testing.
---
### Remotely Sync
**Downloads**: ~38,000
**Repository**: https://github.com/sboesen/remotely-sync
**Overview**: Similar to Remotely Save, syncs to cloud storage.
| Aspect | Remotely Sync | VaultLink |
| ----------------------- | ----------------------- | ------------------- |
| **Self-hosted** | Partial (cloud storage) | Fully self-hosted |
| **Conflict resolution** | Limited/Paid | Free and automatic |
| **Code quality** | No tests | Comprehensive tests |
| **Maintenance** | Low activity | Active development |
**Same concerns as Remotely Save**: No test suite, conflict resolution limitations, cloud storage dependency.
**When to use VaultLink**: See Remotely Save comparison above.
---
### SyncFTP
**Downloads**: ~5,000
**Repository**: https://github.com/alex-donnan/SyncFTP
**Overview**: Simple FTP-based file synchronisation.
| Aspect | SyncFTP | VaultLink |
| ------------------------- | ---------------------- | ---------------- |
| **Conflict resolution** | None (last write wins) | Automatic (OT) |
| **Data loss risk** | High (overwrites) | None (merges) |
| **Collaborative editing** | No | Yes |
| **Sophistication** | Minimal | Production-grade |
**When to use SyncFTP**: Don't use SyncFTP for any scenario where data integrity matters.
**When to use VaultLink**: Any scenario requiring reliable synchronisation.
---
## Feature Comparison Matrix
| Feature | VaultLink | LiveSync | Relay | Git | Remotely Save | Syncthing |
| --------------------------------- | --------- | -------- | ----- | --- | ------------- | --------- |
| **Fully open source** | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ |
| **Self-hosted** | ✅ | ✅ | ❌ | ✅ | Partial | ✅ |
| **Automatic conflict resolution** | ✅ | Basic | ✅ | ❌ | Paid | ❌ |
| **Real-time sync** | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ |
| **Collaborative editing** | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ |
| **Cursor tracking** | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
| **Editor agnostic** | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ |
| **Out-of-band edits** | ✅ | Limited | ❌ | ✅ | ❌ | ✅ |
| **No conflict markers** | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
| **Comprehensive tests** | ✅ | ❌ | ❌ | N/A | ❌ | N/A |
| **Simple deployment** | ✅ | ❌ | N/A | ❌ | ✅ | ❌ |
| **Low infrastructure** | ✅ | ❌ | N/A | ✅ | ✅ | ✅ |
---
## VaultLink's Unique Position
VaultLink is the **only** solution that combines:
1. **Fully open source** self-hosted server
2. **Editor agnostic** operation (not locked to Obsidian)
3. **Automatic conflict-free merging** using operational transformation
4. **Real-time collaborative editing** with cursor tracking
5. **Differential synchronisation** supporting out-of-band edits
6. **Comprehensive test coverage** ensuring reliability
7. **Simple deployment** via Docker or single binary
## Use Case Recommendations
### Choose VaultLink when you:
- Edit vaults with multiple editors (Obsidian + Vim + VS Code)
- Need real-time collaboration with teammates
- Want automatic conflict resolution without manual intervention
- Value full control over infrastructure
- Need production-ready reliability with comprehensive testing
- Want to edit files while offline and sync later seamlessly
### Consider alternatives when you:
- **LiveSync**: Need end-to-end encryption and only use Obsidian
- **Git**: Need full version control with branches and advanced Git features
- **Remotely Save**: Already committed to cloud storage providers
- **Syncthing**: Already use Syncthing and don't need automatic merging
## Migration from Other Solutions
VaultLink works with plain Markdown files, making migration simple:
1. **From Git**: Clone your repo, point VaultLink to the directory
2. **From cloud sync**: Download files, configure VaultLink client
3. **From LiveSync**: Export vault, import to VaultLink
4. **From Syncthing**: Point VaultLink to synced directory
All solutions work with the same Markdown files—VaultLink just syncs them better.
## Beyond Obsidian
Because VaultLink is editor-agnostic, you can use it for:
- **Documentation teams**: Sync technical docs edited in VS Code
- **Academic writing**: Collaborate on papers with various Markdown editors
- **Personal knowledge bases**: Use Obsidian on mobile, Vim on servers
- **Automated workflows**: CLI client for backup systems and CI/CD
- **Multi-tool workflows**: Different team members use different editors
VaultLink doesn't lock you into Obsidian—it's a general-purpose differential sync system that happens to work excellently with Obsidian vaults.
## Next Steps
Ready to try VaultLink?
- [Get started →](/guide/getting-started)
- [Understand the architecture →](/architecture/)
- [See how sync works →](/architecture/sync-algorithm)

532
docs/guide/cli-client.md Normal file
View file

@ -0,0 +1,532 @@
# CLI Client
Sync vaults without Obsidian. Works on servers, automation, backups, headless systems.
## Installation
### Docker (Recommended)
Pull the latest image:
```bash
docker pull ghcr.io/schmelczer/vault-link-cli:latest
```
### npm
Install globally:
```bash
npm install -g @schmelczer/local-client-cli
```
Verify installation:
```bash
vaultlink --version
```
### From Source
Build from the repository:
```bash
git clone https://github.com/schmelczer/vault-link.git
cd vault-link/frontend/local-client-cli
npm install
npm run build
node dist/cli.js --help
```
## Usage
### Basic Usage
```bash
vaultlink \
--local-path /path/to/vault \
--remote-uri wss://sync.example.com \
--token your-auth-token \
--vault-name default
```
### Docker Usage
```bash
docker run -v /path/to/vault:/vault \
ghcr.io/schmelczer/vault-link-cli:latest \
-l /vault \
-r wss://sync.example.com \
-t your-auth-token \
-v default
```
### Docker Compose
Create `docker-compose.yml`:
```yaml
services:
vaultlink-cli:
image: ghcr.io/schmelczer/vault-link-cli:latest
restart: unless-stopped
volumes:
- ./vault:/vault
command:
- "-l"
- "/vault"
- "-r"
- "wss://sync.example.com"
- "-t"
- "your-token"
- "-v"
- "default"
```
Start the client:
```bash
docker compose up -d
```
## Configuration Options
### Required Arguments
| Argument | Short | Description | Example |
| -------------- | ----- | ----------------------- | ------------------------ |
| `--local-path` | `-l` | Local directory to sync | `/vault` |
| `--remote-uri` | `-r` | Server WebSocket URI | `wss://sync.example.com` |
| `--token` | `-t` | Authentication token | `abc123...` |
| `--vault-name` | `-v` | Vault name on server | `default` |
### Optional Arguments
| Argument | Default | Description |
| ------------------------------- | ------- | -------------------------------------- |
| `--sync-concurrency` | `1` | Concurrent file operations |
| `--max-file-size-mb` | `10` | Max file size in MB |
| `--ignore-pattern` | - | Glob pattern to ignore (repeatable) |
| `--websocket-retry-interval-ms` | `3500` | Reconnection interval |
| `--log-level` | `INFO` | Log level: DEBUG, INFO, WARNING, ERROR |
### Environment Variables
Alternative to command-line arguments:
```bash
export VAULTLINK_LOCAL_PATH="/vault"
export VAULTLINK_REMOTE_URI="wss://sync.example.com"
export VAULTLINK_TOKEN="your-token"
export VAULTLINK_VAULT_NAME="default"
vaultlink
```
## Examples
### Basic Sync
Sync a local directory to the server:
```bash
vaultlink \
-l ./my-notes \
-r wss://sync.example.com \
-t my-secure-token \
-v personal
```
### With Ignore Patterns
Exclude specific files or directories:
```bash
vaultlink \
-l ./vault \
-r wss://sync.example.com \
-t token123 \
-v default \
--ignore-pattern "*.tmp" \
--ignore-pattern ".DS_Store" \
--ignore-pattern "node_modules/**"
```
### Debug Logging
Enable verbose logging:
```bash
vaultlink \
-l ./vault \
-r wss://sync.example.com \
-t token123 \
-v default \
--log-level DEBUG
```
### High Concurrency
Faster initial sync:
```bash
vaultlink \
-l ./vault \
-r wss://sync.example.com \
-t token123 \
-v default \
--sync-concurrency 5
```
### Large Files
Allow larger file uploads:
```bash
vaultlink \
-l ./vault \
-r wss://sync.example.com \
-t token123 \
-v default \
--max-file-size-mb 50
```
## Docker Deployment
### Long-Running Sync
Run as a daemon for continuous synchronisation:
```bash
docker run -d \
--name vaultlink-sync \
--restart unless-stopped \
-v $(pwd)/vault:/vault \
ghcr.io/schmelczer/vault-link-cli:latest \
-l /vault \
-r wss://sync.example.com \
-t your-token \
-v default
```
Monitor logs:
```bash
docker logs -f vaultlink-sync
```
### Health Monitoring
The Docker image includes built-in health checks:
```bash
# Check health status
docker ps
# View detailed health info
docker inspect --format='{{json .State.Health}}' vaultlink-sync | jq
```
Health check verifies:
- Health file exists
- Status updated within last 30 seconds
- WebSocket connection is active
Configure custom health check:
```yaml
services:
vaultlink-cli:
image: ghcr.io/schmelczer/vault-link-cli:latest
healthcheck:
test: ["CMD", "node", "/app/healthcheck.js"]
interval: 15s
timeout: 5s
retries: 5
start_period: 20s
```
### Read-Only Vault
Mount vault as read-only to prevent local changes:
```bash
docker run -d \
-v $(pwd)/vault:/vault:ro \
ghcr.io/schmelczer/vault-link-cli:latest \
-l /vault \
-r wss://sync.example.com \
-t token \
-v default
```
::: warning
The CLI needs write access to create `.vaultlink` metadata directory. Mount as read-write or provide a separate writeable directory.
:::
## How It Works
### Initial Sync
On startup:
1. Creates `.vaultlink/` directory for metadata
2. Scans local filesystem
3. Uploads all local files to server
4. Downloads files from server not present locally
5. Resolves conflicts using operational transformation
### Real-Time Synchronization
After initial sync:
1. Watches filesystem for changes using `fs.watch`
2. Uploads changed files immediately
3. Receives real-time updates from server via WebSocket
4. Handles bidirectional sync automatically
### Graceful Shutdown
On SIGINT (Ctrl+C) or SIGTERM:
1. Completes pending uploads
2. Closes WebSocket connection cleanly
3. Flushes metadata to disk
4. Exits gracefully
## Use Cases
### Automated Backups
Continuously backup vaults to a remote server:
```bash
docker run -d \
--name vault-backup \
-v /important/notes:/vault:ro \
ghcr.io/schmelczer/vault-link-cli:latest \
-l /vault -r wss://backup.example.com -t backup-token -v backups
```
### CI/CD Documentation
Sync documentation in automated pipelines:
```bash
# In your CI pipeline
docker run \
-v $(pwd)/docs:/vault \
ghcr.io/schmelczer/vault-link-cli:latest \
-l /vault -r wss://docs.example.com -t ci-token -v prod-docs
```
### Multi-Location Sync
Sync between different geographic locations:
```bash
# Location A
vaultlink -l /data/vault -r wss://hub.example.com -t token -v shared
# Location B
vaultlink -l /backup/vault -r wss://hub.example.com -t token -v shared
```
### Development Environment
Keep documentation in sync across dev environments:
```bash
# In docker-compose.yml for your dev stack
services:
docs-sync:
image: ghcr.io/schmelczer/vault-link-cli:latest
volumes:
- ./docs:/vault
command: ["-l", "/vault", "-r", "wss://docs-server", "-t", "dev-token", "-v", "dev"]
```
## Troubleshooting
### Client won't connect
**Check server accessibility**:
```bash
curl https://sync.example.com/vaults/test/ping
```
**Verify WebSocket protocol**:
- Use `ws://` for HTTP servers
- Use `wss://` for HTTPS servers
**Check authentication**:
- Token must match server config
- User must have access to the vault
### Permission errors
**Docker volume permissions**:
```bash
# Ensure directory is writable
chmod 755 /path/to/vault
# Check Docker user ID
docker run --rm ghcr.io/schmelczer/vault-link-cli:latest id
```
**SELinux issues**:
```bash
# Add :z flag to volume mount
docker run -v /path/to/vault:/vault:z ...
```
### Files not syncing
**Check ignore patterns**:
- View logs to see which files are skipped
- Ensure patterns don't match unintentionally
**File size limits**:
- Check `--max-file-size-mb` setting
- Large files are skipped with a warning
**Check metadata**:
```bash
# View sync metadata
cat /path/to/vault/.vaultlink/metadata.json
```
### High memory usage
**Reduce concurrency**:
```bash
--sync-concurrency 1
```
**Limit file sizes**:
```bash
--max-file-size-mb 5
```
**Check vault size**:
- Very large vaults may need more resources
- Consider splitting into multiple vaults
### Connection keeps dropping
**Increase retry interval**:
```bash
--websocket-retry-interval-ms 5000
```
**Check network stability**:
```bash
# Monitor connection
docker logs -f vaultlink-sync | grep -i websocket
```
**Server timeout settings**:
- Verify reverse proxy WebSocket timeout
- Check server `response_timeout_seconds`
## Advanced Usage
### Custom Healthcheck Script
Create your own health monitoring:
```bash
#!/bin/bash
HEALTH_FILE="/tmp/vaultlink-health.json"
if [ ! -f "$HEALTH_FILE" ]; then
exit 1
fi
# Check file is recent (within 60 seconds)
if [ $(( $(date +%s) - $(stat -c %Y "$HEALTH_FILE") )) -gt 60 ]; then
exit 1
fi
# Check WebSocket is connected
if ! jq -e '.connected == true' "$HEALTH_FILE" > /dev/null; then
exit 1
fi
exit 0
```
### Automated Recovery
Restart on failure with exponential backoff:
```bash
#!/bin/bash
RETRY_DELAY=5
while true; do
vaultlink -l /vault -r wss://server -t token -v default
echo "Client exited, restarting in ${RETRY_DELAY}s..."
sleep $RETRY_DELAY
# Exponential backoff up to 5 minutes
RETRY_DELAY=$((RETRY_DELAY * 2))
if [ $RETRY_DELAY -gt 300 ]; then
RETRY_DELAY=300
fi
done
```
### Integration with systemd
Create `/etc/systemd/system/vaultlink-cli.service`:
```ini
[Unit]
Description=VaultLink CLI Sync
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
Restart=always
RestartSec=10
Environment="VAULTLINK_LOCAL_PATH=/data/vault"
Environment="VAULTLINK_REMOTE_URI=wss://sync.example.com"
Environment="VAULTLINK_TOKEN=your-token"
Environment="VAULTLINK_VAULT_NAME=default"
ExecStart=/usr/local/bin/vaultlink
[Install]
WantedBy=multi-user.target
```
Enable and start:
```bash
sudo systemctl daemon-reload
sudo systemctl enable vaultlink-cli
sudo systemctl start vaultlink-cli
```
## Next Steps
- [Configure server authentication →](/config/authentication)
- [Learn about the sync algorithm →](/architecture/sync-algorithm)
- [Set up Obsidian plugin →](/guide/obsidian-plugin)

View file

@ -0,0 +1,125 @@
# Getting Started
Set up VaultLink in 5 minutes. Deploy server, connect clients, done.
## Prerequisites
- Docker (or Rust toolchain if building from source)
- A server (VPS, home server, or localhost for testing)
## Step 1: Deploy Server
Create `config.yml`:
```yaml
database:
databases_directory_path: databases
max_connections_per_vault: 12
cursor_timeout_seconds: 60
server:
host: 0.0.0.0
port: 3000
max_body_size_mb: 512
max_clients_per_vault: 256
response_timeout_seconds: 60
users:
user_configs:
- name: admin
token: change-this-to-secure-random-token
vault_access:
type: allow_access_to_all
logging:
log_directory: logs
log_rotation: 7days
```
::: tip
Generate secure token: `openssl rand -hex 32`
:::
Run server:
```bash
docker run -d \
--name vaultlink-server \
--restart unless-stopped \
-p 3000:3000 \
-v $(pwd):/data \
ghcr.io/schmelczer/vault-link-server:latest \
/app/sync_server /data/config.yml
```
Verify: `curl http://localhost:3000/vaults/test/ping` should return server version and auth status
## Step 2: Connect Client
### Obsidian Plugin
1. Settings → Community Plugins → Browse
2. Search "VaultLink", install, enable
3. Configure:
- Server URL: `ws://localhost:3000` (or `wss://your-server.com` for SSL)
- Token: Your token from config.yml
- Vault Name: `default`
[Full plugin guide →](/guide/obsidian-plugin)
### CLI Client
```bash
docker run -d \
--name vaultlink-cli \
--restart unless-stopped \
-v /path/to/vault:/vault \
ghcr.io/schmelczer/vault-link-cli:latest \
-l /vault -r ws://localhost:3000 -t your-token -v default
```
[Full CLI guide →](/guide/cli-client)
## Production Setup
For production:
1. **SSL/TLS**: Use Nginx/Caddy reverse proxy for `wss://` ([setup guide](/guide/server-setup#ssl-tls-with-reverse-proxy))
2. **Secure tokens**: Generate with `openssl rand -hex 32`, don't reuse the example
3. **Firewall**: Only expose port 3000 to reverse proxy
4. **Backups**: SQLite databases are in `databases/` directory
## Multiple Users
```yaml
users:
user_configs:
- name: alice
token: alice-token
vault_access:
type: allow_list
allowed:
- personal
- shared
- name: bob
token: bob-token
vault_access:
type: allow_list
allowed:
- shared
```
[Auth docs →](/config/authentication)
## Troubleshooting
**Server won't start**: `docker logs vaultlink-server`
**Client can't connect**:
1. Verify server: `curl http://your-server:3000/vaults/test/ping`
2. Check URL: `ws://` for HTTP, `wss://` for HTTPS
3. Verify token matches config.yml
**Understanding limitations**: [See what VaultLink can and can't do →](/guide/limitations)
**Files not syncing**: Check client logs, verify vault name matches
[Server setup →](/guide/server-setup) | [Architecture →](/architecture/)

192
docs/guide/limitations.md Normal file
View file

@ -0,0 +1,192 @@
# Limitations
VaultLink works well for most Obsidian vaults, but has some constraints you should know about.
## File Type Limitations
### Mergeable Files
Only **`.md`** and **`.txt`** files get automatic conflict-free merging.
Other file types (images, PDFs, etc.) use last-write-wins:
```
User A updates diagram.png → Server stores version 1
User B updates diagram.png → Server stores version 2 (overwrites A's changes)
```
**Workaround**: Avoid editing the same non-text file simultaneously.
### Binary Detection
Files are treated as binary if they:
- Contain NUL bytes (`0x00`)
- Fail UTF-8 validation
Binary files within `.md` or `.txt` extensions still get last-write-wins (no merge).
## Performance Constraints
### Server Limits (Configurable)
| Resource | Default | Maximum Tested |
| ------------------------ | ------- | -------------- |
| Clients per vault | 256 | ~256 |
| Database connections | 12 | 20 |
| Max file size | 512 MB | 4096 MB |
| Request timeout | 60s | 180s |
| WebSocket cursor timeout | 60s | 300s |
| Database busy timeout | 3600s | - |
### Vault Size
- **Small vaults** (< 1000 files): Excellent performance
- **Medium vaults** (1000-10000 files): Good performance
- **Large vaults** (> 10000 files): Works, but initial sync slower
No hard file count limit—constrained by disk space and sync time.
### Resource Usage
Rough estimates (varies by vault size and activity):
- **RAM**: ~50-200 MB base + ~1-5 MB per active client
- **CPU**: Low (< 5%) for typical usage, spikes during merges
- **Disk**: Vault size + version history (grows over time)
## Version History
### Storage
- All versions stored indefinitely (no automatic cleanup)
- Each vault is a separate SQLite database
- Deleted files marked as deleted (not purged)
**Growth**: Version history grows with every change. A 10 MB vault with frequent edits might grow to 100+ MB over months.
**Cleanup**: Manual only (see [Advanced Configuration](/config/advanced#version-history-cleanup)).
### Implications
- Disk usage grows over time
- Database size affects backup time
- No built-in retention policy
## Merge Quality
### Text Merging
VaultLink uses word-level tokenisation for merging:
```markdown
Parent: "The quick brown fox"
User A: "The quick red fox"
User B: "The very quick brown fox"
Result: "The very quick red fox" ← Both changes preserved
```
**Imperfect scenarios**:
- Complex nested Markdown (tables, code blocks)
- Simultaneous edits to the same sentence
- Large structural changes (moving sections around)
**Result**: Merged file might need manual cleanup in ~1-5% of concurrent edits.
## Scalability
### SQLite Limitations
- One SQLite database per vault
- Single-server architecture (no built-in clustering)
- Write serialisation through database
**For high concurrency**: Consider multiple vaults instead of one massive shared vault.
### Horizontal Scaling
Not currently supported. Running multiple servers requires manual vault partitioning.
## Network Requirements
### Latency
- Real-time sync typically < 500ms on good connections
- Mobile/slow networks: 1-5s latency possible
- Timeout failures on very slow connections (> 60s)
### Offline Behaviour
- Clients queue changes locally
- On reconnect, sync all changes since last connection
- Conflicts resolved automatically (for mergeable files)
**Limitation**: No offline conflict preview—merged result appears after reconnect.
## Security
### No End-to-End Encryption
- Server sees all file contents
- Transport encryption only (WSS/TLS)
- Trust your server
**Workaround**: Self-host on infrastructure you control.
### Authentication
- Token-based only (no OAuth, SAML, etc.)
- Tokens configured in server config file
- No runtime user management
## Known Edge Cases
### Simultaneous Deletes and Edits
```
User A deletes note.md
User B edits note.md
Result: Edit wins (file recreated with B's content)
```
Operational transformation prioritises content preservation.
### Large File Uploads
Files > 100 MB may time out on slow connections. Increase `response_timeout_seconds` or split large files.
### Mobile Sync
- Mobile networks may drop WebSocket connections frequently
- Client auto-reconnects, but causes sync delays
- Battery impact from constant reconnections
## What VaultLink is NOT
- **Not a backup solution**: Version history helps but isn't a backup (make backups!)
- **Not Git**: No branching, no commit messages, no diffs to review before merge
- **Not encrypted storage**: Server sees everything
- **Not multi-master**: One server, multiple clients (not peer-to-peer)
## Recommendations
### Good Use Cases
- Personal multi-device sync (< 10 devices)
- Small team collaboration (< 20 people)
- Primarily text/Markdown content
- Trusted server environment
### Poor Use Cases
- Large teams (> 50 concurrent users per vault)
- Primarily binary files (images, videos, large PDFs)
- Untrusted server (need E2E encryption)
- Highly regulated environments (HIPAA, etc.)
## Next Steps
- [Server configuration limits →](/config/server)
- [Advanced tuning →](/config/advanced)
- [Architecture details →](/architecture/)

View file

@ -0,0 +1,276 @@
# Obsidian Plugin
Real-time sync for Obsidian vaults.
## Installation
### From Obsidian Community Plugins
1. Open Obsidian Settings
2. Navigate to **Community Plugins**
3. Click **Browse** and search for "VaultLink"
4. Click **Install**
5. Enable the plugin
### Manual Installation
1. Download the latest release from [GitHub Releases](https://github.com/schmelczer/vault-link/releases)
2. Extract `main.js`, `manifest.json`, and `styles.css`
3. Copy to `.obsidian/plugins/vault-link/` in your vault
4. Reload Obsidian
5. Enable VaultLink in Community Plugins settings
## Configuration
After installation, configure the plugin in **Settings → VaultLink**.
### Required Settings
#### Server URL
The WebSocket URL of your sync server.
- **Development/Local**: `ws://localhost:3000`
- **Production (SSL)**: `wss://sync.example.com`
::: tip
Use `ws://` for unencrypted connections and `wss://` for SSL connections (production).
:::
#### Authentication Token
Your authentication token from the server's `config.yml`.
Generate a secure token:
```bash
openssl rand -hex 32
```
#### Vault Name
The name of the vault on the server. Can be any string.
Multiple Obsidian vaults can sync to the same server vault name (for shared vaults), or use unique names for separate vaults.
### Optional Settings
#### Sync Concurrency
Number of files to sync simultaneously.
- **Default**: 1
- **Range**: 1-10
- Higher values = faster initial sync, more resource usage
#### Max File Size
Maximum file size to sync (in MB).
- **Default**: 10
- Files larger than this are skipped
#### Ignore Patterns
Glob patterns for files to exclude from sync.
Examples:
- `*.tmp` - Ignore temporary files
- `.trash/**` - Ignore trash folder
- `private/**` - Ignore private directory
#### WebSocket Retry Interval
Milliseconds between reconnection attempts when disconnected.
- **Default**: 3500ms
- Increase for flaky networks to avoid connection spam
## Usage
### Initial Sync
When first connecting:
1. The plugin uploads all local files to the server
2. Downloads any missing files from the server
3. Resolves any conflicts using operational transformation
4. Begins real-time synchronisation
Initial sync time depends on vault size and `sync_concurrency` setting.
### Real-Time Sync
Once connected:
- **File changes**: Automatically synced when saved
- **File creation**: New files immediately uploaded
- **File deletion**: Deletions propagated to other clients
- **File renames**: Tracked and synchronised
The plugin watches your vault filesystem and syncs changes in real-time via WebSocket.
### Status Indicators
The plugin provides visual feedback:
- **Connected**: Green status in settings
- **Syncing**: Progress indicator during uploads
- **Disconnected**: Red status, automatic reconnection attempts
- **Error**: Error message in settings and console
Check the Obsidian console (Ctrl+Shift+I / Cmd+Option+I) for detailed logs.
## Features
### Automatic Conflict Resolution
When multiple users edit the same file simultaneously, operational transformation merges changes automatically:
- All edits are preserved
- No manual conflict resolution required
- Changes appear in real-time as others type
### Mobile Support
VaultLink works on Obsidian mobile (iOS and Android):
- Same configuration as desktop
- Real-time sync across all devices
- Handle network changes gracefully
::: warning
Ensure your sync server is accessible from mobile networks (use WSS with a public domain or VPN).
:::
### Offline Support
The plugin handles offline scenarios:
- Continue working when disconnected
- Changes queue locally
- Automatic sync when connection restored
- Conflict resolution if others edited the same files
## Collaboration Workflows
### Personal Multi-Device Sync
Sync the same vault across devices:
1. Configure each Obsidian instance with the same vault name
2. Use the same authentication token
3. All devices stay in sync automatically
### Team Shared Vault
Multiple users collaborating:
1. Each user has their own token (configured in server `config.yml`)
2. All users connect to the same vault name
3. Real-time collaborative editing with automatic conflict resolution
### Selective Sharing
Share specific folders while keeping others private:
1. Use different vault names for shared vs. private content
2. Configure access control on the server per vault
3. Use ignore patterns to exclude sensitive directories
## Troubleshooting
### Plugin won't connect
1. **Verify server is running**:
```bash
curl http://your-server:3000/vaults/test/ping
```
Should return `pong`
2. **Check URL format**:
- Local: `ws://localhost:3000`
- Remote (SSL): `wss://sync.example.com`
- Don't include `/vault/name` in the URL
3. **Verify token**:
- Must match server config exactly
- No extra spaces or quotes
- Check server logs for authentication errors
4. **Check firewall**:
- Ensure port is accessible from your network
- For mobile, server must be publicly accessible or use VPN
### Files not syncing
1. **Check ignore patterns**: File may match an exclusion pattern
2. **File size**: Check if file exceeds `max_file_size_mb`
3. **Permissions**: Ensure vault directory is readable/writable
4. **Console errors**: Open dev tools (Ctrl+Shift+I) and check console
### Slow initial sync
1. **Increase concurrency**: Set `sync_concurrency` higher (e.g., 5)
2. **Network speed**: Check internet connection
3. **Server resources**: Ensure server isn't overloaded
4. **Large files**: Consider increasing timeout settings
### Conflicts not resolving
Operational transformation should handle conflicts automatically. If issues persist:
1. Check console for sync errors
2. Verify both clients are connected
3. Check server logs for processing errors
4. Ensure files are text-based (binary files may not merge well)
### High CPU/Memory usage
1. **Reduce concurrency**: Lower `sync_concurrency`
2. **Add ignore patterns**: Exclude unnecessary files
3. **File watchers**: Large vaults may trigger many filesystem events
4. **Check for sync loops**: Ensure no circular dependencies
## Advanced Configuration
### Multiple Vaults
To sync multiple Obsidian vaults to different server vaults:
1. Each Obsidian vault has its own VaultLink plugin configuration
2. Use different vault names for each
3. Can use the same or different tokens (depending on access control)
### Custom Sync Patterns
Combine ignore patterns for fine-grained control:
```
# Ignore patterns
*.tmp
*.bak
.DS_Store
.trash/**
private/**
drafts/**/*.draft.md
```
### Development/Testing
For plugin development:
1. Clone the repository
2. `cd frontend && npm install`
3. `npm run dev` to build in watch mode
4. Plugin rebuilds automatically on changes
5. Reload Obsidian to test changes
## Next Steps
- [Learn about the sync algorithm →](/architecture/sync-algorithm)
- [Configure the server →](/config/server)
- [Set up the CLI client →](/guide/cli-client)

379
docs/guide/server-setup.md Normal file
View file

@ -0,0 +1,379 @@
# Server Setup
Deploy VaultLink server via Docker, binary, or build from source.
## Deployment Options
### Docker (Recommended)
Easiest deployment path, includes health checks.
#### Basic Docker Deployment
```bash
# Pull the latest image
docker pull ghcr.io/schmelczer/vault-link-server:latest
# Create data directory
mkdir -p ~/vaultlink-data
# Create config.yml (see Configuration section below)
# Run the container
docker run -d \
--name vaultlink-server \
--restart unless-stopped \
-p 3000:3000 \
-v ~/vaultlink-data:/data \
ghcr.io/schmelczer/vault-link-server:latest \
/app/sync_server /data/config.yml
```
#### Docker Compose
Create `docker-compose.yml`:
```yaml
services:
vaultlink-server:
image: ghcr.io/schmelczer/vault-link-server:latest
container_name: vaultlink-server
restart: unless-stopped
ports:
- "3000:3000"
volumes:
- ./data:/data
command: ["/app/sync_server", "/data/config.yml"]
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:3000/vaults/fake/ping"]
interval: 30s
timeout: 5s
retries: 3
start_period: 10s
```
Start the server:
```bash
docker compose up -d
```
### Binary Installation
Download pre-built binaries from [GitHub Releases](https://github.com/schmelczer/vault-link/releases):
```bash
# Download the binary for your platform
wget https://github.com/schmelczer/vault-link/releases/latest/download/sync_server-linux-x86_64
# Make executable
chmod +x sync_server-linux-x86_64
# Run the server
./sync_server-linux-x86_64 config.yml
```
### Build from Source
Requirements: Rust 1.92.0+, SQLite development headers, SQLx CLI
```bash
# Clone the repository
git clone https://github.com/schmelczer/vault-link.git
cd vault-link/sync-server
# Install SQLx CLI
cargo install sqlx-cli
# Set up the database
sqlx database create --database-url sqlite://db.sqlite3
sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3
cargo sqlx prepare --workspace
# Build in release mode
cargo build --release
# Run the server
./target/release/sync_server config.yml
```
## Configuration
Create a `config.yml` file with your server configuration:
```yaml
database:
databases_directory_path: databases
max_connections_per_vault: 12
cursor_timeout_seconds: 60
server:
host: 0.0.0.0
port: 3000
max_body_size_mb: 512
max_clients_per_vault: 256
response_timeout_seconds: 60
users:
user_configs:
- name: admin
token: your-secure-random-token-here
vault_access:
type: allow_access_to_all
logging:
log_directory: logs
log_rotation: 7days
```
### Configuration Fields
#### Database
- `databases_directory_path`: Directory for SQLite databases (one per vault)
- `max_connections_per_vault`: Maximum concurrent database connections
- `cursor_timeout_seconds`: How long to keep database cursors alive
#### Server
- `host`: Bind address (use `0.0.0.0` for all interfaces)
- `port`: Port to listen on (default: 3000)
- `max_body_size_mb`: Maximum upload size
- `max_clients_per_vault`: Concurrent client limit per vault
- `response_timeout_seconds`: Request timeout
#### Users
See [Authentication Configuration →](/config/authentication) for detailed user setup.
#### Logging
- `log_directory`: Where to store log files
- `log_rotation`: How often to rotate logs (e.g., `7days`, `24hours`)
## Production Deployment
### SSL/TLS with Reverse Proxy
VaultLink doesn't handle SSL directly. Use a reverse proxy like Nginx or Caddy.
#### Nginx Configuration
```nginx
upstream vaultlink {
server localhost:3000;
}
server {
listen 443 ssl http2;
server_name sync.example.com;
ssl_certificate /path/to/cert.pem;
ssl_certificate_key /path/to/key.pem;
location / {
proxy_pass http://vaultlink;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
# WebSocket specific
proxy_read_timeout 3600s;
proxy_send_timeout 3600s;
}
}
```
Reload Nginx:
```bash
sudo nginx -t
sudo systemctl reload nginx
```
#### Caddy Configuration
Caddy handles SSL automatically:
```caddy
sync.example.com {
reverse_proxy localhost:3000
}
```
Start Caddy:
```bash
caddy run --config Caddyfile
```
### Systemd Service
Create `/etc/systemd/system/vaultlink.service`:
```ini
[Unit]
Description=VaultLink Sync Server
After=network.target
[Service]
Type=simple
User=vaultlink
WorkingDirectory=/opt/vaultlink
ExecStart=/opt/vaultlink/sync_server /opt/vaultlink/config.yml
Restart=always
RestartSec=10
[Install]
WantedBy=multi-user.target
```
Enable and start:
```bash
sudo systemctl daemon-reload
sudo systemctl enable vaultlink
sudo systemctl start vaultlink
sudo systemctl status vaultlink
```
### Security Best Practices
1. **Use strong tokens**: Generate with `openssl rand -hex 32`
2. **Enable firewall**: Only expose port 3000 to reverse proxy
3. **Regular updates**: Keep Docker images and binaries updated
4. **Backup databases**: SQLite files in `databases_directory_path`
5. **Monitor logs**: Check log directory for errors and anomalies
6. **Limit access**: Use vault-specific access controls per user
### Backup Strategy
The SQLite databases contain all vault data and history:
```bash
# Backup script
#!/bin/bash
BACKUP_DIR="/backup/vaultlink/$(date +%Y%m%d)"
DATA_DIR="/data/databases"
mkdir -p "$BACKUP_DIR"
cp -r "$DATA_DIR" "$BACKUP_DIR/"
# Keep 30 days of backups
find /backup/vaultlink -type d -mtime +30 -exec rm -rf {} +
```
Run daily via cron:
```cron
0 2 * * * /opt/vaultlink/backup.sh
```
### Monitoring
#### Health Checks
The server exposes a ping endpoint:
```bash
curl http://localhost:3000/vaults/test/ping
# Returns: {"server_version":"0.10.1","is_authenticated":false}
```
Replace `test` with any vault name. The endpoint returns:
- `server_version`: Current server version
- `is_authenticated`: Whether the request included a valid token
Docker health check is built-in and checks this endpoint every 30 seconds.
#### Prometheus Metrics
For advanced monitoring, collect Docker stats or implement custom metrics.
#### Log Monitoring
Logs are written to the configured `log_directory`. Monitor for:
- Connection failures
- Authentication errors
- Database errors
- WebSocket disconnections
Example log watching:
```bash
tail -f /data/logs/*.log | grep -i error
```
## Scaling
### Horizontal Scaling
VaultLink currently uses SQLite, which limits horizontal scaling. For multiple servers:
1. Run separate instances for different vaults
2. Use load balancer with sticky sessions (same vault → same server)
3. Consider database architecture for your scale needs
### Vertical Scaling
Increase resources for the server:
- More CPU for handling concurrent connections
- More RAM for database caching
- Faster storage (SSD) for database operations
Tune configuration:
- Increase `max_clients_per_vault` for more concurrent users
- Increase `max_connections_per_vault` for database performance
- Adjust `max_body_size_mb` based on typical file sizes
## Troubleshooting
### Server won't start
```bash
# Check Docker logs
docker logs vaultlink-server
# Common issues:
# - Port already in use: Change port mapping
# - Config syntax error: Validate YAML
# - Permission error: Check volume permissions
```
### High memory usage
- Reduce `max_connections_per_vault`
- Reduce `max_clients_per_vault`
- Check for large vaults (may need database optimisation)
### Database corruption
```bash
# Verify database integrity
sqlite3 databases/your-vault.db "PRAGMA integrity_check;"
# If corrupted, restore from backup
cp /backup/databases/your-vault.db /data/databases/
```
### WebSocket connection drops
- Check reverse proxy timeout settings
- Verify firewall isn't closing connections
- Review client retry intervals
- Check server logs for errors
## Next Steps
- [Configure authentication and access control →](/config/authentication)
- [Set up Obsidian plugin →](/guide/obsidian-plugin)
- [Deploy CLI client →](/guide/cli-client)
- [Understand the architecture →](/architecture/)

View file

@ -0,0 +1,71 @@
# What is VaultLink?
Self-hosted sync for Obsidian vaults with automatic conflict-free merging. Edit with any tool, collaborate in real-time, no conflict markers.
## The Problem
Syncing Obsidian vaults across devices or sharing with teammates sucks:
- **Commercial services**: Lock-in, subscriptions, third-party access to your data
- **Git**: Manual conflict resolution with `<<<<<<<` markers interrupting your workflow
- **Cloud storage**: Last-write-wins data loss or manual conflict resolution
- **CRDT solutions**: Only work if you edit inside Obsidian (break if you use Vim, VS Code, etc.)
## VaultLink's Solution
Differential synchronisation with operational transformation for Markdown and text files.
Edit `.md` and `.txt` files with Obsidian, Vim, VS Code, or any editor. VaultLink compares versions and automatically merges all changes. No operation tracking required, no conflict markers.
**Note**: Binary files (images, PDFs, etc.) use last-write-wins. [See limitations →](/guide/limitations)
## How It Works
1. **Server**: Rust WebSocket server with SQLite stores document versions
2. **Clients**: Obsidian plugin or CLI client watches filesystem changes
3. **Sync**: Changes upload to server, server broadcasts to other clients
4. **Merge**: [reconcile-text](https://schmelczer.dev/reconcile) automatically merges concurrent edits
No CRDT infrastructure. No operation logs. Just file comparison and smart merging.
## Key Advantages
**Editor agnostic**: Edit files with any tool. Other solutions break when you edit outside their ecosystem.
**Self-hosted**: Your data, your server. No third parties, no subscriptions, no surprises.
**Automatic merging**: Operational transformation handles conflicts without interrupting your workflow.
**Production-ready**: Comprehensive tests, E2E tests, battle-tested. Many alternatives have zero tests.
**Collaborative**: Real-time sync with cursor tracking. See where teammates are editing.
## Not Tied to Obsidian
VaultLink syncs Markdown files. Use it for:
- Obsidian vaults (Obsidian desktop + mobile + CLI)
- Technical documentation (VS Code, your-editor, CLI)
- Academic writing (multiple Markdown editors)
- Automated workflows (CLI client for backups/CI/CD)
The Obsidian plugin is just a convenience wrapper around the sync client.
## Quick Comparison
| Feature | VaultLink | Git | Cloud Sync | CRDT Solutions |
| ------------------- | --------- | --- | ---------- | -------------- |
| Self-hosted | ✅ | ✅ | ❌ | Varies |
| Any editor | ✅ | ✅ | ✅ | ❌ |
| No conflict markers | ✅ | ❌ | ❌ | ✅ |
| Real-time | ✅ | ❌ | ❌ | ✅ |
| No subscriptions | ✅ | ✅ | ❌ | Varies |
| Comprehensive tests | ✅ | N/A | N/A | ❌ |
[Detailed comparison with alternatives →](/guide/alternatives)
## Next Steps
- [Get started →](/guide/getting-started) (5 minute setup)
- [See the architecture →](/architecture/) (understand how it works)
- [Compare alternatives →](/guide/alternatives) (why VaultLink vs others)

55
docs/index.md Normal file
View file

@ -0,0 +1,55 @@
---
layout: home
hero:
name: VaultLink
text: Self-Hosted Obsidian Sync
tagline: Edit with any tool. Automatic conflict-free merging. Your infrastructure.
image:
src: /logo.svg
alt: VaultLink
actions:
- theme: brand
text: Get Started
link: /guide/getting-started
- theme: alt
text: Why VaultLink?
link: /guide/what-is-vaultlink
features:
- title: Edit Anywhere
details: Use Obsidian, Vim, VS Code, or any editor. VaultLink syncs files, not keystrokes—edit however you want
- title: Your Data, Your Server
details: Fully self-hosted. No third parties, no subscriptions, no data mining. Single Docker container or binary
- title: No Conflict Markers
details: Automatic merge using operational transformation. Never see conflict markers in your notes again
- title: Real-Time Collaboration
details: See teammate cursors, merge edits instantly. Rust-powered WebSocket server with SQLite
- title: Open Source Everything
details: MIT licensed. Server, clients, and sync algorithm are all open source. No proprietary components
- title: Battle-Tested
details: Comprehensive test suite. E2E tests. Used in production. Unlike alternatives with zero tests
---
## Why Self-Host?
**You own your knowledge base.** Commercial sync services can disappear, change pricing, or lock you out. VaultLink runs on your infrastructure—VPS, home server, or localhost.
**Edit with any tool.** Other solutions require CRDT-aware editors or break when you edit outside Obsidian. VaultLink uses differential sync: edit files however you want, sync handles the rest.
**No conflict markers.** Git forces manual merging. Other tools use last-write-wins. VaultLink's operational transformation automatically merges Markdown and text files without conflict markers or workflow interruption. [See what's supported →](/guide/limitations)
[See how VaultLink compares to alternatives →](/guide/alternatives)
## Quick Start
Deploy server (single command):
```bash
docker run -d -p 3000:3000 -v $(pwd)/data:/data \
ghcr.io/schmelczer/vault-link-server:latest
```
Then install the [Obsidian plugin](/guide/obsidian-plugin) or [CLI client](/guide/cli-client).
[Full setup guide →](/guide/getting-started)

2989
docs/package-lock.json generated Normal file

File diff suppressed because it is too large Load diff

25
docs/package.json Normal file
View file

@ -0,0 +1,25 @@
{
"name": "docs",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"dev": "vitepress dev --host",
"build": "vitepress build",
"preview": "vitepress preview",
"format": "prettier --write \"**/*.md\" \"**/*.mts\"",
"format:check": "prettier --check \"**/*.md\" \"**/*.mts\"",
"spell": "cspell \"**/*.md\" \"**/*.mts\"",
"spell:check": "cspell \"**/*.md\" \"**/*.mts\""
},
"keywords": [],
"author": "",
"license": "ISC",
"devDependencies": {
"@cspell/dict-en-gb": "^5.0.19",
"cspell": "^9.3.2",
"prettier": "^3.6.2",
"vitepress": "^1.6.4",
"vue": "^3.5.24"
}
}

47
docs/public/logo.svg Normal file
View file

@ -0,0 +1,47 @@
<svg width="200" height="200" viewBox="0 0 200 200" xmlns="http://www.w3.org/2000/svg">
<defs>
<linearGradient id="grad1" x1="0%" y1="0%" x2="100%" y2="100%">
<stop offset="0%" style="stop-color:#4A90E2;stop-opacity:1" />
<stop offset="100%" style="stop-color:#357ABD;stop-opacity:1" />
</linearGradient>
</defs>
<!-- Background circle -->
<circle cx="100" cy="100" r="90" fill="url(#grad1)" opacity="0.15"/>
<!-- Main vault icon -->
<g transform="translate(100, 100)">
<!-- Vault body -->
<rect x="-45" y="-50" width="90" height="80" rx="8" fill="none" stroke="url(#grad1)" stroke-width="6"/>
<!-- Vault door circle -->
<circle cx="0" cy="-10" r="22" fill="none" stroke="url(#grad1)" stroke-width="5"/>
<circle cx="0" cy="-10" r="14" fill="none" stroke="url(#grad1)" stroke-width="3"/>
<circle cx="0" cy="-10" r="6" fill="url(#grad1)"/>
<!-- Vault handle -->
<line x1="0" y1="-4" x2="18" y2="-4" stroke="url(#grad1)" stroke-width="3" stroke-linecap="round"/>
<circle cx="18" cy="-4" r="4" fill="url(#grad1)"/>
<!-- Link chain -->
<g opacity="0.9">
<!-- Left link -->
<ellipse cx="-30" cy="40" rx="12" ry="8" fill="none" stroke="url(#grad1)" stroke-width="4"/>
<!-- Right link -->
<ellipse cx="30" cy="40" rx="12" ry="8" fill="none" stroke="url(#grad1)" stroke-width="4"/>
<!-- Center link connecting them -->
<ellipse cx="0" cy="40" rx="12" ry="8" fill="none" stroke="url(#grad1)" stroke-width="4"/>
</g>
<!-- Sync arrows (subtle) -->
<g opacity="0.5">
<!-- Clockwise arrow top-right -->
<path d="M 35 -35 Q 50 -35 50 -20 L 50 -15" fill="none" stroke="url(#grad1)" stroke-width="2.5" stroke-linecap="round"/>
<polygon points="50,-15 47,-22 53,-22" fill="url(#grad1)"/>
<!-- Counter-clockwise arrow bottom-left -->
<path d="M -35 25 Q -50 25 -50 10 L -50 5" fill="none" stroke="url(#grad1)" stroke-width="2.5" stroke-linecap="round"/>
<polygon points="-50,5 -47,12 -53,12" fill="url(#grad1)"/>
</g>
</g>
</svg>

After

Width:  |  Height:  |  Size: 2 KiB

View file

@ -0,0 +1,118 @@
# Deterministic Tests
Scripted multi-client (with an in-memory filesystem) sync tests that run against a real server. Each test defines a sequence of file operations, sync/server controls, and assertions to exercise a specific conflict or edge case.
Complements the fuzz-based E2E tests (`test-client`): fuzz tests discover bugs through random operations; deterministic tests pin down exact reproduction sequences for known scenarios.
## How it works
Each test is a `TestDefinition`: a client count and an ordered list of steps. The test name is derived from the registry key (which matches the file name). The `TestRunner` spins up N `DeterministicAgent` instances (each wrapping a real `SyncClient` with an `InMemoryFileSystem`) pointed at a shared vault on the server, then executes steps one by one.
Tests that don't pause the server share a single server process (vault-name isolation). Tests that use `pause-server`/`resume-server` (SIGSTOP/SIGCONT) each get a dedicated server, since SIGSTOP freezes the entire process.
The runner executes two sequential phases: regular tests on the shared server, then pause-server tests on dedicated servers. Within each phase tests run in parallel up to a concurrency limit.
## Step types
Clients always start with syncing disabled.
**File operations** (per-client, fire-and-forget — sync is enqueued but not awaited):
- `create`, `update`, `rename`, `delete`
- `rename-next-write` — arm a deferred rename that fires the next time the given path is written. Lets a test race a user-rename against an in-flight remote create that's about to land at the same path.
**Sync control:**
- `sync` — wait for a specific client or all clients to finish pending operations
- `barrier` — retry until all clients converge to identical file state (60s timeout)
- `enable-sync` / `disable-sync` — simulate going online/offline
- `reset` — reset a client's tracked sync state (keeps disk files); equivalent to a forced re-handshake on next enable
- `sleep` — wall-clock pause; use sparingly, prefer `barrier` / `sync`
**WebSocket control** (per-client):
- `pause-websocket` / `resume-websocket` — buffer/release WebSocket messages for a specific client
**Server control:**
- `pause-server` / `resume-server` — SIGSTOP/SIGCONT the server process
- `resume-server-until-history-then-pause` — resume the server, wait until a specific client observes a matching history entry (`CREATE`/`UPDATE`/`DELETE` for a path), then re-pause. Used to land exactly one operation across the wire.
**Fault injection** (per-client):
- `drop-next-create-response` — arm a one-shot interceptor that lets the next `POST /documents` reach the server (commit happens) but throws `SyncResetError` before the client sees the response, simulating connection loss after server commit.
- `wait-for-dropped-create-response` — wait until the armed drop has fired.
**Assertions:**
- `assert-consistent` — all clients have identical files; optionally takes a custom `verify(state: AssertableState)` callback
## Running
```sh
# Build server first
cd sync-server && cargo build --release && cd -
# Run all tests
cd frontend && npm run build -w sync-client && npm run test -w deterministic-tests
# Filter by name
npm run test -w deterministic-tests -- --filter=rename
# Control parallelism (default: number of CPU cores)
npm run test -w deterministic-tests -- -j 4
```
## Adding a test
1. Create `src/tests/my-scenario.test.ts`:
```typescript
import type { TestDefinition } from "../test-definition";
export const myScenarioTest: TestDefinition = {
description:
"Client 0 creates A.md offline. After syncing, both clients should have the file.",
clients: 2,
steps: [
{ type: "create", client: 0, path: "A.md", content: "hello" },
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s) => {
s.assertFileCount(1).assertContent("A.md", "hello");
}
}
]
};
```
The `verify` callback receives an `AssertableState` object with chainable assertion methods:
```typescript
s.assertFileCount(n); // exact file count
s.assertFileExists("path"); // file must exist
s.assertFileNotExists("path"); // file must not exist
s.assertContent("path", "expected"); // exact content match
s.assertContains("path", "a", "b"); // all substrings present in file
s.assertContainsAny("path", "a", "b"); // at least one substring present
s.assertAnyFileContains("text"); // substring present in some file
s.assertNoFileContains("text"); // substring absent from every file
s.assertSubstringCount("path", "x", 3); // substring appears exactly N times
s.assertContentInAtMostOneFile("text"); // no duplicate content
s.ifFileExists("path", (s) => { /* … */ }); // conditional block
s.getContent("path"); // raw content (or "" if missing)
```
2. Register it in `src/test-registry.ts`:
```typescript
import { myScenarioTest } from "./tests/my-scenario.test";
const TESTS = {
// ...
"my-scenario": myScenarioTest
};
```

View file

@ -0,0 +1,23 @@
{
"name": "deterministic-tests",
"version": "0.14.0",
"private": true,
"bin": {
"deterministic-tests": "./dist/cli.js"
},
"scripts": {
"dev": "webpack watch --mode development",
"build": "webpack --mode production",
"test": "npm run build && node dist/cli.js"
},
"devDependencies": {
"commander": "^14.0.2",
"@types/node": "^25.0.2",
"sync-client": "file:../sync-client",
"ts-loader": "^9.5.4",
"tslib": "2.8.1",
"typescript": "5.9.3",
"webpack": "^5.103.0",
"webpack-cli": "^6.0.1"
}
}

View file

@ -0,0 +1,243 @@
import { TestRunner } from "./test-runner";
import { ServerControl } from "./server-control";
import { ServerManager } from "./server-manager";
import { PrefixedLogger } from "./prefixed-logger";
import { TESTS } from "./test-registry";
import type { TestDefinition, TestResult } from "./test-definition";
import { parseArgs } from "./parse-args";
import { runWithConcurrency } from "./run-with-concurrency";
import { TOKEN, SERVER_BINARY_PATH, CONFIG_PATH } from "./consts";
import * as path from "node:path";
import * as fs from "node:fs";
import { debugging, Logger } from "sync-client";
const logger = new Logger();
debugging.logToConsole(logger, { useColors: true });
process.on("unhandledRejection", (reason) => {
logger.error(`Unhandled Rejection: ${reason}`);
process.exit(1);
});
process.on("uncaughtException", (error) => {
logger.error(`Uncaught Exception: ${error}`);
process.exit(1);
});
const serverManager = new ServerManager(logger);
serverManager.installSignalHandlers();
function testUsesPauseServer(test: TestDefinition): boolean {
return test.steps.some(
(step) =>
step.type === "pause-server" ||
step.type === "resume-server" ||
step.type === "resume-server-until-history-then-pause"
);
}
/**
* Walk up from the CLI binary's location until we find a directory
* containing `sync-server/` and `frontend/`.
*/
function findProjectRoot(): string {
let dir = path.dirname(__filename);
const root = path.parse(dir).root;
while (dir !== root) {
if (
fs.existsSync(path.join(dir, "sync-server")) &&
fs.existsSync(path.join(dir, "frontend"))
) {
return dir;
}
dir = path.dirname(dir);
}
throw new Error(
`Could not locate project root (no ancestor of ${__filename} contains both 'sync-server' and 'frontend')`
);
}
interface NamedTestResult {
name: string;
result: TestResult;
}
async function runSharedServerTest(
name: string,
test: TestDefinition,
sharedServer: ServerControl
): Promise<NamedTestResult> {
const testLogger = new PrefixedLogger(logger, name);
const runner = new TestRunner(
sharedServer,
testLogger,
TOKEN,
sharedServer.remoteUri
);
const result = await runner.runTest(name, test);
if (result.success) {
logger.info(`PASSED: ${name} (${result.duration}ms)`);
} else {
logger.error(`FAILED: ${name} - ${result.error}`);
}
return { name, result };
}
/**
* Run a test with its own dedicated server (for tests that use pause-server).
* SIGSTOP/SIGCONT affects the entire server process, so these tests need
* isolated servers to avoid interfering with other tests.
*/
async function runDedicatedServerTest(
name: string,
test: TestDefinition,
serverPath: string,
configPath: string
): Promise<NamedTestResult> {
const testLogger = new PrefixedLogger(logger, name);
const server = new ServerControl(serverPath, configPath, testLogger);
serverManager.track(server);
try {
await server.start();
const runner = new TestRunner(
server,
testLogger,
TOKEN,
server.remoteUri
);
const result = await runner.runTest(name, test);
if (result.success) {
logger.info(`PASSED: ${name} (${result.duration}ms)`);
} else {
logger.error(`FAILED: ${name} - ${result.error}`);
}
return { name, result };
} finally {
try {
await server.stop();
} catch {
// best-effort cleanup
}
serverManager.untrack(server);
}
}
async function main(): Promise<void> {
const projectRoot = findProjectRoot();
const serverPath = path.join(projectRoot, SERVER_BINARY_PATH);
if (!fs.existsSync(serverPath)) {
logger.error(`Server binary not found at: ${serverPath}`);
process.exit(1);
}
const configPath = path.join(projectRoot, CONFIG_PATH);
if (!fs.existsSync(configPath)) {
logger.error(`Config file not found at: ${configPath}`);
process.exit(1);
}
const { filter, concurrency } = parseArgs(process.argv);
const testsToRun: [string, TestDefinition][] = [];
for (const [key, test] of Object.entries(TESTS)) {
if (test) {
if (
filter !== undefined &&
filter.length > 0 &&
!key.includes(filter)
) {
continue;
}
testsToRun.push([key, test]);
}
}
if (testsToRun.length === 0) {
logger.error(
filter !== undefined && filter.length > 0
? `No tests matched filter "${filter}"`
: "No tests found"
);
process.exit(1);
}
const regularTests = testsToRun.filter(([, t]) => !testUsesPauseServer(t));
const pauseTests = testsToRun.filter(([, t]) => testUsesPauseServer(t));
logger.info(`Server: ${serverPath}`);
logger.info(`Config: ${configPath}`);
logger.info(
`Tests: ${testsToRun.length} total (${regularTests.length} regular, ${pauseTests.length} server-pause)`
);
logger.info(`Concurrency: ${concurrency}`);
const allResults: NamedTestResult[] = [];
if (regularTests.length > 0) {
logger.info(
`\n--- Running ${regularTests.length} regular tests (shared server, concurrency ${concurrency}) ---`
);
const sharedServer = new ServerControl(serverPath, configPath, logger);
serverManager.track(sharedServer);
try {
await sharedServer.start();
const results = await runWithConcurrency(
regularTests,
concurrency,
async ([name, test]) =>
runSharedServerTest(name, test, sharedServer)
);
allResults.push(...results);
} finally {
try {
await sharedServer.stop();
} catch (error) {
logger.warn(
`Error stopping shared server: ${error instanceof Error ? error.message : String(error)}`
);
}
serverManager.untrack(sharedServer);
}
}
if (pauseTests.length > 0) {
logger.info(
`\n--- Running ${pauseTests.length} server-pause tests (dedicated servers, concurrency ${concurrency}) ---`
);
const results = await runWithConcurrency(
pauseTests,
concurrency,
async ([name, test]) =>
runDedicatedServerTest(name, test, serverPath, configPath)
);
allResults.push(...results);
}
const passed = allResults.filter((r) => r.result.success);
const failed = allResults.filter((r) => !r.result.success);
logger.info(
`\n--- Results: ${passed.length}/${allResults.length} passed ---`
);
if (failed.length > 0) {
for (const { name, result } of failed) {
logger.error(` FAILED: ${name}: ${result.error}`);
}
process.exit(1);
} else {
logger.info("All tests passed!");
process.exit(0);
}
}
main().catch((err: unknown) => {
logger.error(`Unexpected error: ${err}`);
process.exit(1);
});

View file

@ -0,0 +1,17 @@
export const TOKEN = "test-token-change-me";
export const SERVER_BINARY_PATH = "sync-server/target/release/sync_server";
export const CONFIG_PATH = "sync-server/config-e2e.yml";
export const STOP_TIMEOUT_MS = 5_000;
export const CONVERGENCE_TIMEOUT_MS = 60_000;
export const CONVERGENCE_RETRY_DELAY_MS = 500;
export const AGENT_INIT_TIMEOUT_MS = 30_000;
export const IS_SYNC_ENABLED_BY_DEFAULT = false;
export const WAIT_TIMEOUT_MS = 60_000;
export const WEBSOCKET_CONNECT_TIMEOUT_MS = 10_000;
export const WEBSOCKET_POLL_INTERVAL_MS = 50;
export const SERVER_READY_POLL_INTERVAL_MS = 100;
export const SERVER_READY_MAX_ATTEMPTS = 50;
export const SERVER_START_MAX_ATTEMPTS = 5;

View file

@ -0,0 +1,483 @@
import type {
HistoryEntry,
StoredDatabase,
SyncSettings,
RelativePath,
TextWithCursors
} from "sync-client";
import {
SyncClient,
SyncResetError,
debugging,
LogLevel,
utils
} from "sync-client";
import { assert } from "./utils/assert";
import { sleep } from "./utils/sleep";
import { withTimeout } from "./utils/with-timeout";
import {
IS_SYNC_ENABLED_BY_DEFAULT,
WAIT_TIMEOUT_MS,
WEBSOCKET_CONNECT_TIMEOUT_MS,
WEBSOCKET_POLL_INTERVAL_MS
} from "./consts";
import { ManagedWebSocketFactory } from "./managed-websocket";
export class DeterministicAgent extends debugging.InMemoryFileSystem {
public readonly clientId: number;
private readonly logger: (msg: string) => void;
private client!: SyncClient;
private data: Partial<{
settings: Partial<SyncSettings>;
database: Partial<StoredDatabase>;
}> = {};
private isSyncEnabled = IS_SYNC_ENABLED_BY_DEFAULT;
private readonly syncErrors: Error[] = [];
private readonly pendingSyncOperations = new Set<Promise<void>>();
private readonly wsFactory = new ManagedWebSocketFactory();
private nextWriteRename:
| {
oldPath: RelativePath;
newPath: RelativePath;
}
| undefined;
private nextCreateResponseDrop:
| {
dropped: Promise<void>;
resolveDropped: () => void;
}
| undefined;
public constructor(
clientId: number,
initialSettings: Partial<SyncSettings>,
logger: (msg: string) => void
) {
super();
this.clientId = clientId;
this.logger = logger;
this.data.settings = { ...initialSettings };
}
public async init(
fetchImplementation: typeof globalThis.fetch
): Promise<void> {
this.client = await SyncClient.create({
fs: this,
persistence: {
load: async () => this.data,
save: async (data) => void (this.data = data)
},
fetch: this.wrapFetch(fetchImplementation),
webSocket: this.wsFactory.constructorFn
});
this.client.logger.onLogEmitted.add((line) => {
const prefix = `[Client ${this.clientId}]`;
switch (line.level) {
case LogLevel.ERROR:
this.logger(`${prefix} ERROR: ${line.message}`);
break;
case LogLevel.WARNING:
this.logger(`${prefix} WARN: ${line.message}`);
break;
case LogLevel.INFO:
this.logger(`${prefix} INFO: ${line.message}`);
break;
case LogLevel.DEBUG:
this.logger(`${prefix} DEBUG: ${line.message}`);
break;
}
});
await this.client.start();
const connectionCheck = await this.client.checkConnection();
assert(
connectionCheck.isSuccessful,
`Client ${this.clientId} connection check failed`
);
if (this.isSyncEnabled) {
await this.waitForWebSocket();
}
}
public pauseWebSocket(): void {
this.log("Pausing WebSocket message delivery");
this.wsFactory.pause();
}
public resumeWebSocket(): void {
this.log("Resuming WebSocket message delivery");
this.wsFactory.resume();
}
public dropNextCreateResponse(): void {
assert(
this.nextCreateResponseDrop === undefined,
`Client ${this.clientId} already has a create response drop armed`
);
let resolveDropped!: () => void;
const dropped = new Promise<void>((resolve) => {
resolveDropped = resolve;
});
this.nextCreateResponseDrop = {
dropped,
resolveDropped
};
this.log("Armed next create response drop");
}
public async waitForDroppedCreateResponse(): Promise<void> {
assert(
this.nextCreateResponseDrop !== undefined,
`Client ${this.clientId} has no create response drop armed`
);
await withTimeout(
this.nextCreateResponseDrop.dropped,
WAIT_TIMEOUT_MS,
`Client ${this.clientId} timed out waiting for create response drop`
);
this.log("Create response was dropped after server commit");
}
public async waitForHistoryEntry(
matches: (entry: HistoryEntry) => boolean,
onMatch?: (entry: HistoryEntry) => void
): Promise<void> {
const existing = this.client.getHistoryEntries().find(matches);
if (existing !== undefined) {
onMatch?.(existing);
return;
}
await withTimeout(
new Promise<void>((resolve) => {
const unsubscribe = this.client.onSyncHistoryUpdated.add(() => {
const entry = this.client
.getHistoryEntries()
.find(matches);
if (entry === undefined) {
return;
}
unsubscribe();
onMatch?.(entry);
resolve();
});
}),
WAIT_TIMEOUT_MS,
`Client ${this.clientId} timed out waiting for history entry`
);
}
public async waitForSync(): Promise<void> {
this.log("Waiting for sync to complete...");
// Drain agent-level sync operations first. These are the fire-and-forget
// promises from enqueueSync() that call into the SyncClient's methods.
// Without this, waitUntilFinished() might return before the SyncClient
// has even been told about the operation.
await this.drainPendingSyncOperations();
await withTimeout(
this.client.waitUntilFinished(),
WAIT_TIMEOUT_MS,
`Client ${this.clientId} waitForSync timed out after ${WAIT_TIMEOUT_MS}ms`
);
if (this.syncErrors.length > 0) {
const errors = this.syncErrors.splice(0);
throw new Error(
`Client ${this.clientId} had ${errors.length} sync error(s):\n${errors.map((e) => e.message).join("\n")}`
);
}
this.log("Sync complete");
}
public async reset(): Promise<void> {
this.log("Resetting client (clears tracked state, keeps disk files)");
await this.drainPendingSyncOperations();
await this.client.reset();
if (this.isSyncEnabled) {
await this.waitForWebSocket();
}
}
public async disableSync(): Promise<void> {
this.log("Disabling sync");
// Drain pending enqueued operations before disabling so the SyncClient
// knows about all operations that were enqueued while sync was enabled.
await this.drainPendingSyncOperations();
await this.client.setSetting("isSyncEnabled", false);
this.isSyncEnabled = false;
// Wait for in-flight operations to drain. Disabling sync triggers
// a reset, which aborts in-flight fetches with SyncResetError.
try {
await withTimeout(
this.client.waitUntilFinished(),
WAIT_TIMEOUT_MS,
`Client ${this.clientId} disableSync drain timed out`
);
} catch (error) {
if (error instanceof Error && error.name === "SyncResetError") {
this.log("Disable sync drain interrupted by reset (expected)");
} else {
throw error;
}
}
}
public async enableSync(): Promise<void> {
this.log("Enabling sync");
await this.client.setSetting("isSyncEnabled", true);
this.isSyncEnabled = true;
await this.waitForWebSocket();
}
public async getFileContent(path: string): Promise<string> {
const bytes = await this.read(path);
return new TextDecoder().decode(bytes);
}
public renameNextWrite(oldPath: RelativePath, newPath: RelativePath): void {
assert(
this.nextWriteRename === undefined,
`Client ${this.clientId} already has a next-write rename armed`
);
this.nextWriteRename = { oldPath, newPath };
this.log(`Armed next write rename: ${oldPath} -> ${newPath}`);
}
public async cleanup(): Promise<void> {
this.log("Cleaning up...");
// Guard against uninitialized client (init() failed partway).
// The class field uses `!:` so TS thinks this is always defined,
// but at runtime it can be undefined when init() throws partway.
const maybeClient = this.client as SyncClient | undefined;
if (maybeClient === undefined) {
this.log("Client not initialized, nothing to clean up");
return;
}
try {
await this.drainPendingSyncOperations();
await withTimeout(
this.client.waitUntilFinished(),
WAIT_TIMEOUT_MS,
`Client ${this.clientId} cleanup waitUntilFinished timed out`
);
} catch (error) {
if (error instanceof Error && error.name === "SyncResetError") {
this.log(`Cleanup interrupted by reset (expected): ${error}`);
} else {
this.log(`Cleanup waitUntilFinished failed: ${error}`);
}
}
// Surface any background sync errors that arrived after the last
// waitForSync (e.g. between the final assert-consistent and here).
// Without this, regressions that fault the engine during the very
// last step of a test would be silently swallowed.
const pendingErrors = this.syncErrors.splice(0);
await this.client.destroy();
this.log("Cleanup complete");
if (pendingErrors.length > 0) {
throw new Error(
`Client ${this.clientId} had ${pendingErrors.length} background sync error(s) during cleanup:\n${pendingErrors.map((e) => e.message).join("\n")}`
);
}
}
public override async read(path: RelativePath): Promise<Uint8Array> {
await Promise.resolve();
return super.read(path);
}
public override async write(
path: RelativePath,
content: Uint8Array
): Promise<void> {
await Promise.resolve();
const isNew = !this.files.has(path);
await super.write(path, content);
if (this.isSyncEnabled && isNew) {
this.enqueueSync(async () => {
this.client.syncLocallyCreatedFile(path);
});
}
const nextWriteRename = this.nextWriteRename;
if (
nextWriteRename !== undefined &&
nextWriteRename.oldPath === path
) {
this.nextWriteRename = undefined;
await super.rename(
nextWriteRename.oldPath,
nextWriteRename.newPath
);
if (this.isSyncEnabled) {
this.enqueueSync(async () => {
this.client.syncLocallyUpdatedFile({
oldPath: nextWriteRename.oldPath,
relativePath: nextWriteRename.newPath
});
});
}
// The rename consumed `path`. Skip the post-update enqueue below
// — it would send a syncLocallyUpdatedFile for a path that no
// longer exists.
return;
}
if (!this.isSyncEnabled) {
return;
}
if (!isNew) {
this.enqueueSync(async () => {
this.client.syncLocallyUpdatedFile({ relativePath: path });
});
}
}
public override async atomicUpdateText(
path: RelativePath,
updater: (current: TextWithCursors) => TextWithCursors
): Promise<string> {
const result = await super.atomicUpdateText(path, updater);
if (this.isSyncEnabled) {
this.enqueueSync(async () => {
this.client.syncLocallyUpdatedFile({ relativePath: path });
});
}
return result;
}
public override async delete(path: RelativePath): Promise<void> {
await super.delete(path);
if (this.isSyncEnabled) {
this.enqueueSync(async () => {
this.client.syncLocallyDeletedFile(path);
});
}
}
public override async rename(
oldPath: RelativePath,
newPath: RelativePath
): Promise<void> {
await super.rename(oldPath, newPath);
if (this.isSyncEnabled) {
this.enqueueSync(async () => {
this.client.syncLocallyUpdatedFile({
oldPath,
relativePath: newPath
});
});
}
}
private async waitForWebSocket(): Promise<void> {
const deadline = Date.now() + WEBSOCKET_CONNECT_TIMEOUT_MS;
while (!this.client.isWebSocketConnected && Date.now() < deadline) {
await sleep(WEBSOCKET_POLL_INTERVAL_MS);
}
assert(
this.client.isWebSocketConnected,
`Client ${this.clientId} WebSocket failed to connect within ${WEBSOCKET_CONNECT_TIMEOUT_MS}ms`
);
}
/**
* Wait until all agent-level enqueued sync operations have completed.
* Uses a loop because completing one operation can trigger new enqueues.
*/
private async drainPendingSyncOperations(): Promise<void> {
while (this.pendingSyncOperations.size > 0) {
await utils.awaitAll([...this.pendingSyncOperations]);
}
}
private enqueueSync(operation: () => Promise<void>): void {
const promise = this.executeSyncOperation(operation).catch(
(error: unknown) => {
const err =
error instanceof Error ? error : new Error(String(error));
this.log(`Background sync failed: ${err.message}`);
this.syncErrors.push(err);
}
);
this.pendingSyncOperations.add(promise);
void promise.finally(() => {
this.pendingSyncOperations.delete(promise);
});
}
private async executeSyncOperation(
operation: () => Promise<void>
): Promise<void> {
try {
await operation();
} catch (error) {
if (error instanceof Error && error.name === "SyncResetError") {
this.log(`Sync operation interrupted by reset: ${error}`);
return;
}
if (
error instanceof Error &&
error.message.includes("has been destroyed")
) {
this.log(`Sync operation interrupted by destroy: ${error}`);
return;
}
throw error;
}
}
private log(message: string): void {
this.logger(`[Client ${this.clientId}] ${message}`);
}
private wrapFetch(
fetchImplementation: typeof globalThis.fetch
): typeof globalThis.fetch {
return async (input, init) => {
const response = await fetchImplementation(input, init);
const drop = this.nextCreateResponseDrop;
if (
drop !== undefined &&
DeterministicAgent.isCreateDocumentRequest(input, init)
) {
this.nextCreateResponseDrop = undefined;
try {
await response.body?.cancel();
} catch {
// Best-effort — body may already be consumed/closed.
}
drop.resolveDropped();
throw new SyncResetError();
}
return response;
};
}
private static isCreateDocumentRequest(
input: RequestInfo | URL,
init: RequestInit | undefined
): boolean {
const method =
init?.method ??
(typeof Request !== "undefined" && input instanceof Request
? input.method
: "GET");
if (method.toUpperCase() !== "POST") {
return false;
}
const url =
input instanceof URL
? input
: new URL(typeof input === "string" ? input : input.url);
return /\/documents\/?$/.test(url.pathname);
}
}

View file

@ -0,0 +1,245 @@
/**
* A WebSocket wrapper that can pause and resume message delivery.
* When paused, incoming messages are buffered. When resumed, buffered
* messages are delivered in order via the onmessage handler.
*
* Member layout follows typescript-eslint default member-ordering: all
* accessor properties are declared with `declare` and wired through the
* constructor using Object.defineProperty so we don't need conflicting
* get/set accessor pairs.
*/
class ManagedWebSocket implements WebSocket {
public static readonly CONNECTING = WebSocket.CONNECTING;
public static readonly OPEN = WebSocket.OPEN;
public static readonly CLOSING = WebSocket.CLOSING;
public static readonly CLOSED = WebSocket.CLOSED;
public readonly CONNECTING = WebSocket.CONNECTING;
public readonly OPEN = WebSocket.OPEN;
public readonly CLOSING = WebSocket.CLOSING;
public readonly CLOSED = WebSocket.CLOSED;
declare public readonly readyState: number;
declare public readonly url: string;
declare public readonly protocol: string;
declare public readonly extensions: string;
declare public readonly bufferedAmount: number;
declare public binaryType: BinaryType;
declare public onopen: ((this: WebSocket, ev: Event) => unknown) | null;
declare public onclose:
| ((this: WebSocket, ev: CloseEvent) => unknown)
| null;
declare public onerror: ((this: WebSocket, ev: Event) => unknown) | null;
declare public onmessage:
| ((this: WebSocket, ev: MessageEvent) => unknown)
| null;
private readonly ws: WebSocket;
private readonly bufferedMessages: MessageEvent[] = [];
private paused = false;
private externalOnMessage: ((event: MessageEvent) => unknown) | null = null;
public constructor(url: string | URL, protocols?: string | string[]) {
this.ws = new WebSocket(url, protocols);
const { ws } = this;
Object.defineProperties(this, {
readyState: {
get: (): number => ws.readyState,
enumerable: true,
configurable: true
},
url: {
get: (): string => ws.url,
enumerable: true,
configurable: true
},
protocol: {
get: (): string => ws.protocol,
enumerable: true,
configurable: true
},
extensions: {
get: (): string => ws.extensions,
enumerable: true,
configurable: true
},
bufferedAmount: {
get: (): number => ws.bufferedAmount,
enumerable: true,
configurable: true
},
binaryType: {
get: (): BinaryType => ws.binaryType,
set: (v: BinaryType): void => {
ws.binaryType = v;
},
enumerable: true,
configurable: true
},
onopen: {
get: (): ((this: WebSocket, ev: Event) => unknown) | null =>
ws.onopen,
set: (
h: ((this: WebSocket, ev: Event) => unknown) | null
): void => {
ws.onopen = h;
},
enumerable: true,
configurable: true
},
onclose: {
get: ():
| ((this: WebSocket, ev: CloseEvent) => unknown)
| null => ws.onclose,
set: (
h: ((this: WebSocket, ev: CloseEvent) => unknown) | null
): void => {
ws.onclose = h;
},
enumerable: true,
configurable: true
},
onerror: {
get: (): ((this: WebSocket, ev: Event) => unknown) | null =>
ws.onerror,
set: (
h: ((this: WebSocket, ev: Event) => unknown) | null
): void => {
ws.onerror = h;
},
enumerable: true,
configurable: true
},
onmessage: {
get: ():
| ((this: WebSocket, ev: MessageEvent) => unknown)
| null => this.externalOnMessage,
set: (
h: ((this: WebSocket, ev: MessageEvent) => unknown) | null
): void => {
this.externalOnMessage = h;
},
enumerable: true,
configurable: true
}
});
this.ws.onmessage = (event: MessageEvent): void => {
if (this.paused) {
this.bufferedMessages.push(event);
} else {
this.externalOnMessage?.(event);
}
};
}
public pause(): void {
this.paused = true;
}
public resume(): void {
// Drain buffered messages BEFORE flipping `paused` to false.
// If `externalOnMessage` is async (its return type is `unknown`),
// dispatch yields control between buffered messages, and a fresh
// live `ws.onmessage` event firing during that yield would jump
// ahead of unprocessed buffered messages — silently reordering
// events relative to the wire. Keeping `paused = true` during the
// drain forces the live handler to keep buffering, so we splice
// those late arrivals onto the tail and dispatch them in order.
while (this.bufferedMessages.length > 0) {
const messages = this.bufferedMessages.splice(0);
for (const msg of messages) {
this.externalOnMessage?.(msg);
}
}
this.paused = false;
}
public send(data: string | ArrayBufferLike | Blob | ArrayBufferView): void {
this.ws.send(data);
}
public close(code?: number, reason?: string): void {
this.ws.close(code, reason);
}
public addEventListener(
...args: Parameters<WebSocket["addEventListener"]>
): void {
// Only the `.onmessage` setter routes through the pause buffer.
// If sync-client ever attaches "message" listeners via
// addEventListener instead, those messages would bypass pause/resume
// and deterministic tests would silently lose their fault injection.
if (args[0] === "message") {
throw new Error(
"ManagedWebSocket: addEventListener('message') bypasses the " +
"pause buffer. Use the .onmessage setter instead, or " +
"extend ManagedWebSocket to route message listeners."
);
}
this.ws.addEventListener(...args);
}
public removeEventListener(
...args: Parameters<WebSocket["removeEventListener"]>
): void {
this.ws.removeEventListener(...args);
}
public dispatchEvent(event: Event): boolean {
return this.ws.dispatchEvent(event);
}
}
/**
* Factory that creates ManagedWebSocket instances and tracks them
* for pause/resume control from the test harness
*/
export class ManagedWebSocketFactory {
// Append-only: closed sockets stay tracked. Bounded per test (one
// factory per agent, each test discards its agents on cleanup), so
// not a real leak — but iterating over closed instances on
// pause/resume is a deliberate no-op since their `.onmessage` is
// already detached.
private readonly instances: ManagedWebSocket[] = [];
// Sticky pause state: applied to current instances on `pause()` AND
// to any new instance created later (e.g. WS reconnect after a
// `disable-sync` / `reset` cycle). Without this, a test pausing the
// WS before the agent reconnects would silently see the new socket
// start un-paused and miss the messages it meant to buffer.
private currentlyPaused = false;
public get constructorFn(): typeof globalThis.WebSocket {
const trackInstance = (instance: ManagedWebSocket): void => {
this.instances.push(instance);
if (this.currentlyPaused) {
instance.pause();
}
};
class TrackedManagedWebSocket extends ManagedWebSocket {
public constructor(
url: string | URL,
protocols?: string | string[]
) {
super(url, protocols);
trackInstance(this);
}
}
return TrackedManagedWebSocket;
}
public pause(): void {
this.currentlyPaused = true;
for (const ws of this.instances) {
ws.pause();
}
}
public resume(): void {
this.currentlyPaused = false;
for (const ws of this.instances) {
ws.resume();
}
}
}

View file

@ -0,0 +1,43 @@
import * as os from "node:os";
import { Command, InvalidArgumentError } from "commander";
export interface CliArgs {
filter: string | undefined;
concurrency: number;
}
function parsePositiveInt(value: string): number {
const n = parseInt(value, 10);
if (isNaN(n) || n <= 0) {
throw new InvalidArgumentError("must be a positive integer");
}
return n;
}
export function parseArgs(argv: string[]): CliArgs {
const program = new Command();
program
.name("deterministic-tests")
.description("Scripted multi-client sync tests against a real server")
.option(
"-f, --filter <substring>",
"Run only tests whose name contains this substring"
)
.option(
"-j, --concurrency <number>",
"Number of tests to run in parallel",
parsePositiveInt,
os.cpus().length
);
program.parse(argv);
/* eslint-disable @typescript-eslint/no-unsafe-type-assertion */
const opts = program.opts();
const filter = opts.filter as string | undefined;
const concurrency = opts.concurrency as number;
/* eslint-enable @typescript-eslint/no-unsafe-type-assertion */
return { filter, concurrency };
}

View file

@ -0,0 +1,28 @@
import { Logger } from "sync-client";
export class PrefixedLogger extends Logger {
private readonly base: Logger;
private readonly prefix: string;
public constructor(base: Logger, prefix: string) {
super();
this.base = base;
this.prefix = prefix;
}
public override debug(message: string): void {
this.base.debug(`[${this.prefix}] ${message}`);
}
public override info(message: string): void {
this.base.info(`[${this.prefix}] ${message}`);
}
public override warn(message: string): void {
this.base.warn(`[${this.prefix}] ${message}`);
}
public override error(message: string): void {
this.base.error(`[${this.prefix}] ${message}`);
}
}

View file

@ -0,0 +1,33 @@
export async function runWithConcurrency<T, R>(
items: T[],
concurrency: number,
fn: (item: T) => Promise<R>
): Promise<R[]> {
const results: R[] = [];
const errors: unknown[] = [];
const executing = new Set<Promise<void>>();
for (let i = 0; i < items.length; i++) {
const index = i;
const p = fn(items[index])
.then((result) => {
results[index] = result;
})
.catch((error: unknown) => {
errors.push(error);
})
.finally(() => executing.delete(p));
executing.add(p);
if (executing.size >= concurrency) {
await Promise.race(executing);
}
}
// eslint-disable-next-line no-restricted-properties
await Promise.all(executing);
if (errors.length > 0) {
throw errors[0];
}
return results;
}

View file

@ -0,0 +1,296 @@
import { spawn, type ChildProcess } from "node:child_process";
import * as fs from "node:fs";
import * as os from "node:os";
import * as path from "node:path";
import { sleep } from "./utils/sleep";
import { findFreePort } from "./utils/find-free-port";
import type { Logger } from "sync-client";
import {
STOP_TIMEOUT_MS,
SERVER_READY_POLL_INTERVAL_MS,
SERVER_READY_MAX_ATTEMPTS,
SERVER_START_MAX_ATTEMPTS
} from "./consts";
export class ServerControl {
private process: ChildProcess | null = null;
private readonly serverPath: string;
private readonly baseConfigPath: string;
private readonly logger: Logger;
private _port: number | undefined;
private tempDir: string | undefined;
private _isPaused = false;
public constructor(serverPath: string, configPath: string, logger: Logger) {
this.serverPath = serverPath;
this.baseConfigPath = configPath;
this.logger = logger;
}
public get port(): number {
if (this._port === undefined) {
throw new Error("Server has not been started yet");
}
return this._port;
}
public get remoteUri(): string {
return `http://localhost:${this.port}`;
}
public async start(): Promise<void> {
if (this.process !== null) {
throw new Error("Server is already running");
}
// Retry on bind failure: findFreePort closes its probe before we
// spawn, so under heavy parallelism another process can grab the
// same port. Each attempt picks a fresh port.
let lastError: unknown;
for (let attempt = 1; attempt <= SERVER_START_MAX_ATTEMPTS; attempt++) {
try {
await this.startOnce();
return;
} catch (error) {
lastError = error;
this.logger.warn(
`Server start attempt ${attempt}/${SERVER_START_MAX_ATTEMPTS} failed: ${error instanceof Error ? error.message : String(error)}`
);
// startOnce already cleaned up its child + tempdir on failure.
}
}
throw new Error(
`Server failed to start after ${SERVER_START_MAX_ATTEMPTS} attempts: ${lastError instanceof Error ? lastError.message : String(lastError)}`,
{ cause: lastError instanceof Error ? lastError : undefined }
);
}
private async startOnce(): Promise<void> {
const reservation = await findFreePort();
this._port = reservation.port;
const tmpBase = os.tmpdir();
this.tempDir = fs.mkdtempSync(path.join(tmpBase, "vault-link-test-"));
const tempConfigPath = path.join(this.tempDir, "config.yml");
const dbDir = path.join(this.tempDir, "databases");
this.writeConfigFile(tempConfigPath, dbDir);
this.logger.info(
`Starting server: ${this.serverPath} (port ${this._port})`
);
// Release the port reservation right before spawning to minimize
// the TOCTOU window between port discovery and server binding.
reservation.release();
this.process = spawn(this.serverPath, [tempConfigPath], {
stdio: ["ignore", "pipe", "pipe"],
detached: false
});
this.process.stdout?.on("data", (data: Buffer) => {
this.logger.info(`[SERVER] ${data.toString().trim()}`);
});
this.process.stderr?.on("data", (data: Buffer) => {
this.logger.info(`[SERVER] ${data.toString().trim()}`);
});
this.process.on("error", (err) => {
this.logger.error(`[SERVER] Process error: ${err.message}`);
});
const currentProcess = this.process;
currentProcess.on("exit", (code, signal) => {
this.logger.info(
`Server exited with code ${code}, signal ${signal}`
);
// Only clear state if this handler is for the current process.
// A fast stop→start cycle could create a new process before this
// handler fires — clearing state here would corrupt the new one.
if (this.process === currentProcess) {
this.process = null;
this._isPaused = false;
}
});
try {
await this.waitForReady();
} catch (error) {
// Kill the spawned process if it failed to become ready,
// preventing a zombie process from lingering.
try {
await this.stop();
} catch {
// Best-effort cleanup
}
throw error;
}
}
public async waitForReady(
maxAttempts: number = SERVER_READY_MAX_ATTEMPTS
): Promise<void> {
const pingUrl = `${this.remoteUri}/vaults/test/ping`;
for (let i = 0; i < maxAttempts; i++) {
if (this.process?.exitCode !== null) {
throw new Error(
"Server process died while waiting for it to become ready"
);
}
try {
const response = await fetch(pingUrl);
if (response.ok) {
this.logger.info("[SERVER] Ready");
return;
}
} catch {
// Server not ready yet, continue polling
}
await sleep(SERVER_READY_POLL_INTERVAL_MS);
}
throw new Error("Server failed to start within timeout");
}
public pause(): void {
if (this.process?.pid === undefined) {
throw new Error("Server is not running");
}
if (this._isPaused) {
this.logger.warn("Server is already paused, skipping double-pause");
return;
}
this.logger.info("Server pausing...");
try {
process.kill(this.process.pid, "SIGSTOP");
this._isPaused = true;
this.logger.info("Server paused (SIGSTOP sent)");
} catch (error) {
throw new Error(
`Failed to pause server (pid ${this.process.pid}): ${error instanceof Error ? error.message : String(error)}`
);
}
}
public resume(): void {
if (this.process?.pid === undefined) {
throw new Error("Server is not running");
}
if (!this._isPaused) {
return;
}
this.logger.info("Server resuming...");
try {
process.kill(this.process.pid, "SIGCONT");
this._isPaused = false;
this.logger.info("Server resumed (SIGCONT sent)");
} catch (error) {
throw new Error(
`Failed to resume server (pid ${this.process.pid}): ${error instanceof Error ? error.message : String(error)}`
);
}
}
public async stop(): Promise<void> {
const proc = this.process;
if (proc?.pid === undefined) {
this.cleanupTempDir();
return;
}
// Resume if paused — a SIGSTOP'd process ignores SIGKILL
if (this._isPaused) {
try {
process.kill(proc.pid, "SIGCONT");
} catch {
// Process may already be gone
}
this._isPaused = false;
}
this.logger.info("Server stopping...");
// Set up a promise that resolves when the process actually exits.
const exitPromise = new Promise<void>((resolve) => {
if (proc.exitCode !== null) {
resolve();
return;
}
proc.on("exit", () => {
resolve();
});
});
try {
process.kill(proc.pid, "SIGKILL");
} catch {
// Process already gone
}
// Wait for the process to actually exit before cleaning up,
// with a 5s safety timeout to avoid hanging forever.
await Promise.race([exitPromise, sleep(STOP_TIMEOUT_MS)]);
this.process = null;
this._isPaused = false;
this.cleanupTempDir();
}
public isRunning(): boolean {
const proc = this.process;
return (
proc !== null &&
proc.pid !== undefined &&
proc.exitCode === null &&
proc.signalCode === null
);
}
/**
* Synchronously SIGCONT-then-SIGKILL the child process. Safe to call
* from a `process.on("exit", ...)` handler, where async work cannot
* run. Used as a last-resort cleanup so a SIGSTOP'd server doesn't
* outlive the test runner and wedge the next CI invocation.
*/
public forceKillSync(): void {
const proc = this.process;
if (proc?.pid === undefined) {
return;
}
try {
process.kill(proc.pid, "SIGCONT");
} catch {
// Process may already be gone or never paused.
}
try {
process.kill(proc.pid, "SIGKILL");
} catch {
// Process already gone.
}
}
private writeConfigFile(destPath: string, dbDir: string): void {
// Assumes config-e2e.yml has exactly one 2-space-indented `port:` and
// one `databases_directory_path:` (under `server:` and `database:`
// respectively)
const baseConfig = fs.readFileSync(this.baseConfigPath, "utf-8");
const config = baseConfig
.replace(/^\s*port:\s*\d+/m, ` port: ${this._port}`)
.replace(
/^\s*databases_directory_path:\s*.+/m,
` databases_directory_path: ${dbDir}`
);
fs.writeFileSync(destPath, config);
}
private cleanupTempDir(): void {
if (this.tempDir !== undefined) {
try {
fs.rmSync(this.tempDir, { recursive: true, force: true });
} catch {
// Best-effort cleanup
}
this.tempDir = undefined;
}
}
}

View file

@ -0,0 +1,71 @@
import type { ServerControl } from "./server-control";
import type { Logger } from "sync-client";
export class ServerManager {
private readonly activeServers = new Set<ServerControl>();
private readonly logger: Logger;
private isShuttingDown = false;
public constructor(logger: Logger) {
this.logger = logger;
}
public track(server: ServerControl): void {
this.activeServers.add(server);
}
public untrack(server: ServerControl): void {
this.activeServers.delete(server);
}
public async stopAll(): Promise<void> {
if (this.isShuttingDown) {
return;
}
this.isShuttingDown = true;
const servers = Array.from(this.activeServers);
// eslint-disable-next-line no-restricted-properties
await Promise.all(
servers.map(async (server) => {
try {
await server.stop();
} catch {
// Best-effort cleanup during shutdown
}
})
);
}
public installSignalHandlers(): void {
process.on("SIGINT", () => {
this.logger.info("Received SIGINT, shutting down...");
void this.stopAll()
.catch(() => {
/* no-op */
})
.then(() => process.exit(130));
});
process.on("SIGTERM", () => {
this.logger.info("Received SIGTERM, shutting down...");
void this.stopAll()
.catch(() => {
/* no-op */
})
.then(() => process.exit(143));
});
// Last-resort synchronous cleanup. Runs even when the process is
// exiting via process.exit() from unhandledRejection /
// uncaughtException — paths where async stopAll() cannot complete.
// SIGSTOP'd servers MUST receive SIGCONT before SIGKILL or the
// kernel keeps them as zombies holding the test's tmpdir, and the
// next CI run can't reuse the port.
process.on("exit", () => {
for (const server of this.activeServers) {
server.forceKillSync();
}
});
}
}

View file

@ -0,0 +1,49 @@
import type { AssertableState } from "./utils/assertable-state";
export interface ClientState {
files: Map<string, string>;
clientFiles: Map<string, string>[];
}
export type TestStep =
| { type: "create"; client: number; path: string; content: string }
| { type: "update"; client: number; path: string; content: string }
| { type: "rename"; client: number; oldPath: string; newPath: string }
| {
type: "rename-next-write";
client: number;
oldPath: string;
newPath: string;
}
| { type: "delete"; client: number; path: string }
| { type: "sync"; client?: number }
| { type: "disable-sync"; client: number }
| { type: "enable-sync"; client: number }
| { type: "pause-server" }
| { type: "resume-server" }
| {
type: "resume-server-until-history-then-pause";
client: number;
syncType: "CREATE" | "UPDATE" | "DELETE";
path: string;
}
| { type: "barrier" }
| { type: "assert-consistent"; verify?: (state: AssertableState) => void }
| { type: "pause-websocket"; client: number }
| { type: "resume-websocket"; client: number }
| { type: "drop-next-create-response"; client: number }
| { type: "wait-for-dropped-create-response"; client: number }
| { type: "sleep"; ms: number }
| { type: "reset"; client: number };
export interface TestDefinition {
description?: string;
clients: number;
steps: TestStep[];
}
export interface TestResult {
success: boolean;
error?: string;
duration?: number;
}

View file

@ -0,0 +1,245 @@
import type { TestDefinition } from "./test-definition";
import { renameCreateConflictTest } from "./tests/rename-create-conflict.test";
import { renameChainTest } from "./tests/rename-chain.test";
import { renameUpdateConflictTest } from "./tests/rename-update-conflict.test";
import { deleteRenameConflictTest } from "./tests/delete-rename-conflict.test";
import { multiFileOperationsTest } from "./tests/multi-file-operations.test";
import { deleteRecreateSamePathTest } from "./tests/delete-recreate-same-path.test";
import { offlineRenameAndEditTest } from "./tests/offline-rename-and-edit.test";
import { simultaneousCreateDeleteSamePathTest } from "./tests/simultaneous-create-delete-same-path.test";
import { idempotencyAfterServerPauseTest } from "./tests/idempotency-after-server-pause.test";
import { sequentialCreateDuplicateContentTest } from "./tests/sequential-create-duplicate-content.test";
import { mcThreeClientRenameOfflineUpdateTest } from "./tests/mc-three-client-rename-offline-update.test";
import { mcMultiDeleteOfflineRenameTest } from "./tests/mc-multi-delete-offline-rename.test";
import { mcCrossCreateRenameSameTargetTest } from "./tests/mc-cross-create-rename-same-target.test";
import { mcDeleteThenOfflineRenameTest } from "./tests/mc-delete-then-offline-rename.test";
import { offlineMixedOperationsTest } from "./tests/offline-mixed-operations.test";
import { offlineConcurrentRenamesTest } from "./tests/offline-concurrent-renames.test";
import { offlineMultipleEditsTest } from "./tests/offline-multiple-edits.test";
import { serverPauseBothClientsCreateTest } from "./tests/server-pause-both-clients-create.test";
import { serverPauseUpdateAndCreateTest } from "./tests/server-pause-update-and-create.test";
import { renameSwapTest } from "./tests/rename-swap.test";
import { renameCircularTest } from "./tests/rename-circular.test";
import { renameRoundtripTest } from "./tests/rename-roundtrip.test";
import { offlineRenameRemoteCreateOldPathTest } from "./tests/offline-rename-remote-create-old-path.test";
import { offlineEditRemoteRenameTest } from "./tests/offline-edit-remote-rename.test";
import { renameChainThenDeleteTest } from "./tests/rename-chain-then-delete.test";
import { offlineDeleteRemoteRenameTest } from "./tests/offline-delete-remote-rename.test";
import { overlappingEditsSameSectionTest } from "./tests/overlapping-edits-same-section.test";
import { rapidUpdatesAfterMergeTest } from "./tests/rapid-updates-after-merge.test";
import { deleteRecreateConcurrentUpdateTest } from "./tests/delete-recreate-concurrent-update.test";
import { moveAndConcurrentRemoteUpdateTest } from "./tests/move-and-concurrent-remote-update.test";
import { offlineDeleteVsRemoteUpdateTest } from "./tests/offline-delete-vs-remote-update.test";
import { doubleOfflineCycleTest } from "./tests/double-offline-cycle.test";
import { serverPauseRenameEditResumeTest } from "./tests/server-pause-rename-edit-resume.test";
import { offlineUpdateBothThenDeleteOneTest } from "./tests/offline-update-both-then-delete-one.test";
import { offlineCreateSamePathMergeableTest } from "./tests/offline-create-same-path-mergeable.test";
import { deleteDuringPendingCreateTest } from "./tests/delete-during-pending-create.test";
import { threeClientRenameCreateDeleteTest } from "./tests/three-client-rename-create-delete.test";
import { renameToPathOfUnconfirmedDeleteTest } from "./tests/rename-to-path-of-unconfirmed-delete.test";
import { offlineEditThenMoveSameContentTest } from "./tests/offline-edit-then-move-same-content.test";
import { rapidCreateUpdateDeleteCycleTest } from "./tests/rapid-create-update-delete-cycle.test";
import { serverPauseBothEditSameFileTest } from "./tests/server-pause-both-edit-same-file.test";
import { deleteRecreateDifferentContentTest } from "./tests/delete-recreate-different-content.test";
import { updateDuringCreateProcessingTest } from "./tests/update-during-create-processing.test";
import { offlineMoveThenRemoteDeleteTest } from "./tests/offline-move-then-remote-delete.test";
import { resetClearsRecentlyDeletedResurrectionTest } from "./tests/reset-clears-recently-deleted-resurrection.test";
import { moveThenDeleteStalePathTest } from "./tests/move-then-delete-stale-path.test";
import { interruptedDeleteRetryTest } from "./tests/interrupted-delete-retry.test";
import { updateDoesNotSurviveRemoteDeleteTest } from "./tests/update-does-not-survive-remote-delete.test";
import { movePreservesRemoteUpdateTest } from "./tests/move-preserves-remote-update.test";
import { recentlyDeletedClearedOnReconnectTest } from "./tests/recently-deleted-cleared-on-reconnect.test";
import { watermarkAdvancesOnSkipTest } from "./tests/watermark-advances-on-skip.test";
import { watermarkGapRemoteUpdateNotRecordedTest } from "./tests/watermark-gap-remote-update-not-recorded.test";
import { queueResetLosesCoalescedLocalEditTest } from "./tests/queue-reset-loses-coalesced-local-edit.test";
import { renameToPendingPathFallbackTest } from "./tests/rename-to-pending-path-fallback.test";
import { moveRemoteUpdateRevertsRenameTest } from "./tests/move-remote-update-reverts-rename.test";
import { localEditLostDuringCreateMergeTest } from "./tests/local-edit-lost-during-create-merge.test";
import { renamePendingCreateBeforeResponseTest } from "./tests/rename-pending-create-before-response.test";
import { createRenameResponseSkipsFileTest } from "./tests/create-rename-response-skips-file.test";
import { onlineCreateRenameConcurrentCreateOrphanTest } from "./tests/online-create-rename-concurrent-create-orphan.test";
import { concurrentRenameFirstWinsTest } from "./tests/concurrent-rename-first-wins.test";
import { binaryToTextTransitionTest } from "./tests/binary-to-text-transition.test";
import { textPendingCreateNotDisplacedTest } from "./tests/text-pending-create-not-displaced.test";
import { binaryPendingCreateNotDisplacedTest } from "./tests/binary-pending-create-not-displaced.test";
import { coalesceUpdateRemoteUpdateDataLossTest } from "./tests/coalesce-update-remote-update-data-loss.test";
import { coalescedRemoteUpdateWatermarkLossTest } from "./tests/coalesced-remote-update-watermark-loss.test";
import { concurrentDeleteDuringRemoteUpdateTest } from "./tests/concurrent-delete-during-remote-update.test";
import { concurrentEditExactSamePositionTest } from "./tests/concurrent-edit-exact-same-position.test";
import { concurrentRenameAndCreateAtTargetRenameFirstTest } from "./tests/concurrent-rename-and-create-at-target-rename-first.test";
import { concurrentRenameAndCreateAtTargetCreateFirstTest } from "./tests/concurrent-rename-and-create-at-target-create-first.test";
import { concurrentRenameSameTargetTest } from "./tests/concurrent-rename-same-target.test";
import { concurrentUpdateDiffConsistencyTest } from "./tests/concurrent-update-diff-consistency.test";
import { userParenthesizedFileNotDeletedTest } from "./tests/user-parenthesized-file-not-deleted.test";
import { createDeleteNoopTest } from "./tests/create-delete-noop.test";
import { createMergeDeleteTest } from "./tests/create-merge-delete.test";
import { moveIdenticalContentAmbiguityTest } from "./tests/move-identical-content-ambiguity.test";
import { createUpdateCoalesceServerPauseTest } from "./tests/create-update-coalesce-server-pause.test";
import { createDuringReconciliationTest } from "./tests/create-during-reconciliation.test";
import { createMergePreservesRenamedUpdateTest } from "./tests/create-merge-preserves-renamed-update.test";
import { createRenameCreateSamePathTest } from "./tests/create-rename-create-same-path.test";
import { moveChainThreeFilesTest } from "./tests/move-chain-three-files.test";
import { deleteByOtherClientThenRecreateTest } from "./tests/delete-by-other-client-then-recreate.test";
import { onlineDeleteRecreateRapidCycleTest } from "./tests/online-delete-recreate-rapid-cycle.test";
import { onlineEditVsDeleteConvergenceTest } from "./tests/online-edit-vs-delete-convergence.test";
import { rapidEditDeleteOnlineConvergenceTest } from "./tests/rapid-edit-delete-online-convergence.test";
import { serverPauseDeleteRecreateTest } from "./tests/server-pause-delete-recreate.test";
import { onlineBothCreateSamePathDeconflictTest } from "./tests/online-both-create-same-path-deconflict.test";
import { onlineCreateUpdateWhileOtherCreatesSamePathTest } from "./tests/online-create-update-while-other-creates-same-path.test";
import { displacedFileNotMarkedDeletedTest } from "./tests/displaced-file-not-marked-deleted.test";
import { remoteUpdateResurrectsDeletedDocTest } from "./tests/remote-update-resurrects-deleted-doc.test";
import { localUpdateSurvivesRemoteRenameTest } from "./tests/local-update-survives-remote-rename.test";
import { mergingUpdateResponseSurvivesUserRenameTest } from "./tests/merging-update-response-survives-user-rename.test";
import { catchupCreateAndUpdateNotSkippedTest } from "./tests/catchup-create-and-update-not-skipped.test";
import { localRenameSurvivesRemoteRenameTest } from "./tests/local-rename-survives-remote-rename.test";
import { renameChainDuringPendingCreateTest } from "./tests/rename-chain-during-pending-create.test";
import { remoteRenameCollidesWithPendingLocalCreateTest } from "./tests/remote-rename-collides-with-pending-local-create.test";
import { remoteUpdateSurvivesUserRenameTest } from "./tests/remote-update-survives-user-rename.test";
import { sameDocIdCollapseOnLocalCreateAfterRemoteCreateTest } from "./tests/same-doc-id-collapse-on-local-create-after-remote-create.test";
import { sameDocIdCollapseAfterRemoteQuickWriteAndPendingRenameTest } from "./tests/same-doc-id-collapse-after-remote-quick-write-and-pending-rename.test";
import { renameOverwritesPendingCreateThenDeleteTest } from "./tests/rename-overwrites-pending-create-then-delete.test";
import { deleteRecreatedPendingCreateWithStaleDeletingRecordTest } from "./tests/delete-recreated-pending-create-with-stale-deleting-record.test";
import { queuedCreateDeleteDoesNotHijackReusedPathTest } from "./tests/queued-create-delete-does-not-hijack-reused-path.test";
import { renamedPendingCreateReusedPathThenDeleteTest } from "./tests/renamed-pending-create-reused-path-then-delete.test";
import { renamePendingCreateOntoPendingDeletePathTest } from "./tests/rename-pending-create-onto-pending-delete-path.test";
import { remoteQuickWriteRenameBeforeRecordTest } from "./tests/remote-quick-write-rename-before-record.test";
import { selfMergePendingRenameAliasesSecondCreateTest } from "./tests/self-merge-pending-rename-aliases-second-create.test";
export const TESTS: Partial<Record<string, TestDefinition>> = {
"rename-create-conflict": renameCreateConflictTest,
"rename-chain": renameChainTest,
"rename-update-conflict": renameUpdateConflictTest,
"delete-rename-conflict": deleteRenameConflictTest,
"multi-file-operations": multiFileOperationsTest,
"delete-recreate-same-path": deleteRecreateSamePathTest,
"offline-rename-and-edit": offlineRenameAndEditTest,
"simultaneous-create-delete-same-path":
simultaneousCreateDeleteSamePathTest,
"idempotency-after-server-pause": idempotencyAfterServerPauseTest,
"sequential-create-duplicate-content": sequentialCreateDuplicateContentTest,
"mc-three-client-rename-offline-update":
mcThreeClientRenameOfflineUpdateTest,
"mc-multi-delete-offline-rename": mcMultiDeleteOfflineRenameTest,
"mc-cross-create-rename-same-target": mcCrossCreateRenameSameTargetTest,
"mc-delete-then-offline-rename": mcDeleteThenOfflineRenameTest,
"offline-mixed-operations": offlineMixedOperationsTest,
"offline-concurrent-renames": offlineConcurrentRenamesTest,
"offline-multiple-edits": offlineMultipleEditsTest,
"server-pause-both-clients-create": serverPauseBothClientsCreateTest,
"server-pause-update-and-create": serverPauseUpdateAndCreateTest,
"rename-swap": renameSwapTest,
"rename-circular": renameCircularTest,
"rename-roundtrip": renameRoundtripTest,
"offline-rename-remote-create-old-path":
offlineRenameRemoteCreateOldPathTest,
"offline-edit-remote-rename": offlineEditRemoteRenameTest,
"rename-chain-then-delete": renameChainThenDeleteTest,
"offline-delete-remote-rename": offlineDeleteRemoteRenameTest,
"overlapping-edits-same-section": overlappingEditsSameSectionTest,
"rapid-updates-after-merge": rapidUpdatesAfterMergeTest,
"delete-recreate-concurrent-update": deleteRecreateConcurrentUpdateTest,
"move-and-concurrent-remote-update": moveAndConcurrentRemoteUpdateTest,
"double-offline-cycle": doubleOfflineCycleTest,
"server-pause-rename-edit-resume": serverPauseRenameEditResumeTest,
"offline-update-both-then-delete-one": offlineUpdateBothThenDeleteOneTest,
"offline-create-same-path-mergeable": offlineCreateSamePathMergeableTest,
"delete-during-pending-create": deleteDuringPendingCreateTest,
"three-client-rename-create-delete": threeClientRenameCreateDeleteTest,
"rename-to-path-of-unconfirmed-delete": renameToPathOfUnconfirmedDeleteTest,
"offline-edit-then-move-same-content": offlineEditThenMoveSameContentTest,
"rapid-create-update-delete-cycle": rapidCreateUpdateDeleteCycleTest,
"server-pause-both-edit-same-file": serverPauseBothEditSameFileTest,
"delete-recreate-different-content": deleteRecreateDifferentContentTest,
"update-during-create-processing": updateDuringCreateProcessingTest,
"offline-move-then-remote-delete": offlineMoveThenRemoteDeleteTest,
"reset-clears-recently-deleted-resurrection":
resetClearsRecentlyDeletedResurrectionTest,
"move-then-delete-stale-path": moveThenDeleteStalePathTest,
"offline-delete-vs-remote-update": offlineDeleteVsRemoteUpdateTest,
"interrupted-delete-retry": interruptedDeleteRetryTest,
"update-does-not-survive-remote-delete": updateDoesNotSurviveRemoteDeleteTest,
"move-preserves-remote-update": movePreservesRemoteUpdateTest,
"recently-deleted-cleared-on-reconnect":
recentlyDeletedClearedOnReconnectTest,
"watermark-advances-on-skip": watermarkAdvancesOnSkipTest,
"watermark-gap-remote-update-not-recorded":
watermarkGapRemoteUpdateNotRecordedTest,
"queue-reset-loses-coalesced-local-edit":
queueResetLosesCoalescedLocalEditTest,
"rename-to-pending-path-fallback": renameToPendingPathFallbackTest,
"move-remote-update-reverts-rename": moveRemoteUpdateRevertsRenameTest,
"local-edit-lost-during-create-merge": localEditLostDuringCreateMergeTest,
"rename-pending-create-before-response":
renamePendingCreateBeforeResponseTest,
"create-rename-response-skips-file": createRenameResponseSkipsFileTest,
"online-create-rename-concurrent-create-orphan":
onlineCreateRenameConcurrentCreateOrphanTest,
"concurrent-rename-first-wins": concurrentRenameFirstWinsTest,
"binary-to-text-transition": binaryToTextTransitionTest,
"text-pending-create-not-displaced": textPendingCreateNotDisplacedTest,
"binary-pending-create-not-displaced": binaryPendingCreateNotDisplacedTest,
"coalesce-update-remote-update-data-loss":
coalesceUpdateRemoteUpdateDataLossTest,
"coalesced-remote-update-watermark-loss":
coalescedRemoteUpdateWatermarkLossTest,
"concurrent-delete-during-remote-update":
concurrentDeleteDuringRemoteUpdateTest,
"concurrent-edit-exact-same-position": concurrentEditExactSamePositionTest,
"concurrent-rename-and-create-at-target-rename-first":
concurrentRenameAndCreateAtTargetRenameFirstTest,
"concurrent-rename-and-create-at-target-create-first":
concurrentRenameAndCreateAtTargetCreateFirstTest,
"concurrent-rename-same-target": concurrentRenameSameTargetTest,
"concurrent-update-diff-consistency": concurrentUpdateDiffConsistencyTest,
"user-parenthesized-file-not-deleted": userParenthesizedFileNotDeletedTest,
"create-delete-noop": createDeleteNoopTest,
"create-merge-delete": createMergeDeleteTest,
"move-identical-content-ambiguity": moveIdenticalContentAmbiguityTest,
"create-update-coalesce-server-pause": createUpdateCoalesceServerPauseTest,
"create-during-reconciliation": createDuringReconciliationTest,
"create-merge-preserves-renamed-update":
createMergePreservesRenamedUpdateTest,
"create-rename-create-same-path": createRenameCreateSamePathTest,
"move-chain-three-files": moveChainThreeFilesTest,
"delete-by-other-client-then-recreate": deleteByOtherClientThenRecreateTest,
"online-delete-recreate-rapid-cycle": onlineDeleteRecreateRapidCycleTest,
"online-edit-vs-delete-convergence": onlineEditVsDeleteConvergenceTest,
"rapid-edit-delete-online-convergence":
rapidEditDeleteOnlineConvergenceTest,
"server-pause-delete-recreate": serverPauseDeleteRecreateTest,
"online-both-create-same-path-deconflict":
onlineBothCreateSamePathDeconflictTest,
"online-create-update-while-other-creates-same-path":
onlineCreateUpdateWhileOtherCreatesSamePathTest,
"displaced-file-not-marked-deleted": displacedFileNotMarkedDeletedTest,
"remote-update-resurrects-deleted-doc":
remoteUpdateResurrectsDeletedDocTest,
"local-update-survives-remote-rename": localUpdateSurvivesRemoteRenameTest,
"merging-update-response-survives-user-rename":
mergingUpdateResponseSurvivesUserRenameTest,
"catchup-create-and-update-not-skipped":
catchupCreateAndUpdateNotSkippedTest,
"local-rename-survives-remote-rename": localRenameSurvivesRemoteRenameTest,
"rename-chain-during-pending-create": renameChainDuringPendingCreateTest,
"remote-rename-collides-with-pending-local-create":
remoteRenameCollidesWithPendingLocalCreateTest,
"remote-update-survives-user-rename": remoteUpdateSurvivesUserRenameTest,
"same-doc-id-collapse-on-local-create-after-remote-create":
sameDocIdCollapseOnLocalCreateAfterRemoteCreateTest,
"renamed-pending-create-reused-path-then-delete":
renamedPendingCreateReusedPathThenDeleteTest,
"rename-pending-create-onto-pending-delete-path":
renamePendingCreateOntoPendingDeletePathTest,
"rename-overwrites-pending-create-then-delete":
renameOverwritesPendingCreateThenDeleteTest,
"same-doc-id-collapse-after-remote-quick-write-and-pending-rename":
sameDocIdCollapseAfterRemoteQuickWriteAndPendingRenameTest,
"delete-recreated-pending-create-with-stale-deleting-record":
deleteRecreatedPendingCreateWithStaleDeletingRecordTest,
"queued-create-delete-does-not-hijack-reused-path":
queuedCreateDeleteDoesNotHijackReusedPathTest,
"remote-quick-write-rename-before-record":
remoteQuickWriteRenameBeforeRecordTest,
"self-merge-pending-rename-aliases-second-create":
selfMergePendingRenameAliasesSecondCreateTest
};

View file

@ -0,0 +1,399 @@
import type { TestDefinition, TestResult, TestStep } from "./test-definition";
import { DeterministicAgent } from "./deterministic-agent";
import type { ServerControl } from "./server-control";
import type { SyncSettings, Logger } from "sync-client";
import { assert } from "./utils/assert";
import { AssertableState } from "./utils/assertable-state";
import { sleep } from "./utils/sleep";
import { withTimeout } from "./utils/with-timeout";
import {
CONVERGENCE_TIMEOUT_MS,
CONVERGENCE_RETRY_DELAY_MS,
AGENT_INIT_TIMEOUT_MS,
IS_SYNC_ENABLED_BY_DEFAULT
} from "./consts";
import { randomUUID } from "node:crypto";
export class TestRunner {
private agents: DeterministicAgent[] = [];
private readonly serverControl: ServerControl;
private readonly token: string;
private readonly remoteUri: string;
private readonly logger: Logger;
public constructor(
serverControl: ServerControl,
logger: Logger,
token: string,
remoteUri: string
) {
this.serverControl = serverControl;
this.logger = logger;
this.token = token;
this.remoteUri = remoteUri;
}
public async runTest(
name: string,
test: TestDefinition
): Promise<TestResult> {
const startTime = Date.now();
this.logger.info(`Running test: ${name}`);
if (test.description !== undefined && test.description !== "") {
this.logger.info(`Description: ${test.description}`);
}
this.logger.info(`Clients: ${test.clients}`);
this.logger.info(`Steps: ${test.steps.length}`);
try {
assert(
this.serverControl.isRunning(),
"Server is not running before test start"
);
await this.initializeAgents(test.clients);
for (let i = 0; i < test.steps.length; i++) {
const step = test.steps[i];
this.logger.info(
`Step ${i + 1}/${test.steps.length}: ${JSON.stringify(step)}`
);
await this.executeStep(step);
}
await this.cleanup();
const duration = Date.now() - startTime;
this.logger.info(`\n✓ Test passed: ${name} (${duration}ms)`);
return {
success: true,
duration
};
} catch (error) {
const duration = Date.now() - startTime;
const errorMessage =
error instanceof Error ? error.message : String(error);
this.logger.info(`\n✗ Test failed: ${name}`);
this.logger.info(`Error: ${errorMessage}`);
await this.cleanup();
return {
success: false,
error: errorMessage,
duration
};
}
}
private async initializeAgents(count: number): Promise<void> {
assert(count > 0, `Client count must be positive, got ${count}`);
const vaultName = `test-${randomUUID()}`;
this.logger.info(
`Initializing ${count} agents with vault: ${vaultName}`
);
for (let i = 0; i < count; i++) {
const settings: Partial<SyncSettings> = {
isSyncEnabled: IS_SYNC_ENABLED_BY_DEFAULT,
token: this.token,
vaultName,
remoteUri: this.remoteUri
};
const agent = new DeterministicAgent(i, settings, (msg) => {
this.logger.info(msg);
});
// Push before init so cleanup() handles this agent if init fails
this.agents.push(agent);
await withTimeout(
agent.init(fetch),
AGENT_INIT_TIMEOUT_MS,
`Client ${i} init timed out after ${AGENT_INIT_TIMEOUT_MS}ms`
);
this.logger.info(`Initialized client ${i}`);
}
this.logger.info("All agents initialized");
}
private getAgent(index: number): DeterministicAgent {
assert(
index >= 0 && index < this.agents.length,
`Client index ${index} out of bounds (have ${this.agents.length} agents)`
);
return this.agents[index];
}
private async executeStep(step: TestStep): Promise<void> {
switch (step.type) {
case "create":
case "update":
await this.getAgent(step.client).write(
step.path,
new TextEncoder().encode(step.content)
);
break;
case "rename":
await this.getAgent(step.client).rename(
step.oldPath,
step.newPath
);
break;
case "rename-next-write":
this.getAgent(step.client).renameNextWrite(
step.oldPath,
step.newPath
);
break;
case "delete":
await this.getAgent(step.client).delete(step.path);
break;
case "sync":
if (step.client !== undefined) {
await this.getAgent(step.client).waitForSync();
} else {
for (const agent of this.agents) {
await agent.waitForSync();
}
}
break;
case "disable-sync":
await this.getAgent(step.client).disableSync();
break;
case "enable-sync":
await this.getAgent(step.client).enableSync();
break;
case "pause-server":
this.serverControl.pause();
break;
case "resume-server":
this.serverControl.resume();
// Verify the server is actually responsive before proceeding.
// This replaces relying solely on hardcoded waits.
await this.serverControl.waitForReady();
break;
case "resume-server-until-history-then-pause": {
const agent = this.getAgent(step.client);
const historySeen = agent.waitForHistoryEntry(
(entry) =>
entry.details.type === step.syncType &&
entry.details.relativePath === step.path,
() => this.serverControl.pause()
);
this.serverControl.resume();
await historySeen;
break;
}
case "barrier":
await this.waitForConvergence();
break;
case "assert-consistent":
await this.assertConsistent(step.verify);
break;
case "pause-websocket":
this.getAgent(step.client).pauseWebSocket();
break;
case "resume-websocket":
this.getAgent(step.client).resumeWebSocket();
break;
case "drop-next-create-response":
this.getAgent(step.client).dropNextCreateResponse();
break;
case "wait-for-dropped-create-response":
await this.getAgent(step.client).waitForDroppedCreateResponse();
break;
case "sleep":
await sleep(step.ms);
break;
case "reset":
await this.getAgent(step.client).reset();
break;
default: {
const unknownStep = step as { type: string };
throw new Error(`Unknown step type: ${unknownStep.type}`);
}
}
}
/**
* Wait for all agents to reach a consistent state.
*
* Waiting for agents is done in two full rounds: the first round
* drains in-flight operations, but completing those operations can
* trigger new work on OTHER agents via server broadcasts. The second
* round waits for that cascading work to settle. Deeper cascades
* are handled by the outer retry loop.
*/
private async waitForConvergence(): Promise<void> {
this.logger.info("Barrier: waiting for convergence...");
const deadline = Date.now() + CONVERGENCE_TIMEOUT_MS;
let lastError: Error | undefined = undefined;
while (Date.now() < deadline) {
await this.waitAllAgentsSettled();
try {
await this.assertConsistent();
this.logger.info("Barrier complete: all clients converged");
return;
} catch (error) {
lastError =
error instanceof Error ? error : new Error(String(error));
this.logger.info("Barrier: not yet converged, retrying...");
await sleep(CONVERGENCE_RETRY_DELAY_MS);
}
}
throw new Error(
`Convergence timed out after ${CONVERGENCE_TIMEOUT_MS}ms: ${lastError?.message ?? "no consistency check ran"}`,
{ cause: lastError }
);
}
/**
* Wait for all agents to be simultaneously idle.
*
* Completing work on agent A can trigger a server broadcast that
* enqueues new work on agent B, which can cascade further. With N
* agents the worst-case cascade depth is N (a chain ABCA),
* so we run N+1 sequential passes to drain it. Extra passes are
* essentially free when there is no outstanding work.
*
* The outer {@link waitForConvergence} loop with consistency checks
* remains the ultimate guarantee this method just minimizes how
* many slow retry iterations are needed.
*/
private async waitAllAgentsSettled(): Promise<void> {
const rounds = this.agents.length + 1;
for (let round = 0; round < rounds; round++) {
for (const agent of this.agents) {
await agent.waitForSync();
}
}
}
private async assertConsistent(
verify?: (state: AssertableState) => void
): Promise<void> {
this.logger.info("Asserting all clients are consistent...");
assert(
this.agents.length >= 2,
"Need at least 2 agents for consistency check"
);
// Snapshot all agents' file states upfront to minimize the window
// where background sync could mutate state between reads.
const clientFiles: Map<string, string>[] = [];
for (const agent of this.agents) {
const sortedFiles = (await agent.listFilesRecursively()).sort();
const fileMap = new Map<string, string>();
for (const file of sortedFiles) {
const content = await agent.getFileContent(file);
fileMap.set(file, content);
}
clientFiles.push(fileMap);
}
const referenceFiles = Array.from(clientFiles[0].keys());
this.logger.info(
`Reference client has ${referenceFiles.length} files: ${referenceFiles.join(", ")}`
);
for (let i = 1; i < clientFiles.length; i++) {
const agentFileKeys = Array.from(clientFiles[i].keys());
this.logger.info(
`Client ${i} has ${agentFileKeys.length} files: ${agentFileKeys.join(", ")}`
);
assert(
agentFileKeys.length === referenceFiles.length,
`File count mismatch: client 0 has ${referenceFiles.length} files, client ${i} has ${agentFileKeys.length} files`
);
for (let j = 0; j < agentFileKeys.length; j++) {
assert(
agentFileKeys[j] === referenceFiles[j],
`File list mismatch at index ${j}: client 0 has "${referenceFiles[j]}", client ${i} has "${agentFileKeys[j]}"`
);
}
for (const file of referenceFiles) {
const referenceContent = clientFiles[0].get(file);
const agentContent = clientFiles[i].get(file);
assert(
referenceContent === agentContent,
`Content mismatch for ${file}:\nClient 0: "${referenceContent}"\nClient ${i}: "${agentContent}"`
);
}
}
this.logger.info("✓ All clients are consistent");
if (verify) {
this.logger.info("Running custom verification...");
try {
verify(
new AssertableState({
files: clientFiles[0],
clientFiles
})
);
} catch (error) {
const msg =
error instanceof Error ? error.message : String(error);
throw new Error(`Custom verification failed: ${msg}`);
}
this.logger.info("✓ Custom verification passed");
}
}
private async cleanup(): Promise<void> {
// Always resume the server in case a test paused it and then
// failed before reaching the resume step. Without this, all
// subsequent tests would hang because the server process is
// frozen (SIGSTOP) and can't respond to HTTP or WebSocket.
try {
this.serverControl.resume();
} catch {
// Server wasn't paused or isn't running — safe to ignore
}
this.logger.info("\nCleaning up agents...");
for (const agent of this.agents) {
try {
await agent.cleanup();
} catch (error) {
this.logger.warn(
`Agent cleanup error: ${error instanceof Error ? error.message : String(error)}`
);
}
}
this.agents = [];
this.logger.info("Cleanup complete");
}
}

View file

@ -0,0 +1,40 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const binaryPendingCreateNotDisplacedTest: TestDefinition = {
description:
"Two clients each create a binary file at the same path while offline. " +
"After syncing, both files should exist on both clients at separate paths.",
clients: 2,
steps: [
{
type: "create",
client: 0,
path: "data.bin",
content: "binary data from client 0"
},
{
type: "create",
client: 1,
path: "data.bin",
content: "binary data from client 1"
},
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertFileCount(2)
.assertFileExists("data.bin")
.assertFileExists("data (1).bin")
.assertAnyFileContains(
"binary data from client 0",
"binary data from client 1"
);
}
}
]
};

View file

@ -0,0 +1,97 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const binaryToTextTransitionTest: TestDefinition = {
description:
"A .bin file is created and synced. Both clients edit it offline " +
"(binary last-write-wins), then client 0 renames it to .md and " +
"writes a clean text baseline. Both clients edit different sections " +
"offline. The text merge should preserve both edits.",
clients: 2,
steps: [
{
type: "create",
client: 0,
path: "data.bin",
content: "original content"
},
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertContent("data.bin", "original content");
}
},
{ type: "disable-sync", client: 0 },
{ type: "disable-sync", client: 1 },
{ type: "update", client: 0, path: "data.bin", content: "version A" },
{ type: "update", client: 1, path: "data.bin", content: "version B" },
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertFileCount(1).assertContainsAny(
"data.bin",
"version A",
"version B"
);
}
},
{ type: "disable-sync", client: 1 },
{ type: "rename", client: 0, oldPath: "data.bin", newPath: "data.md" },
{
type: "update",
client: 0,
path: "data.md",
content: "top line\nmiddle line\nbottom line"
},
{ type: "sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertContent(
"data.md",
"top line\nmiddle line\nbottom line"
);
}
},
{ type: "disable-sync", client: 0 },
{ type: "disable-sync", client: 1 },
{
type: "update",
client: 0,
path: "data.md",
content: "alpha\nmiddle line\nbottom line"
},
{
type: "update",
client: 1,
path: "data.md",
content: "top line\nmiddle line\nbeta"
},
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertFileCount(1).assertContains("data.md", "alpha", "beta");
}
}
]
};

View file

@ -0,0 +1,66 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const catchupCreateAndUpdateNotSkippedTest: TestDefinition = {
description:
"Client 1 disconnects (sync disabled). Client 0 creates a doc and " +
"then updates it. When Client 1 reconnects, the server's catch-up " +
"stream sends only the doc's *latest* version (the update), not the " +
"full history. Pre-fix the wire's `is_new_file` was set to " +
"`creation == latest_version`, so the catch-up flagged the doc as " +
"non-new even though Client 1 had never seen its creation. Client " +
"1's `processRemoteChange` then dropped it as a 'stale RemoteChange " +
"for untracked, non-new document' and the doc was silently lost. " +
"Post-fix `is_new_file` in the catch-up stream means 'new relative " +
"to the recipient's watermark' (`creation > last_seen_vault_update_id`).",
clients: 2,
steps: [
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
// Establish a baseline so Client 1's last_seen is non-zero before
// we take it offline. This makes the bug genuinely about catch-up
// missing the create rather than just an empty-vault first sync.
{ type: "create", client: 0, path: "warmup.md", content: "w\n" },
{ type: "barrier" },
// Client 1 goes offline.
{ type: "disable-sync", client: 1 },
// Client 0 creates the doc (vault_update_id v_C, after Client 1's
// watermark). Client 1 doesn't see this because it's offline.
{ type: "create", client: 0, path: "doc.md", content: "v1\n" },
// Wait for the create's HTTP to land before the update; otherwise
// both writes are coalesced into a single POST and the server
// never sees the doc as "create followed by update".
{ type: "sync", client: 0 },
// Client 0 updates the doc (vault_update_id v_X > v_C). The
// server's `latest_document_versions` view now returns the
// *update* row — its `creation_vault_update_id != vault_update_id`.
{
type: "update",
client: 0,
path: "doc.md",
content: "v1\nupdate\n"
},
{ type: "sync", client: 0 },
// Client 1 reconnects. Server's catch-up replays docs with
// `vault_update_id > last_seen`. For doc.md it sends v_X with
// `is_new_file` derived from `creation_vault_update_id >
// last_seen_vault_update_id` (post-fix) — so Client 1 treats it
// as a fresh create and downloads the latest content.
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (state: AssertableState): void => {
state.assertFileCount(2);
state.assertFileExists("doc.md");
state.assertContent("doc.md", "v1\nupdate\n");
state.assertContent("warmup.md", "w\n");
}
}
]
};

View file

@ -0,0 +1,59 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const coalesceUpdateRemoteUpdateDataLossTest: TestDefinition = {
description:
"Divergent offline edits with text-merge expectation. Client 0's " +
"remote update fully lands before Client 1 reconnects (`sync`-after " +
"the c0 update enforces this), so Client 1's offline edit merges " +
"against a server-known version, not a coalesced batch. Both " +
"additions must survive in the final merged content. (Filename's " +
"'coalesce' framing is aspirational — a true update-coalesce test " +
"would skip the c0 sync and queue overlapping local + remote " +
"updates against the same parent version.)",
clients: 2,
steps: [
{
type: "create",
client: 0,
path: "doc.md",
content: "line 1\nline 2\nline 3"
},
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{ type: "disable-sync", client: 1 },
{
type: "update",
client: 0,
path: "doc.md",
content: "line 1\nline 2\nline 3\nclient 0 addition"
},
{ type: "sync", client: 0 },
{
type: "update",
client: 1,
path: "doc.md",
content: "client 1 addition\nline 1\nline 2\nline 3"
},
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (state: AssertableState): void => {
state
.assertFileCount(1)
.assertContains(
"doc.md",
"client 0 addition",
"client 1 addition"
);
}
}
]
};

View file

@ -0,0 +1,53 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const coalescedRemoteUpdateWatermarkLossTest: TestDefinition = {
description:
"Client 0 sends three rapid updates. After syncing, both clients " +
"disconnect and reconnect twice. Content should remain correct " +
"after each reconnect.",
clients: 2,
steps: [
{ type: "create", client: 0, path: "doc.md", content: "original" },
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{ type: "update", client: 0, path: "doc.md", content: "update 1" },
{ type: "update", client: 0, path: "doc.md", content: "update 2" },
{ type: "update", client: 0, path: "doc.md", content: "final update" },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertFileCount(1).assertContent("doc.md", "final update");
}
},
{ type: "disable-sync", client: 0 },
{ type: "disable-sync", client: 1 },
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertFileCount(1).assertContent("doc.md", "final update");
}
},
{ type: "disable-sync", client: 0 },
{ type: "disable-sync", client: 1 },
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertFileCount(1).assertContent("doc.md", "final update");
}
}
]
};

View file

@ -0,0 +1,32 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const concurrentDeleteDuringRemoteUpdateTest: TestDefinition = {
description:
"One client updates a file while the other deletes it at the same " +
"time. Both clients should converge without errors.",
clients: 2,
steps: [
{ type: "create", client: 0, path: "doc.md", content: "original" },
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{ type: "disable-sync", client: 0 },
{ type: "disable-sync", client: 1 },
{ type: "update", client: 0, path: "doc.md", content: "updated by 0" },
{ type: "delete", client: 1, path: "doc.md" },
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (state: AssertableState): void => {
state.assertFileCount(0);
}
}
]
};

View file

@ -0,0 +1,49 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const concurrentEditExactSamePositionTest: TestDefinition = {
description:
"Both clients replace the same word in a file with different text " +
"while offline. After syncing, the merged result should contain " +
"both replacements.",
clients: 2,
steps: [
{
type: "create",
client: 0,
path: "doc.md",
content: "the quick brown fox"
},
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{ type: "disable-sync", client: 0 },
{ type: "disable-sync", client: 1 },
{
type: "update",
client: 0,
path: "doc.md",
content: "the slow brown fox"
},
{
type: "update",
client: 1,
path: "doc.md",
content: "the fast brown fox"
},
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (state: AssertableState): void => {
state
.assertFileCount(1)
.assertContains("doc.md", "slow", "fast", "brown fox");
}
}
]
};

View file

@ -0,0 +1,49 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const concurrentRenameAndCreateAtTargetCreateFirstTest: TestDefinition = {
description:
"One client renames X to Y while another creates a new file at Y, " +
"both offline. After syncing, Y should contain merged content from " +
"both the renamed file and the newly created file.",
clients: 2,
steps: [
{
type: "create",
client: 0,
path: "X.md",
content: "original file X"
},
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{ type: "disable-sync", client: 0 },
{ type: "disable-sync", client: 1 },
{ type: "rename", client: 0, oldPath: "X.md", newPath: "Y.md" },
{
type: "create",
client: 1,
path: "Y.md",
content: "brand new Y content"
},
{ type: "enable-sync", client: 1 },
{ type: "sync", client: 1 },
{ type: "enable-sync", client: 0 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (state: AssertableState): void => {
state
.assertFileCount(2)
.assertContains("Y (1).md", "original file X")
.assertContains("Y.md", "brand new Y content");
}
}
]
};

View file

@ -0,0 +1,52 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const concurrentRenameAndCreateAtTargetRenameFirstTest: TestDefinition = {
description:
"One client renames X to Y while another creates a new file at Y, " +
"both offline. We can't merge the create because it would result in a cycle",
clients: 2,
steps: [
{
type: "create",
client: 0,
path: "X.md",
content: "original file X"
},
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{ type: "disable-sync", client: 0 },
{ type: "disable-sync", client: 1 },
{ type: "rename", client: 0, oldPath: "X.md", newPath: "Y.md" },
{
type: "create",
client: 1,
path: "Y.md",
content: "brand new Y content"
},
{ type: "enable-sync", client: 0 },
{ type: "sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (state: AssertableState): void => {
state
.assertFileNotExists("X.md")
.assertFileExists("Y.md")
.assertFileExists("Y (1).md")
.assertAnyFileContains(
"original file X",
"brand new Y content"
);
}
}
]
};

View file

@ -0,0 +1,61 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const concurrentRenameFirstWinsTest: TestDefinition = {
description:
"Both clients start online with the same file. Both go offline, " +
"rename the file to different paths, and edit it. When they reconnect, " +
"the first rename to reach the server wins the path and both content " +
"edits are merged.",
clients: 2,
steps: [
{
type: "create",
client: 0,
path: "A.md",
content: "line 1\nline 2\nline 3"
},
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertContent("A.md", "line 1\nline 2\nline 3");
}
},
{ type: "disable-sync", client: 0 },
{ type: "disable-sync", client: 1 },
{ type: "rename", client: 0, oldPath: "A.md", newPath: "B.md" },
{
type: "update",
client: 0,
path: "B.md",
content: "edit from 0\nline 2\nline 3"
},
{ type: "rename", client: 1, oldPath: "A.md", newPath: "C.md" },
{
type: "update",
client: 1,
path: "C.md",
content: "line 1\nline 2\nedit from 1"
},
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertFileNotExists("A.md")
.assertFileCount(2)
.assertContent("B.md", "edit from 0\nline 2\nline 3")
.assertContent("C.md", "line 1\nline 2\nedit from 1");
}
}
]
};

View file

@ -0,0 +1,39 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const concurrentRenameSameTargetTest: TestDefinition = {
description:
"One client renames A to C while the other renames B to C, both offline. " +
"After syncing, both file contents should be preserved via path deconfliction.",
clients: 2,
steps: [
{ type: "create", client: 0, path: "A.md", content: "content-a" },
{ type: "create", client: 0, path: "B.md", content: "content-b" },
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{ type: "disable-sync", client: 1 },
{ type: "rename", client: 0, oldPath: "A.md", newPath: "C.md" },
{ type: "sync", client: 0 },
{ type: "rename", client: 1, oldPath: "B.md", newPath: "C.md" },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (state: AssertableState): void => {
state
.assertFileCount(2)
.assertFileNotExists("A.md")
.assertFileNotExists("B.md")
.assertFileExists("C.md")
.assertFileExists("C (1).md")
.assertAnyFileContains("content-a", "content-b");
}
}
]
};

View file

@ -0,0 +1,51 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const concurrentUpdateDiffConsistencyTest: TestDefinition = {
description:
"Both clients edit different sections of the same file while offline. " +
"After syncing, the merged file should contain both edits.",
clients: 2,
steps: [
{
type: "create",
client: 0,
path: "doc.md",
content: "header\nmiddle\nfooter"
},
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{ type: "disable-sync", client: 0 },
{ type: "disable-sync", client: 1 },
{
type: "update",
client: 0,
path: "doc.md",
content: "header by 0\nmiddle\nfooter"
},
{
type: "update",
client: 1,
path: "doc.md",
content: "header\nmiddle\nfooter by 1"
},
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (state: AssertableState): void => {
state
.assertFileCount(1)
.assertContent(
"doc.md",
"header by 0\nmiddle\nfooter by 1"
);
}
}
]
};

View file

@ -0,0 +1,27 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const createDeleteNoopTest: TestDefinition = {
description:
"A client creates a file, updates it multiple times, then deletes it, all while " +
"offline. After syncing, neither client should have the file.",
clients: 2,
steps: [
{ type: "enable-sync", client: 1 },
{ type: "create", client: 0, path: "temp.md", content: "version 1" },
{ type: "update", client: 0, path: "temp.md", content: "version 2" },
{ type: "update", client: 0, path: "temp.md", content: "version 3" },
{ type: "delete", client: 0, path: "temp.md" },
{ type: "enable-sync", client: 0 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertFileNotExists("temp.md");
}
}
]
};

View file

@ -0,0 +1,50 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const createDuringReconciliationTest: TestDefinition = {
description:
"Client creates two files while offline, reconnects, then immediately " +
"creates a third file. All three files should sync to the other client.",
clients: 2,
steps: [
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{ type: "disable-sync", client: 0 },
{
type: "create",
client: 0,
path: "A.md",
content: "offline A"
},
{
type: "create",
client: 0,
path: "B.md",
content: "offline B"
},
{ type: "enable-sync", client: 0 },
{
type: "create",
client: 0,
path: "C.md",
content: "post-reconnect C"
},
{ type: "barrier" },
{
type: "assert-consistent",
verify: (state: AssertableState): void => {
state
.assertFileCount(3)
.assertContent("A.md", "offline A")
.assertContent("B.md", "offline B")
.assertContent("C.md", "post-reconnect C");
}
}
]
};

View file

@ -0,0 +1,37 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const createMergeDeleteTest: TestDefinition = {
description:
"Two clients create A.md offline with different content. Both come online and " +
"the content is merged. Then one client deletes A.md. Both clients should " +
"converge on an empty state.",
clients: 2,
steps: [
{ type: "create", client: 0, path: "A.md", content: "from-zero" },
{ type: "create", client: 1, path: "A.md", content: "from-one" },
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (state: AssertableState): void => {
state
.assertFileCount(1)
.assertContains("A.md", "from-zero", "from-one");
}
},
{ type: "delete", client: 0, path: "A.md" },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertFileCount(0).assertFileNotExists("A.md");
}
}
]
};

View file

@ -0,0 +1,59 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const createMergePreservesRenamedUpdateTest: TestDefinition = {
description:
"Both clients create the same file, which gets merged. One client goes " +
"offline, renames the file, updates it, and creates a new file at the " +
"original path. After reconnecting, the updated content must be preserved.",
clients: 2,
steps: [
{ type: "create", client: 0, path: "doc.md", content: "alpha" },
{ type: "create", client: 1, path: "doc.md", content: "beta" },
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (state: AssertableState): void => {
state.assertContains("doc.md", "alpha", "beta");
}
},
{ type: "disable-sync", client: 1 },
{
type: "rename",
client: 1,
oldPath: "doc.md",
newPath: "moved.md"
},
{
type: "update",
client: 1,
path: "moved.md",
content: "alpha beta extra-update"
},
{
type: "create",
client: 1,
path: "doc.md",
content: "new-content"
},
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (state: AssertableState): void => {
state
.assertContent("moved.md", "alpha beta extra-update")
.assertContent("doc.md", "new-content");
}
}
]
};

View file

@ -0,0 +1,34 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const createRenameCreateSamePathTest: TestDefinition = {
description:
"Client creates A.md, renames to B.md, creates new A.md, renames " +
"to C.md, creates yet another A.md. All three files should exist " +
"as separate documents on both clients.",
clients: 2,
steps: [
{ type: "create", client: 0, path: "A.md", content: "first file" },
{ type: "rename", client: 0, oldPath: "A.md", newPath: "B.md" },
{ type: "create", client: 0, path: "A.md", content: "second file" },
{ type: "rename", client: 0, oldPath: "A.md", newPath: "C.md" },
{ type: "create", client: 0, path: "A.md", content: "third file" },
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (state: AssertableState): void => {
state
.assertFileCount(3)
.assertContent("B.md", "first file")
.assertContent("C.md", "second file")
.assertContent("A.md", "third file");
}
}
]
};

View file

@ -0,0 +1,36 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const createRenameResponseSkipsFileTest: TestDefinition = {
description:
"Client 0 creates a file online then immediately renames it. " +
"Client 1 must receive the file content at the renamed path.",
clients: 2,
steps: [
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{
type: "create",
client: 0,
path: "doc.md",
content: "the-content"
},
{
type: "rename",
client: 0,
oldPath: "doc.md",
newPath: "renamed.md"
},
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertFileCount(1).assertAnyFileContains("the-content");
}
}
]
};

View file

@ -0,0 +1,32 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const createUpdateCoalesceServerPauseTest: TestDefinition = {
description:
"Client creates a file and immediately updates it while the server is " +
"paused. When the server resumes, both clients should have the final " +
"updated content.",
clients: 2,
steps: [
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "pause-server" },
{ type: "create", client: 0, path: "doc.md", content: "initial" },
{ type: "update", client: 0, path: "doc.md", content: "final version" },
{ type: "resume-server" },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (state: AssertableState): void => {
state
.assertFileCount(1)
.assertContent("doc.md", "final version");
}
}
]
};

View file

@ -0,0 +1,40 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const deleteByOtherClientThenRecreateTest: TestDefinition = {
description:
"Client 1 deletes a file and the delete propagates. Then client 0 " +
"creates a new file at the same path. Both clients must have the file.",
clients: 2,
steps: [
{ type: "create", client: 0, path: "A.md", content: "original" },
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{ type: "delete", client: 1, path: "A.md" },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertFileNotExists("A.md");
}
},
{
type: "create",
client: 0,
path: "A.md",
content: "recreated by client 0"
},
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertContent("A.md", "recreated by client 0");
}
}
]
};

View file

@ -0,0 +1,35 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const deleteDuringPendingCreateTest: TestDefinition = {
description:
"Client 0 creates a file while the server is paused, then deletes it before the server resumes. " +
"After resume, the file should end up deleted on both clients.",
clients: 2,
steps: [
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{ type: "pause-server" },
{
type: "create",
client: 0,
path: "ephemeral.md",
content: "this will be deleted"
},
{ type: "delete", client: 0, path: "ephemeral.md" },
{ type: "resume-server" },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertFileCount(0).assertFileNotExists("ephemeral.md");
}
}
]
};

View file

@ -0,0 +1,42 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const deleteRecreateConcurrentUpdateTest: TestDefinition = {
description:
"Client 0 deletes and recreates A.md with new content while offline. Client 1 updates A.md concurrently. " +
"After client 0 reconnects, both clients must converge with client 0's recreated content preserved.",
clients: 2,
steps: [
{ type: "create", client: 0, path: "A.md", content: "original" },
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{ type: "disable-sync", client: 0 },
{ type: "delete", client: 0, path: "A.md" },
{
type: "create",
client: 0,
path: "A.md",
content: "recreated by client 0"
},
{
type: "update",
client: 1,
path: "A.md",
content: "updated by client 1"
},
{ type: "sync", client: 1 },
{ type: "enable-sync", client: 0 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertFileExists("A.md").assertContains("A.md", "recreated");
}
}
]
};

View file

@ -0,0 +1,54 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const deleteRecreateDifferentContentTest: TestDefinition = {
description:
"Client 0 deletes and recreates A.md with new content offline while client 1 edits A.md offline. " +
"Both clients should converge with content from both sides merged.",
clients: 2,
steps: [
{
type: "create",
client: 0,
path: "A.md",
content: "original content here"
},
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{ type: "disable-sync", client: 0 },
{ type: "disable-sync", client: 1 },
{ type: "delete", client: 0, path: "A.md" },
{
type: "create",
client: 0,
path: "A.md",
content: "brand new content"
},
{
type: "update",
client: 1,
path: "A.md",
content: "edit from client 1"
},
{ type: "enable-sync", client: 0 },
{ type: "sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertFileCount(1).assertContains(
"A.md",
"brand new",
"client 1"
);
}
}
]
};

View file

@ -0,0 +1,34 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const deleteRecreateSamePathTest: TestDefinition = {
description:
"Client 0 creates A.md, syncs. Then deletes A.md and creates a new A.md " +
"with different content. Both clients should converge on the new content.",
clients: 2,
steps: [
{ type: "create", client: 0, path: "A.md", content: "version 1" },
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertContent("A.md", "version 1");
}
},
{ type: "disable-sync", client: 0 },
{ type: "delete", client: 0, path: "A.md" },
{ type: "create", client: 0, path: "A.md", content: "version 2" },
{ type: "enable-sync", client: 0 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertContent("A.md", "version 2");
}
}
]
};

View file

@ -0,0 +1,52 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const deleteRecreatedPendingCreateWithStaleDeletingRecordTest: TestDefinition =
{
description:
"A local delete for a recreated pending create must target the " +
"new pending create, not an older same-path record whose server " +
"delete has been acked but whose WebSocket delete receipt is " +
"still paused.",
clients: 2,
steps: [
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{ type: "pause-websocket", client: 0 },
{ type: "pause-server" },
{
type: "create",
client: 0,
path: "binary-14.bin",
content: "BINARY:first"
},
{ type: "sleep", ms: 100 },
{ type: "delete", client: 0, path: "binary-14.bin" },
{ type: "resume-server" },
{ type: "sync", client: 0 },
{ type: "pause-server" },
{
type: "create",
client: 0,
path: "binary-14.bin",
content: "BINARY:second"
},
{ type: "sleep", ms: 100 },
{ type: "delete", client: 0, path: "binary-14.bin" },
{ type: "resume-server" },
{ type: "sync", client: 0 },
{ type: "resume-websocket", client: 0 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (state: AssertableState): void => {
state.assertFileCount(0);
}
}
]
};

View file

@ -0,0 +1,43 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const deleteRenameConflictTest: TestDefinition = {
description:
"Client 0 deletes A.md while client 1 renames A.md to C.md offline. " +
"After client 1 reconnects, both clients should converge to the same state.",
clients: 2,
steps: [
{ type: "create", client: 0, path: "A.md", content: "content-a" },
{ type: "create", client: 0, path: "B.md", content: "content-b" },
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertFileExists("A.md").assertFileExists("B.md");
}
},
{ type: "disable-sync", client: 1 },
{ type: "delete", client: 0, path: "A.md" },
{ type: "sync", client: 0 },
{ type: "rename", client: 1, oldPath: "A.md", newPath: "C.md" },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertContent("B.md", "content-b");
s.assertFileNotExists("A.md");
s.ifFileExists("C.md", (inner) =>
inner.assertContent("C.md", "content-a")
);
}
}
]
};

View file

@ -0,0 +1,38 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const displacedFileNotMarkedDeletedTest: TestDefinition = {
description:
"Client 0 creates a new file at path B.md while client 1 renames " +
"A.md to B.md. The remote download of B.md displaces client 1's " +
"renamed file. The displaced document must not be permanently " +
"marked as recently deleted, so it can still be synced.",
clients: 2,
steps: [
{ type: "create", client: 0, path: "A.md", content: "content of A" },
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{ type: "disable-sync", client: 1 },
{ type: "create", client: 0, path: "B.md", content: "content of B" },
{ type: "rename", client: 0, oldPath: "A.md", newPath: "C.md" },
{ type: "sync", client: 0 },
{ type: "rename", client: 1, oldPath: "A.md", newPath: "B.md" },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (state: AssertableState): void => {
state
.assertFileCount(2)
.assertContent("B.md", "content of B")
.assertContent("C.md", "content of A");
}
}
]
};

View file

@ -0,0 +1,77 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const doubleOfflineCycleTest: TestDefinition = {
description:
"Client 0 goes through three offline-edit-reconnect cycles. " +
"Each offline edit must propagate to client 1 after reconnection.",
clients: 2,
steps: [
{
type: "create",
client: 0,
path: "doc.md",
content: "initial"
},
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertContent("doc.md", "initial");
}
},
{ type: "disable-sync", client: 0 },
{
type: "update",
client: 0,
path: "doc.md",
content: "first edit"
},
{ type: "enable-sync", client: 0 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertContent("doc.md", "first edit");
}
},
{ type: "disable-sync", client: 0 },
{
type: "update",
client: 0,
path: "doc.md",
content: "second edit"
},
{ type: "enable-sync", client: 0 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertContent("doc.md", "second edit");
}
},
{ type: "disable-sync", client: 0 },
{
type: "update",
client: 0,
path: "doc.md",
content: "third edit"
},
{ type: "enable-sync", client: 0 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertFileCount(1).assertContent("doc.md", "third edit");
}
}
]
};

View file

@ -0,0 +1,33 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const idempotencyAfterServerPauseTest: TestDefinition = {
description:
"Client 0 creates a file, then the server is paused mid-response. " +
"After the server resumes, both clients must converge to a single copy of the file with no duplicates.",
clients: 2,
steps: [
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "create",
client: 0,
path: "doc.md",
content: "important data"
},
{ type: "pause-server" },
{ type: "resume-server" },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertFileCount(1).assertContent("doc.md", "important data");
}
}
]
};

View file

@ -0,0 +1,29 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const interruptedDeleteRetryTest: TestDefinition = {
description:
"Client 0 deletes a file, then the server is paused. " +
"After the server resumes, both clients should have zero files.",
clients: 2,
steps: [
{ type: "create", client: 0, path: "doc.md", content: "to be deleted" },
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{ type: "delete", client: 0, path: "doc.md" },
{ type: "pause-server" },
{ type: "resume-server" },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertFileCount(0);
}
}
]
};

View file

@ -0,0 +1,41 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const localEditLostDuringCreateMergeTest: TestDefinition = {
description:
"Both clients create doc.md with different content while offline. " +
"Client 0 also edits the file before syncing. After both connect, " +
"the merged result should contain content from both clients.",
clients: 2,
steps: [
{ type: "create", client: 1, path: "doc.md", content: "from-client-1" },
{
type: "create",
client: 0,
path: "doc.md",
content: "from-client-0"
},
{
type: "update",
client: 0,
path: "doc.md",
content: "local-edit-during-create"
},
{ type: "enable-sync", client: 1 },
{ type: "sync", client: 1 },
{ type: "enable-sync", client: 0 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertFileCount(1).assertContains(
"doc.md",
"from-client-1",
"local-edit-during-create"
);
}
}
]
};

View file

@ -0,0 +1,80 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const localRenameSurvivesRemoteRenameTest: TestDefinition = {
description:
"Drain processes a RemoteChange (remote rename for doc D) while a " +
"LocalUpdate (user rename of D) is also queued behind it. " +
"`processRemoteUpdate` moves the disk file and, because there is a " +
"pending LocalUpdate, takes the else branch — but its setDocument " +
"uses the stale `record.path` (= the user-rename target) instead of " +
"the actualPath the file just moved to. The queued LocalUpdate then " +
"reads from `record.path`, throws FileNotFoundError, and is " +
"silently dropped. Setup pins the queue order: a sentinel " +
"LocalUpdate keeps drain busy on a SIGSTOPped HTTP roundtrip while " +
"we resume client 0's WebSocket (enqueues RemoteChange) and then " +
"user-rename D (enqueues LocalUpdate after the RemoteChange). On " +
"server resume the drain pops the sentinel, then RemoteChange, then " +
"LocalUpdate — exactly the order that triggers the bug.",
clients: 2,
steps: [
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "create", client: 0, path: "doc.md", content: "v1\n" },
{ type: "create", client: 0, path: "sentinel.md", content: "s\n" },
{ type: "barrier" },
// Pause client 0's WebSocket so the upcoming remote rename buffers.
{ type: "pause-websocket", client: 0 },
// Server applies remote rename of doc.md -> remote.md. Broadcast
// is buffered on client 0's WebSocket.
{ type: "rename", client: 1, oldPath: "doc.md", newPath: "remote.md" },
{ type: "sync", client: 1 },
// Pause the server BEFORE arming the sentinel, so the sentinel's
// HTTP request will buffer at the kernel and keep drain occupied.
{ type: "pause-server" },
// Sentinel: a LocalUpdate on a *different* doc that drain pops
// first. Its HTTP roundtrip stalls on SIGSTOP, freezing drain
// until we resume the server. While drain is frozen we can grow
// the queue with additional events whose order we control.
{
type: "update",
client: 0,
path: "sentinel.md",
content: "s\nedit\n"
},
// Resume the WebSocket — buffered remote rename enqueues as a
// RemoteChange. Drain is still stuck on the sentinel HTTP.
{ type: "resume-websocket", client: 0 },
// User renames doc.md -> local.md on client 0. queue.enqueue
// mutates the doc's record.path to "local.md" and pushes a
// LocalUpdate(rename) onto the tail of the queue. Queue is now
// [sentinel-update (in-flight), RemoteChange, LocalUpdate-rename].
{ type: "rename", client: 0, oldPath: "doc.md", newPath: "local.md" },
// Resume the server. Drain pops sentinel-update (succeeds), then
// RemoteChange. Pre-fix: processRemoteUpdate moves disk
// local.md -> remote.md, takes the else branch, and
// setDocument(record.path = "local.md", …) leaves record.path
// stale. Drain pops the LocalUpdate-rename and reads from the
// stale record.path, hits FileNotFoundError, silent skip.
// Post-fix: when a local event is pending, we re-queue the
// remote update without touching disk or record, so the local
// rename drains first and both ends converge.
{ type: "resume-server" },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (state: AssertableState): void => {
state.assertFileCount(2);
}
}
]
};

View file

@ -0,0 +1,69 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const localUpdateSurvivesRemoteRenameTest: TestDefinition = {
description:
"Client 0 has a local content edit pending while a remote rename for " +
"the same doc arrives over the WebSocket. The remote rename's internal " +
"move relocates the disk file from the old path (where the user wrote) " +
"to the new server path. Previously, the queued LocalUpdate's " +
"`event.path` was left pointing at the now-vacated old path, so " +
"`skipIfOversized`'s `getFileSize(event.path)` threw " +
"`FileNotFoundError`, which `processEvent`'s catch silently swallowed " +
"as 'Skipping sync event 'local-update' because the file no longer " +
"exists' — and the user's edit was lost. The fix routes the size " +
"check through `tracked.path` (the doc's current disk path), " +
"matching the path `processLocalUpdate` itself reads from.",
clients: 2,
steps: [
{ type: "create", client: 0, path: "doc.md", content: "v1\n" },
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
// Pause client 0's WebSocket so the upcoming remote rename buffers
// there until we've already enqueued client 0's local content
// edit. This guarantees the LocalUpdate sits in client 0's queue
// when the rename's RemoteChange drains.
{ type: "pause-websocket", client: 0 },
{
type: "rename",
client: 1,
oldPath: "doc.md",
newPath: "renamed.md"
},
{ type: "sync", client: 1 },
// Client 0 still believes the file is at `doc.md` (its WebSocket is
// paused, so the rename hasn't reached it). The user edits content
// at `doc.md`. This pushes a LocalUpdate(D, path=doc.md,
// originalPath=doc.md, isUserRename=false) into client 0's queue.
{
type: "update",
client: 0,
path: "doc.md",
content: "v1\nclient 0 edit\n"
},
// Resume the WebSocket. The buffered remote rename (server-broadcast)
// drains. `processRemoteUpdate` does an internal `move(doc.md,
// renamed.md)` and, because there's a pending LocalUpdate for D,
// takes the else branch (re-enqueue v_K, setDocument(renamed.md, …)).
// Then drain reaches the LocalUpdate. Pre-fix: skipped silently.
// Post-fix: PUTs the user's content to the doc (at its new path,
// since this is a content-only edit, not a user rename).
{ type: "resume-websocket", client: 0 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (state: AssertableState): void => {
state.assertFileCount(1);
state.assertFileExists("renamed.md");
state.assertContent("renamed.md", "v1\nclient 0 edit\n");
}
}
]
};

View file

@ -0,0 +1,46 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const mcCrossCreateRenameSameTargetTest: TestDefinition = {
description:
"Client 0 creates X.md, Client 1 creates Y.md. Both sync. Client 0 renames " +
"X.md -> Z.md. Client 1 (offline) renames Y.md -> Z.md. Both must converge " +
"with both contents preserved via path deconfliction.",
clients: 2,
steps: [
{ type: "create", client: 0, path: "X.md", content: "content-x" },
{ type: "create", client: 1, path: "Y.md", content: "content-y" },
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertFileExists("X.md").assertFileExists("Y.md");
}
},
{ type: "disable-sync", client: 1 },
{ type: "rename", client: 0, oldPath: "X.md", newPath: "Z.md" },
{ type: "sync", client: 0 },
{ type: "rename", client: 1, oldPath: "Y.md", newPath: "Z.md" },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertFileCount(2)
.assertFileNotExists("X.md")
.assertFileNotExists("Y.md")
.assertFileExists("Z.md")
.assertAnyFileContains("content-x", "content-y");
}
}
]
};

View file

@ -0,0 +1,39 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const mcDeleteThenOfflineRenameTest: TestDefinition = {
description:
"Client 0 creates A.md, both sync. Client 1 goes offline. Client 0 deletes " +
"A.md and syncs. Client 1 (offline) renames A.md to B.md. Client 1 reconnects. " +
"Both must converge. C.md (unrelated) must be unaffected.",
clients: 2,
steps: [
{ type: "create", client: 0, path: "A.md", content: "original" },
{ type: "create", client: 0, path: "C.md", content: "unrelated" },
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{ type: "disable-sync", client: 1 },
{ type: "delete", client: 0, path: "A.md" },
{ type: "sync", client: 0 },
{ type: "rename", client: 1, oldPath: "A.md", newPath: "B.md" },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertContent("C.md", "unrelated").assertFileNotExists(
"A.md"
);
s.ifFileExists("B.md", (inner) =>
inner.assertContent("B.md", "original")
);
}
}
]
};

View file

@ -0,0 +1,49 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const mcMultiDeleteOfflineRenameTest: TestDefinition = {
description:
"Client 0 creates 5 files. Client 1 deletes 2 while Client 0 (offline) " +
"renames one of the deleted files. Both must converge.",
clients: 2,
steps: [
{ type: "create", client: 0, path: "file-1.md", content: "content-1" },
{ type: "create", client: 0, path: "file-2.md", content: "content-2" },
{ type: "create", client: 0, path: "file-3.md", content: "content-3" },
{ type: "create", client: 0, path: "file-4.md", content: "content-4" },
{ type: "create", client: 0, path: "file-5.md", content: "content-5" },
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{ type: "disable-sync", client: 0 },
{ type: "delete", client: 1, path: "file-2.md" },
{ type: "delete", client: 1, path: "file-4.md" },
{ type: "sync", client: 1 },
{
type: "rename",
client: 0,
oldPath: "file-2.md",
newPath: "renamed.md"
},
{ type: "enable-sync", client: 0 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertFileExists("file-1.md")
.assertFileExists("file-3.md")
.assertFileExists("file-5.md")
.assertFileNotExists("file-2.md")
.assertFileNotExists("file-4.md");
s.ifFileExists("renamed.md", (inner) =>
inner.assertContent("renamed.md", "content-2")
);
}
}
]
};

View file

@ -0,0 +1,41 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const mcThreeClientRenameOfflineUpdateTest: TestDefinition = {
description:
"Client 0 creates A.md. Client 1 renames to B.md. Client 2 (offline) " +
"updates A.md. All three converge with updated content at B.md.",
clients: 3,
steps: [
{ type: "create", client: 0, path: "A.md", content: "original" },
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "enable-sync", client: 2 },
{ type: "barrier" },
{ type: "disable-sync", client: 2 },
{ type: "rename", client: 1, oldPath: "A.md", newPath: "B.md" },
{ type: "sync", client: 1 },
{ type: "sync", client: 0 },
{
type: "update",
client: 2,
path: "A.md",
content: "updated-by-client-2"
},
{ type: "enable-sync", client: 2 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertFileCount(1)
.assertFileNotExists("A.md")
.assertContains("B.md", "updated-by-client-2");
}
}
]
};

View file

@ -0,0 +1,77 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const mergingUpdateResponseSurvivesUserRenameTest: TestDefinition = {
description:
"Client 1 sends a content update with a stale `parent_version_id` " +
"(its WebSocket is paused, so it hasn't seen Client 0's intervening " +
"edit). The server merges and replies with `MergingUpdate` carrying " +
"the merged text. Before the response lands, the user renames the " +
"doc on Client 1, vacating the disk path the in-flight " +
"`processLocalUpdate` captured. Pre-fix: " +
"`handleMaybeMergingResponse`'s `operations.write(diskPath, …)` " +
"hits the `we wont recreate it` early-return inside `write`, " +
"silently dropping the server-merged content — Client 0's edit is " +
"lost on Client 1's disk, and Client 1's next local-update PUT " +
"(rebased on the now-untracked merged version) deletes Client 0's " +
"edit on the server too. Post-fix: the response is written to the " +
"doc's current tracked disk path, preserving both edits.",
clients: 2,
steps: [
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "create", client: 0, path: "doc.md", content: "0\n" },
{ type: "barrier" },
// Stop Client 1 from seeing Client 0's next edit, so its next
// outbound PUT carries a stale `parent_version_id` and the server
// is forced to merge.
{ type: "pause-websocket", client: 1 },
// Server now holds v_b = "0\nA\n". Client 1's tracked parent
// version stays at v_a = "0\n".
{ type: "update", client: 0, path: "doc.md", content: "0\nA\n" },
{ type: "sync", client: 0 },
// Pause the server. Subsequent HTTP PUTs from Client 1 buffer at
// the OS layer until resume. This guarantees the merge response
// for Client 1's update is still in flight when the rename below
// mutates `queue.documents`.
{ type: "pause-server" },
// Client 1 edits doc.md with "B". The drain pops the LocalUpdate,
// captures `diskPath = "doc.md"`, reads the file, and sends the
// HTTP PUT — which buffers because the server is SIGSTOPped.
{ type: "update", client: 1, path: "doc.md", content: "0\nB\n" },
// User renames the file while the previous PUT is still in flight.
// `queue.enqueue`'s rename branch updates `documents` to point at
// `renamed.md` synchronously, but `processLocalUpdate`'s captured
// `diskPath` ("doc.md") is a local — it can't be retargeted.
{ type: "rename", client: 1, oldPath: "doc.md", newPath: "renamed.md" },
// Resume the server. It reconciles parent=v_a, latest=v_b,
// new="0\nB\n" → v_c with both edits, replies `MergingUpdate`.
// Pre-fix: write("doc.md", …) sees no file at that path
// (renamed.md now holds the data) and bails out without ever
// writing the merged bytes. Post-fix: the merged bytes land at
// the tracked path (renamed.md).
{ type: "resume-server" },
{ type: "resume-websocket", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (state: AssertableState): void => {
state.assertFileCount(1);
state.assertFileExists("renamed.md");
state.assertFileNotExists("doc.md");
// Both edits survive: Client 0's "A" and Client 1's "B".
// The reconcile may interleave them either way; assert
// both tokens are present in the converged content.
state.assertContains("renamed.md", "A", "B");
}
}
]
};

View file

@ -0,0 +1,43 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const moveAndConcurrentRemoteUpdateTest: TestDefinition = {
description:
"Client 0 renames A.md to B.md offline while client 1 updates A.md. " +
"After client 0 reconnects, both should have B.md with client 1's updated content.",
clients: 2,
steps: [
{
type: "create",
client: 0,
path: "A.md",
content: "original content"
},
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{ type: "disable-sync", client: 0 },
{ type: "rename", client: 0, oldPath: "A.md", newPath: "B.md" },
{
type: "update",
client: 1,
path: "A.md",
content: "updated by client 1"
},
{ type: "sync", client: 1 },
{ type: "enable-sync", client: 0 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertFileCount(1)
.assertFileNotExists("A.md")
.assertContains("B.md", "updated by client 1");
}
}
]
};

View file

@ -0,0 +1,42 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const moveChainThreeFilesTest: TestDefinition = {
description:
"Three files have their contents rotated (A gets C's content, B gets A's, C gets B's) " +
"while offline. After reconnecting, both clients should converge with the rotated contents.",
clients: 2,
steps: [
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "create", client: 0, path: "A.md", content: "was A" },
{ type: "create", client: 0, path: "B.md", content: "was B" },
{ type: "create", client: 0, path: "C.md", content: "was C" },
{ type: "barrier" },
{ type: "disable-sync", client: 0 },
{ type: "delete", client: 0, path: "A.md" },
{ type: "delete", client: 0, path: "B.md" },
{ type: "delete", client: 0, path: "C.md" },
{ type: "create", client: 0, path: "A.md", content: "was C" },
{ type: "create", client: 0, path: "B.md", content: "was A" },
{ type: "create", client: 0, path: "C.md", content: "was B" },
{ type: "enable-sync", client: 0 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (state: AssertableState): void => {
state
.assertFileCount(3)
.assertContent("A.md", "was C")
.assertContent("B.md", "was A")
.assertContent("C.md", "was B");
}
}
]
};

View file

@ -0,0 +1,44 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const moveIdenticalContentAmbiguityTest: TestDefinition = {
description:
"Two files with identical content exist. One is deleted and the other renamed " +
"while offline. The system should still converge correctly despite the ambiguity.",
clients: 2,
steps: [
{
type: "create",
client: 0,
path: "A.md",
content: "identical content"
},
{
type: "create",
client: 0,
path: "B.md",
content: "identical content"
},
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{ type: "disable-sync", client: 1 },
{ type: "delete", client: 1, path: "A.md" },
{ type: "rename", client: 1, oldPath: "B.md", newPath: "C.md" },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (state: AssertableState): void => {
state
.assertFileCount(1)
.assertFileNotExists("A.md")
.assertFileNotExists("B.md")
.assertContent("C.md", "identical content");
}
}
]
};

View file

@ -0,0 +1,48 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const movePreservesRemoteUpdateTest: TestDefinition = {
description:
"Client 0 renames a file offline while client 1 edits it offline. " +
"After both reconnect, the renamed file should contain client 1's edit.",
clients: 2,
steps: [
{
type: "create",
client: 0,
path: "doc.md",
content: "line 1\nline 2"
},
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{ type: "disable-sync", client: 0 },
{ type: "disable-sync", client: 1 },
{ type: "rename", client: 0, oldPath: "doc.md", newPath: "renamed.md" },
{
type: "update",
client: 1,
path: "doc.md",
content: "line 1\nclient 1 edit\nline 2"
},
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertFileCount(1);
const [content] = Array.from(s.files.values());
if (!content.includes("client 1 edit")) {
throw new Error(
`Expected merged content to include "client 1 edit", got: "${content}"`
);
}
}
}
]
};

View file

@ -0,0 +1,38 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const moveRemoteUpdateRevertsRenameTest: TestDefinition = {
description:
"Client 1 updates a file while client 0 is offline. Client 0 reconnects and renames the file. " +
"Both clients should converge with client 1's updated content.",
clients: 2,
steps: [
{ type: "create", client: 0, path: "doc.md", content: "original" },
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{ type: "disable-sync", client: 0 },
{
type: "update",
client: 1,
path: "doc.md",
content: "updated by client 1"
},
{ type: "sync", client: 1 },
{ type: "enable-sync", client: 0 },
{ type: "rename", client: 0, oldPath: "doc.md", newPath: "renamed.md" },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertFileCount(1).assertContent(
"renamed.md",
"updated by client 1"
);
}
}
]
};

View file

@ -0,0 +1,34 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const moveThenDeleteStalePathTest: TestDefinition = {
description:
"Client 0 renames A.md to B.md and immediately deletes B.md. " +
"Both clients should end up with zero files.",
clients: 2,
steps: [
{
type: "create",
client: 0,
path: "A.md",
content: "content to delete"
},
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{ type: "rename", client: 0, oldPath: "A.md", newPath: "B.md" },
{ type: "delete", client: 0, path: "B.md" },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertFileCount(0)
.assertFileNotExists("A.md")
.assertFileNotExists("B.md");
}
}
]
};

View file

@ -0,0 +1,45 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const multiFileOperationsTest: TestDefinition = {
description:
"Client 0 deletes A.md while client 1 is offline. Client 1 updates B.md and renames A.md to D.md offline. " +
"After client 1 reconnects, both clients must converge with B.md updated and C.md intact.",
clients: 2,
steps: [
{ type: "create", client: 0, path: "A.md", content: "content-a" },
{ type: "create", client: 0, path: "B.md", content: "content-b" },
{ type: "create", client: 0, path: "C.md", content: "content-c" },
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{ type: "disable-sync", client: 1 },
{ type: "delete", client: 0, path: "A.md" },
{ type: "sync", client: 0 },
{
type: "update",
client: 1,
path: "B.md",
content: "updated by client 1"
},
{ type: "rename", client: 1, oldPath: "A.md", newPath: "D.md" },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertContains("B.md", "updated")
.assertFileExists("C.md")
.assertFileNotExists("A.md");
s.ifFileExists("D.md", (inner) =>
inner.assertContent("D.md", "content-a")
);
}
}
]
};

View file

@ -0,0 +1,59 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const offlineConcurrentRenamesTest: TestDefinition = {
description:
"Client 0 creates A.md and syncs to both clients. Both clients go offline. " +
"Client 0 renames A.md to B.md. Client 1 renames A.md to C.md. " +
"Both reconnect. The system must converge -- both clients should " +
"agree on the final state and the content must not be lost.",
clients: 2,
steps: [
{ type: "create", client: 0, path: "A.md", content: "shared-content" },
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertContent("A.md", "shared-content");
}
},
{ type: "disable-sync", client: 0 },
{ type: "disable-sync", client: 1 },
{
type: "rename",
client: 0,
oldPath: "A.md",
newPath: "B.md"
},
{
type: "rename",
client: 1,
oldPath: "A.md",
newPath: "C.md"
},
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertFileNotExists("A.md")
.assertFileCount(1)
.assertAnyFileContains("shared-content");
s.ifFileExists("B.md", (inner) =>
inner.assertContent("B.md", "shared-content")
);
s.ifFileExists("C.md", (inner) =>
inner.assertContent("C.md", "shared-content")
);
}
}
]
};

View file

@ -0,0 +1,41 @@
import type { AssertableState } from "../utils/assertable-state";
import type { TestDefinition } from "../test-definition";
export const offlineCreateSamePathMergeableTest: TestDefinition = {
description:
"Both clients create a file at the same path while offline with different text content. " +
"After both sync, both clients must converge to a merged result containing both contributions.",
clients: 2,
steps: [
{
type: "create",
client: 0,
path: "notes.md",
content: "alpha wrote this line"
},
{
type: "create",
client: 1,
path: "notes.md",
content: "beta wrote this different line"
},
{ type: "enable-sync", client: 0 },
{ type: "sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
{
type: "assert-consistent",
verify: (s: AssertableState): void => {
s.assertFileCount(1)
.assertFileExists("notes.md")
.assertContains(
"notes.md",
"alpha wrote this line",
"beta wrote this different line"
);
}
}
]
};

Some files were not shown because too many files have changed in this diff Show more