Add docs
This commit is contained in:
parent
56c1f4d58b
commit
50a95b114d
19 changed files with 4663 additions and 1 deletions
65
.github/workflows/deploy-docs.yml
vendored
Normal file
65
.github/workflows/deploy-docs.yml
vendored
Normal file
|
|
@ -0,0 +1,65 @@
|
||||||
|
name: Deploy Documentation
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
paths:
|
||||||
|
- 'docs/**'
|
||||||
|
- '.github/workflows/deploy-docs.yml'
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
pages: write
|
||||||
|
id-token: write
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: pages
|
||||||
|
cancel-in-progress: false
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
|
||||||
|
- name: Setup Node
|
||||||
|
uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: 20
|
||||||
|
cache: npm
|
||||||
|
cache-dependency-path: docs/package-lock.json
|
||||||
|
|
||||||
|
- name: Setup Pages
|
||||||
|
uses: actions/configure-pages@v4
|
||||||
|
|
||||||
|
- name: Install dependencies
|
||||||
|
run: |
|
||||||
|
cd docs
|
||||||
|
npm ci
|
||||||
|
|
||||||
|
- name: Build documentation
|
||||||
|
run: |
|
||||||
|
cd docs
|
||||||
|
npm run build
|
||||||
|
|
||||||
|
- name: Upload artifact
|
||||||
|
uses: actions/upload-pages-artifact@v3
|
||||||
|
with:
|
||||||
|
path: docs/.vitepress/dist
|
||||||
|
|
||||||
|
deploy:
|
||||||
|
environment:
|
||||||
|
name: github-pages
|
||||||
|
url: ${{ steps.deployment.outputs.page_url }}
|
||||||
|
needs: build
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
name: Deploy
|
||||||
|
steps:
|
||||||
|
- name: Deploy to GitHub Pages
|
||||||
|
id: deployment
|
||||||
|
uses: actions/deploy-pages@v4
|
||||||
13
CLAUDE.md
13
CLAUDE.md
|
|
@ -29,7 +29,10 @@ cd sync-server
|
||||||
cargo run config-e2e.yml # Start development server
|
cargo run config-e2e.yml # Start development server
|
||||||
cargo test --verbose # Run Rust tests
|
cargo test --verbose # Run Rust tests
|
||||||
cargo clippy --all-targets --all-features # Lint Rust code
|
cargo clippy --all-targets --all-features # Lint Rust code
|
||||||
|
cargo clippy --all-targets --all-features --fix --allow-dirty --allow-staged # Auto-fix clippy warnings
|
||||||
cargo fmt --all -- --check # Check Rust formatting
|
cargo fmt --all -- --check # Check Rust formatting
|
||||||
|
cargo fmt --all # Auto-format Rust code
|
||||||
|
cargo machete --with-metadata # Detect unused dependencies
|
||||||
```
|
```
|
||||||
|
|
||||||
### Frontend Development
|
### Frontend Development
|
||||||
|
|
@ -49,8 +52,15 @@ sqlx migrate run --source src/app_state/database/migrations --database-url sqlit
|
||||||
cargo sqlx prepare --workspace
|
cargo sqlx prepare --workspace
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Initial Setup
|
||||||
|
```bash
|
||||||
|
# Install required cargo tools
|
||||||
|
cargo install sqlx-cli cargo-machete cargo-edit
|
||||||
|
```
|
||||||
|
|
||||||
### Scripts
|
### Scripts
|
||||||
- `scripts/check.sh`: Full CI check (builds, lints, tests both server and frontend)
|
- `scripts/check.sh`: Full CI check (builds, lints, tests both server and frontend)
|
||||||
|
- `scripts/check.sh --fix`: Same as above but auto-fixes linting and formatting issues
|
||||||
- `scripts/e2e.sh`: End-to-end testing
|
- `scripts/e2e.sh`: End-to-end testing
|
||||||
- `scripts/clean-up.sh`: Clean logs and database files
|
- `scripts/clean-up.sh`: Clean logs and database files
|
||||||
- `scripts/bump-version.sh patch`: Publish new version
|
- `scripts/bump-version.sh patch`: Publish new version
|
||||||
|
|
@ -59,10 +69,11 @@ cargo sqlx prepare --workspace
|
||||||
## Code Structure
|
## Code Structure
|
||||||
|
|
||||||
### Workspace Configuration
|
### Workspace Configuration
|
||||||
The frontend uses npm workspaces with three packages:
|
The frontend uses npm workspaces with four packages:
|
||||||
- `sync-client`: Core synchronization logic
|
- `sync-client`: Core synchronization logic
|
||||||
- `obsidian-plugin`: Obsidian-specific integration
|
- `obsidian-plugin`: Obsidian-specific integration
|
||||||
- `test-client`: Testing utilities
|
- `test-client`: Testing utilities
|
||||||
|
- `local-client-cli`: Standalone CLI for VaultLink sync client
|
||||||
|
|
||||||
### Type Generation
|
### Type Generation
|
||||||
Rust structs generate TypeScript types via ts-rs crate, stored in `sync-server/bindings/` and used by frontend packages.
|
Rust structs generate TypeScript types via ts-rs crate, stored in `sync-server/bindings/` and used by frontend packages.
|
||||||
|
|
|
||||||
4
docs/.gitignore
vendored
Normal file
4
docs/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,4 @@
|
||||||
|
node_modules/
|
||||||
|
.vitepress/dist/
|
||||||
|
.vitepress/cache/
|
||||||
|
package-lock.json
|
||||||
62
docs/.vitepress/config.mts
Normal file
62
docs/.vitepress/config.mts
Normal file
|
|
@ -0,0 +1,62 @@
|
||||||
|
import { defineConfig } from 'vitepress'
|
||||||
|
|
||||||
|
export default defineConfig({
|
||||||
|
title: 'VaultLink',
|
||||||
|
description: 'Self-hosted real-time synchronization for Obsidian',
|
||||||
|
base: '/vault-link/',
|
||||||
|
themeConfig: {
|
||||||
|
logo: '/logo.svg',
|
||||||
|
nav: [
|
||||||
|
{ text: 'Home', link: '/' },
|
||||||
|
{ text: 'Guide', link: '/guide/getting-started' },
|
||||||
|
{ text: 'Architecture', link: '/architecture/' },
|
||||||
|
{ text: 'GitHub', link: 'https://github.com/schmelczer/vault-link' }
|
||||||
|
],
|
||||||
|
sidebar: [
|
||||||
|
{
|
||||||
|
text: 'Introduction',
|
||||||
|
items: [
|
||||||
|
{ text: 'What is VaultLink?', link: '/guide/what-is-vaultlink' },
|
||||||
|
{ text: 'Getting Started', link: '/guide/getting-started' }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
text: 'Setup',
|
||||||
|
items: [
|
||||||
|
{ text: 'Server Setup', link: '/guide/server-setup' },
|
||||||
|
{ text: 'Obsidian Plugin', link: '/guide/obsidian-plugin' },
|
||||||
|
{ text: 'CLI Client', link: '/guide/cli-client' }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
text: 'Configuration',
|
||||||
|
items: [
|
||||||
|
{ text: 'Server Configuration', link: '/config/server' },
|
||||||
|
{ text: 'Authentication', link: '/config/authentication' },
|
||||||
|
{ text: 'Advanced Options', link: '/config/advanced' }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
text: 'Architecture',
|
||||||
|
items: [
|
||||||
|
{ text: 'Overview', link: '/architecture/' },
|
||||||
|
{ text: 'Sync Algorithm', link: '/architecture/sync-algorithm' },
|
||||||
|
{ text: 'Data Flow', link: '/architecture/data-flow' }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
socialLinks: [
|
||||||
|
{ icon: 'github', link: 'https://github.com/schmelczer/vault-link' }
|
||||||
|
],
|
||||||
|
footer: {
|
||||||
|
message: 'Released under the MIT License.',
|
||||||
|
copyright: 'Copyright © 2024-present Andras Schmelczer'
|
||||||
|
},
|
||||||
|
search: {
|
||||||
|
provider: 'local'
|
||||||
|
}
|
||||||
|
},
|
||||||
|
head: [
|
||||||
|
['link', { rel: 'icon', type: 'image/svg+xml', href: '/vault-link/logo.svg' }]
|
||||||
|
]
|
||||||
|
})
|
||||||
130
docs/README.md
Normal file
130
docs/README.md
Normal file
|
|
@ -0,0 +1,130 @@
|
||||||
|
# VaultLink Documentation
|
||||||
|
|
||||||
|
This directory contains the VaultLink documentation site built with [VitePress](https://vitepress.dev/).
|
||||||
|
|
||||||
|
## Development
|
||||||
|
|
||||||
|
### Prerequisites
|
||||||
|
|
||||||
|
- Node.js 18+
|
||||||
|
- npm
|
||||||
|
|
||||||
|
### Setup
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd docs
|
||||||
|
npm install
|
||||||
|
```
|
||||||
|
|
||||||
|
### Local Development
|
||||||
|
|
||||||
|
Start the development server with hot reload:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm run dev
|
||||||
|
```
|
||||||
|
|
||||||
|
The site will be available at `http://localhost:5173/vault-link/`
|
||||||
|
|
||||||
|
### Build
|
||||||
|
|
||||||
|
Build the static site:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm run build
|
||||||
|
```
|
||||||
|
|
||||||
|
Output will be in `.vitepress/dist/`
|
||||||
|
|
||||||
|
### Preview
|
||||||
|
|
||||||
|
Preview the built site:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm run preview
|
||||||
|
```
|
||||||
|
|
||||||
|
## Deployment
|
||||||
|
|
||||||
|
The documentation is automatically deployed to GitHub Pages when changes are pushed to the `main` branch.
|
||||||
|
|
||||||
|
The deployment workflow is configured in `.github/workflows/deploy-docs.yml`.
|
||||||
|
|
||||||
|
## Structure
|
||||||
|
|
||||||
|
```
|
||||||
|
docs/
|
||||||
|
├── .vitepress/
|
||||||
|
│ └── config.ts # VitePress configuration
|
||||||
|
├── public/ # Static assets
|
||||||
|
│ └── logo.svg # VaultLink logo
|
||||||
|
├── guide/ # User guides
|
||||||
|
│ ├── what-is-vaultlink.md
|
||||||
|
│ ├── getting-started.md
|
||||||
|
│ ├── server-setup.md
|
||||||
|
│ ├── obsidian-plugin.md
|
||||||
|
│ └── cli-client.md
|
||||||
|
├── architecture/ # Architecture documentation
|
||||||
|
│ ├── index.md
|
||||||
|
│ ├── sync-algorithm.md
|
||||||
|
│ └── data-flow.md
|
||||||
|
├── config/ # Configuration reference
|
||||||
|
│ ├── server.md
|
||||||
|
│ ├── authentication.md
|
||||||
|
│ └── advanced.md
|
||||||
|
└── index.md # Home page
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
## Writing Documentation
|
||||||
|
|
||||||
|
### Markdown Features
|
||||||
|
|
||||||
|
VitePress supports:
|
||||||
|
- GitHub Flavored Markdown
|
||||||
|
- Custom containers (tip, warning, danger)
|
||||||
|
- Code syntax highlighting
|
||||||
|
- Mermaid diagrams
|
||||||
|
- Emoji :rocket:
|
||||||
|
|
||||||
|
### Custom Containers
|
||||||
|
|
||||||
|
```markdown
|
||||||
|
::: tip
|
||||||
|
This is a tip
|
||||||
|
:::
|
||||||
|
|
||||||
|
::: warning
|
||||||
|
This is a warning
|
||||||
|
:::
|
||||||
|
|
||||||
|
::: danger
|
||||||
|
This is a danger message
|
||||||
|
:::
|
||||||
|
```
|
||||||
|
|
||||||
|
### Code Blocks
|
||||||
|
|
||||||
|
````markdown
|
||||||
|
```bash
|
||||||
|
npm install
|
||||||
|
```
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
server:
|
||||||
|
port: 3000
|
||||||
|
```
|
||||||
|
````
|
||||||
|
|
||||||
|
## Contributing
|
||||||
|
|
||||||
|
When adding new pages:
|
||||||
|
|
||||||
|
1. Create the markdown file in the appropriate directory
|
||||||
|
2. Add it to the sidebar in `.vitepress/config.ts`
|
||||||
|
3. Test locally with `npm run dev`
|
||||||
|
4. Submit a pull request
|
||||||
|
|
||||||
|
## License
|
||||||
|
|
||||||
|
MIT - Same as VaultLink
|
||||||
532
docs/architecture/data-flow.md
Normal file
532
docs/architecture/data-flow.md
Normal file
|
|
@ -0,0 +1,532 @@
|
||||||
|
# Data Flow
|
||||||
|
|
||||||
|
This document provides a detailed look at how data flows through the VaultLink system, from client to server and back.
|
||||||
|
|
||||||
|
## Connection Lifecycle
|
||||||
|
|
||||||
|
### 1. Initial Connection
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
sequenceDiagram
|
||||||
|
participant C as Client
|
||||||
|
participant S as Server
|
||||||
|
participant DB as Database
|
||||||
|
|
||||||
|
C->>S: WebSocket connect
|
||||||
|
S->>S: Accept connection
|
||||||
|
C->>S: Auth message (token + vault)
|
||||||
|
S->>S: Validate token
|
||||||
|
S->>S: Check vault access
|
||||||
|
S-->>C: Auth success
|
||||||
|
Note over C,S: Connection established
|
||||||
|
```
|
||||||
|
|
||||||
|
**Steps**:
|
||||||
|
1. Client initiates WebSocket connection to server
|
||||||
|
2. Server accepts connection
|
||||||
|
3. Client sends authentication message with token and vault name
|
||||||
|
4. Server validates token against `config.yml`
|
||||||
|
5. Server checks if user has access to requested vault
|
||||||
|
6. Server responds with success or error
|
||||||
|
7. Connection is ready for syncing
|
||||||
|
|
||||||
|
### 2. Initial Sync
|
||||||
|
|
||||||
|
After authentication, the client performs initial synchronization:
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
sequenceDiagram
|
||||||
|
participant C as Client
|
||||||
|
participant S as Server
|
||||||
|
participant DB as SQLite
|
||||||
|
|
||||||
|
C->>C: Scan local filesystem
|
||||||
|
C->>S: Request file list
|
||||||
|
S->>DB: Query all files
|
||||||
|
DB-->>S: File metadata
|
||||||
|
S-->>C: File list with versions
|
||||||
|
|
||||||
|
loop For each local file
|
||||||
|
C->>C: Check if file on server
|
||||||
|
alt File not on server
|
||||||
|
C->>S: Upload file
|
||||||
|
S->>DB: Store file + metadata
|
||||||
|
else File on server (different version)
|
||||||
|
C->>C: Compare versions
|
||||||
|
C->>S: Upload newer or merge
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
loop For each server file
|
||||||
|
C->>C: Check if file local
|
||||||
|
alt File not local
|
||||||
|
C->>S: Download file
|
||||||
|
S->>DB: Retrieve file
|
||||||
|
DB-->>S: File content
|
||||||
|
S-->>C: File content
|
||||||
|
C->>C: Write to disk
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
S-->>C: Sync complete message
|
||||||
|
```
|
||||||
|
|
||||||
|
**Process**:
|
||||||
|
1. Client scans local filesystem
|
||||||
|
2. Client requests file list from server
|
||||||
|
3. Server queries database and returns metadata
|
||||||
|
4. Client uploads missing or changed local files
|
||||||
|
5. Client downloads missing files from server
|
||||||
|
6. Server sends sync complete notification
|
||||||
|
|
||||||
|
### 3. Real-Time Synchronization
|
||||||
|
|
||||||
|
After initial sync, changes are pushed in real-time:
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
sequenceDiagram
|
||||||
|
participant FS as Filesystem
|
||||||
|
participant C1 as Client 1
|
||||||
|
participant S as Server
|
||||||
|
participant DB as Database
|
||||||
|
participant C2 as Client 2
|
||||||
|
|
||||||
|
FS->>C1: File changed (fs.watch)
|
||||||
|
C1->>C1: Read file content
|
||||||
|
C1->>S: Upload file
|
||||||
|
S->>DB: Store new version
|
||||||
|
S->>S: Apply OT if needed
|
||||||
|
S-->>C1: Upload ACK
|
||||||
|
S->>C2: File update notification
|
||||||
|
C2->>S: Download file
|
||||||
|
S->>DB: Retrieve file
|
||||||
|
DB-->>S: File content
|
||||||
|
S-->>C2: File content
|
||||||
|
C2->>FS: Write to disk
|
||||||
|
```
|
||||||
|
|
||||||
|
**Flow**:
|
||||||
|
1. Filesystem watcher detects local change
|
||||||
|
2. Client reads file content
|
||||||
|
3. Client uploads file via WebSocket
|
||||||
|
4. Server stores in database
|
||||||
|
5. Server applies operational transformation if concurrent edits
|
||||||
|
6. Server acknowledges upload to sender
|
||||||
|
7. Server broadcasts update to other clients
|
||||||
|
8. Other clients download and apply changes
|
||||||
|
|
||||||
|
## File Operations
|
||||||
|
|
||||||
|
### Upload
|
||||||
|
|
||||||
|
```
|
||||||
|
┌─────────┐
|
||||||
|
│ Client │
|
||||||
|
└────┬────┘
|
||||||
|
│ 1. Detect file change
|
||||||
|
│
|
||||||
|
├─► 2. Read file content
|
||||||
|
│
|
||||||
|
├─► 3. Create upload message
|
||||||
|
│ {
|
||||||
|
│ type: "upload_file",
|
||||||
|
│ path: "notes/daily.md",
|
||||||
|
│ content: "...",
|
||||||
|
│ version: 42,
|
||||||
|
│ timestamp: "2024-01-01T12:00:00Z"
|
||||||
|
│ }
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
┌─────────┐
|
||||||
|
│ Server │
|
||||||
|
└────┬────┘
|
||||||
|
│ 4. Validate message
|
||||||
|
│
|
||||||
|
├─► 5. Check permissions
|
||||||
|
│
|
||||||
|
├─► 6. Apply OT (if conflicts)
|
||||||
|
│
|
||||||
|
├─► 7. Store in database
|
||||||
|
│
|
||||||
|
├─► 8. Update version
|
||||||
|
│
|
||||||
|
├─► 9. Broadcast to clients
|
||||||
|
│
|
||||||
|
└─► 10. Send ACK to uploader
|
||||||
|
```
|
||||||
|
|
||||||
|
### Download
|
||||||
|
|
||||||
|
```
|
||||||
|
┌─────────┐
|
||||||
|
│ Server │
|
||||||
|
└────┬────┘
|
||||||
|
│ 1. File updated by another client
|
||||||
|
│
|
||||||
|
├─► 2. Broadcast notification
|
||||||
|
│ {
|
||||||
|
│ type: "file_updated",
|
||||||
|
│ path: "notes/daily.md",
|
||||||
|
│ version: 43
|
||||||
|
│ }
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
┌─────────┐
|
||||||
|
│ Client │
|
||||||
|
└────┬────┘
|
||||||
|
│ 3. Receive notification
|
||||||
|
│
|
||||||
|
├─► 4. Request file download
|
||||||
|
│ {
|
||||||
|
│ type: "download_file",
|
||||||
|
│ path: "notes/daily.md",
|
||||||
|
│ version: 43
|
||||||
|
│ }
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
┌─────────┐
|
||||||
|
│ Server │
|
||||||
|
└────┬────┘
|
||||||
|
│ 5. Retrieve from database
|
||||||
|
│
|
||||||
|
└─► 6. Send file content
|
||||||
|
{
|
||||||
|
type: "file_content",
|
||||||
|
path: "notes/daily.md",
|
||||||
|
content: "...",
|
||||||
|
version: 43
|
||||||
|
}
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
┌─────────┐
|
||||||
|
│ Client │
|
||||||
|
└────┬────┘
|
||||||
|
│ 7. Write to filesystem
|
||||||
|
│
|
||||||
|
└─► 8. Update local metadata
|
||||||
|
```
|
||||||
|
|
||||||
|
### Delete
|
||||||
|
|
||||||
|
```
|
||||||
|
┌─────────┐
|
||||||
|
│ Client │
|
||||||
|
└────┬────┘
|
||||||
|
│ 1. File deleted locally
|
||||||
|
│
|
||||||
|
├─► 2. Send delete message
|
||||||
|
│ {
|
||||||
|
│ type: "delete_file",
|
||||||
|
│ path: "notes/old.md"
|
||||||
|
│ }
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
┌─────────┐
|
||||||
|
│ Server │
|
||||||
|
└────┬────┘
|
||||||
|
│ 3. Mark as deleted in DB
|
||||||
|
│ (soft delete for history)
|
||||||
|
│
|
||||||
|
├─► 4. Broadcast deletion
|
||||||
|
│
|
||||||
|
└─► 5. ACK to sender
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
┌─────────┐
|
||||||
|
│ Other │
|
||||||
|
│ Clients │
|
||||||
|
└────┬────┘
|
||||||
|
│ 6. Delete local file
|
||||||
|
│
|
||||||
|
└─► 7. Update metadata
|
||||||
|
```
|
||||||
|
|
||||||
|
## Conflict Resolution Flow
|
||||||
|
|
||||||
|
### Concurrent Edits Scenario
|
||||||
|
|
||||||
|
```
|
||||||
|
Time →
|
||||||
|
|
||||||
|
Client A Server Client B
|
||||||
|
│ │ │
|
||||||
|
│ Edit file v10 │ │
|
||||||
|
│ "Add line A" │ │ Edit file v10
|
||||||
|
│ │ │ "Add line B"
|
||||||
|
│ │ │
|
||||||
|
├─── Upload @ t1 ─────────►│ │
|
||||||
|
│ │◄────── Upload @ t2 ────────┤
|
||||||
|
│ │ │
|
||||||
|
│ │ 1. Receive both edits │
|
||||||
|
│ │ (based on v10) │
|
||||||
|
│ │ │
|
||||||
|
│ │ 2. Apply first edit │
|
||||||
|
│ │ → v11 (line A added) │
|
||||||
|
│ │ │
|
||||||
|
│ │ 3. Transform second edit │
|
||||||
|
│ │ against first │
|
||||||
|
│ │ │
|
||||||
|
│ │ 4. Apply transformed edit │
|
||||||
|
│ │ → v12 (both lines) │
|
||||||
|
│ │ │
|
||||||
|
│◄──── v12 content ────────┤ │
|
||||||
|
│ ├───── v12 content ─────────►│
|
||||||
|
│ │ │
|
||||||
|
│ Apply v12 │ │ Apply v12
|
||||||
|
│ (has both lines) │ │ (has both lines)
|
||||||
|
│ │ │
|
||||||
|
```
|
||||||
|
|
||||||
|
### Conflict Resolution Steps
|
||||||
|
|
||||||
|
1. **Detection**: Server receives two edits based on the same version
|
||||||
|
2. **Ordering**: Determine which edit to apply first (by timestamp or client ID)
|
||||||
|
3. **First edit**: Apply directly to database
|
||||||
|
4. **Transformation**: Transform second edit against first using OT
|
||||||
|
5. **Second edit**: Apply transformed edit to database
|
||||||
|
6. **Broadcast**: Send merged result to all clients
|
||||||
|
7. **Application**: Clients apply merged version locally
|
||||||
|
|
||||||
|
## Database Schema
|
||||||
|
|
||||||
|
### Core Tables
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Document metadata
|
||||||
|
CREATE TABLE documents (
|
||||||
|
id INTEGER PRIMARY KEY,
|
||||||
|
path TEXT NOT NULL,
|
||||||
|
version INTEGER NOT NULL,
|
||||||
|
content_hash TEXT,
|
||||||
|
size INTEGER,
|
||||||
|
created_at TIMESTAMP,
|
||||||
|
updated_at TIMESTAMP,
|
||||||
|
deleted BOOLEAN DEFAULT FALSE
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Version history
|
||||||
|
CREATE TABLE versions (
|
||||||
|
id INTEGER PRIMARY KEY,
|
||||||
|
document_id INTEGER,
|
||||||
|
version INTEGER,
|
||||||
|
content BLOB,
|
||||||
|
created_at TIMESTAMP,
|
||||||
|
FOREIGN KEY (document_id) REFERENCES documents(id)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Client sync cursors
|
||||||
|
CREATE TABLE cursors (
|
||||||
|
client_id TEXT PRIMARY KEY,
|
||||||
|
last_version INTEGER,
|
||||||
|
last_updated TIMESTAMP
|
||||||
|
);
|
||||||
|
```
|
||||||
|
|
||||||
|
### Queries
|
||||||
|
|
||||||
|
**Get files since version**:
|
||||||
|
```sql
|
||||||
|
SELECT * FROM documents
|
||||||
|
WHERE version > ? AND deleted = FALSE
|
||||||
|
ORDER BY version ASC;
|
||||||
|
```
|
||||||
|
|
||||||
|
**Store new version**:
|
||||||
|
```sql
|
||||||
|
INSERT INTO versions (document_id, version, content, created_at)
|
||||||
|
VALUES (?, ?, ?, ?);
|
||||||
|
|
||||||
|
UPDATE documents
|
||||||
|
SET version = ?, updated_at = ?
|
||||||
|
WHERE id = ?;
|
||||||
|
```
|
||||||
|
|
||||||
|
**Update cursor**:
|
||||||
|
```sql
|
||||||
|
INSERT OR REPLACE INTO cursors (client_id, last_version, last_updated)
|
||||||
|
VALUES (?, ?, ?);
|
||||||
|
```
|
||||||
|
|
||||||
|
## Message Protocol
|
||||||
|
|
||||||
|
### Client → Server Messages
|
||||||
|
|
||||||
|
**Upload File**:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"type": "upload_file",
|
||||||
|
"path": "notes/example.md",
|
||||||
|
"content": "File content here...",
|
||||||
|
"base_version": 10,
|
||||||
|
"timestamp": "2024-01-01T12:00:00Z"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Download File**:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"type": "download_file",
|
||||||
|
"path": "notes/example.md"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Delete File**:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"type": "delete_file",
|
||||||
|
"path": "notes/old.md"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**List Files**:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"type": "list_files",
|
||||||
|
"since_version": 0
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Server → Client Messages
|
||||||
|
|
||||||
|
**File Updated**:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"type": "file_updated",
|
||||||
|
"path": "notes/example.md",
|
||||||
|
"version": 11,
|
||||||
|
"size": 1024,
|
||||||
|
"hash": "abc123..."
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**File Content**:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"type": "file_content",
|
||||||
|
"path": "notes/example.md",
|
||||||
|
"content": "Updated content...",
|
||||||
|
"version": 11
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**File Deleted**:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"type": "file_deleted",
|
||||||
|
"path": "notes/old.md",
|
||||||
|
"version": 12
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Sync Complete**:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"type": "sync_complete",
|
||||||
|
"total_files": 150,
|
||||||
|
"current_version": 200
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Error**:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"type": "error",
|
||||||
|
"message": "File too large",
|
||||||
|
"code": "FILE_TOO_LARGE"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Error Handling
|
||||||
|
|
||||||
|
### Client-Side Errors
|
||||||
|
|
||||||
|
**Network failure**:
|
||||||
|
1. Detect WebSocket disconnect
|
||||||
|
2. Queue pending operations
|
||||||
|
3. Retry connection with exponential backoff
|
||||||
|
4. Replay queued operations on reconnect
|
||||||
|
|
||||||
|
**File read error**:
|
||||||
|
1. Log error
|
||||||
|
2. Skip file
|
||||||
|
3. Continue with other files
|
||||||
|
4. Report to user
|
||||||
|
|
||||||
|
**Write conflict**:
|
||||||
|
1. Receive updated version from server
|
||||||
|
2. Apply OT merge locally
|
||||||
|
3. Overwrite local file
|
||||||
|
4. Continue syncing
|
||||||
|
|
||||||
|
### Server-Side Errors
|
||||||
|
|
||||||
|
**Database error**:
|
||||||
|
1. Log error
|
||||||
|
2. Return error to client
|
||||||
|
3. Client retries operation
|
||||||
|
|
||||||
|
**Invalid operation**:
|
||||||
|
1. Validate message format
|
||||||
|
2. Return specific error code
|
||||||
|
3. Client handles error appropriately
|
||||||
|
|
||||||
|
**Authentication failure**:
|
||||||
|
1. Reject connection
|
||||||
|
2. Send auth error
|
||||||
|
3. Client prompts for new credentials
|
||||||
|
|
||||||
|
## Performance Optimizations
|
||||||
|
|
||||||
|
### Batching
|
||||||
|
|
||||||
|
- Small, rapid changes are batched together
|
||||||
|
- Reduces message overhead
|
||||||
|
- Applied as single atomic update
|
||||||
|
|
||||||
|
### Compression
|
||||||
|
|
||||||
|
- Large files compressed before transmission
|
||||||
|
- Reduces bandwidth usage
|
||||||
|
- Transparent to application layer
|
||||||
|
|
||||||
|
### Incremental Sync
|
||||||
|
|
||||||
|
- Only changed portions of files sent
|
||||||
|
- Uses content-based diffing
|
||||||
|
- Significantly reduces data transfer
|
||||||
|
|
||||||
|
### Caching
|
||||||
|
|
||||||
|
- Server caches recent file versions
|
||||||
|
- Reduces database queries
|
||||||
|
- Improves response time
|
||||||
|
|
||||||
|
## Monitoring Data Flow
|
||||||
|
|
||||||
|
### Server Logs
|
||||||
|
|
||||||
|
```
|
||||||
|
2024-01-01 12:00:00 INFO WebSocket connection from 192.168.1.100
|
||||||
|
2024-01-01 12:00:01 INFO User 'alice' authenticated for vault 'personal'
|
||||||
|
2024-01-01 12:00:05 INFO Upload: notes/daily.md (v10 -> v11)
|
||||||
|
2024-01-01 12:00:06 INFO Broadcast to 3 clients
|
||||||
|
2024-01-01 12:00:10 INFO Conflict resolved: notes/shared.md (v12)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Client Logs
|
||||||
|
|
||||||
|
```
|
||||||
|
2024-01-01 12:00:00 INFO Connecting to ws://sync.example.com
|
||||||
|
2024-01-01 12:00:01 INFO Connected, authenticating...
|
||||||
|
2024-01-01 12:00:01 INFO Authentication successful
|
||||||
|
2024-01-01 12:00:02 INFO Starting initial sync
|
||||||
|
2024-01-01 12:00:10 INFO Sync complete: 150 files, 200 MB
|
||||||
|
2024-01-01 12:00:15 INFO Uploaded: notes/daily.md
|
||||||
|
2024-01-01 12:00:20 INFO Downloaded: notes/shared.md (merged)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Next Steps
|
||||||
|
|
||||||
|
- [Understand the sync algorithm →](/architecture/sync-algorithm)
|
||||||
|
- [Configure the server →](/config/server)
|
||||||
|
- [Deploy VaultLink →](/guide/getting-started)
|
||||||
344
docs/architecture/index.md
Normal file
344
docs/architecture/index.md
Normal file
|
|
@ -0,0 +1,344 @@
|
||||||
|
# Architecture Overview
|
||||||
|
|
||||||
|
VaultLink is built as a distributed system with a central sync server and multiple clients. This document explains the high-level architecture and design decisions.
|
||||||
|
|
||||||
|
## System Components
|
||||||
|
|
||||||
|
```
|
||||||
|
┌─────────────────────────────────────────────────────────────┐
|
||||||
|
│ Clients │
|
||||||
|
├─────────────────────┬───────────────────┬───────────────────┤
|
||||||
|
│ Obsidian Plugin │ Obsidian Plugin │ CLI Client │
|
||||||
|
│ (User A - Device1) │ (User A - Device2│ (Server/Backup) │
|
||||||
|
└──────────┬──────────┴─────────┬─────────┴──────────┬────────┘
|
||||||
|
│ │ │
|
||||||
|
│ WebSocket │ WebSocket │ WebSocket
|
||||||
|
│ │ │
|
||||||
|
└────────────────────┼────────────────────┘
|
||||||
|
│
|
||||||
|
┌───────────▼───────────┐
|
||||||
|
│ Sync Server │
|
||||||
|
│ (Rust + Axum) │
|
||||||
|
│ │
|
||||||
|
│ ┌─────────────────┐ │
|
||||||
|
│ │ WebSocket Hub │ │
|
||||||
|
│ └────────┬────────┘ │
|
||||||
|
│ │ │
|
||||||
|
│ ┌────────▼────────┐ │
|
||||||
|
│ │ Sync Engine │ │
|
||||||
|
│ │ (OT Algorithm) │ │
|
||||||
|
│ └────────┬────────┘ │
|
||||||
|
│ │ │
|
||||||
|
│ ┌────────▼────────┐ │
|
||||||
|
│ │ SQLite Database │ │
|
||||||
|
│ │ (Per Vault) │ │
|
||||||
|
│ └─────────────────┘ │
|
||||||
|
└───────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
## Core Components
|
||||||
|
|
||||||
|
### Sync Server
|
||||||
|
|
||||||
|
The central authority for synchronization, written in Rust using Axum framework.
|
||||||
|
|
||||||
|
**Responsibilities**:
|
||||||
|
- Accept WebSocket connections from clients
|
||||||
|
- Authenticate users via token-based auth
|
||||||
|
- Store document versions in SQLite
|
||||||
|
- Coordinate real-time updates between clients
|
||||||
|
- Apply operational transformation for conflict resolution
|
||||||
|
- Manage vault access control
|
||||||
|
|
||||||
|
**Technology**:
|
||||||
|
- **Language**: Rust 1.89+
|
||||||
|
- **Framework**: Axum (async web framework)
|
||||||
|
- **Database**: SQLite with SQLx
|
||||||
|
- **Protocol**: WebSockets for real-time communication
|
||||||
|
- **Sync Algorithm**: reconcile-text (operational transformation)
|
||||||
|
|
||||||
|
### Sync Client Library
|
||||||
|
|
||||||
|
TypeScript library providing core synchronization logic, used by both the Obsidian plugin and CLI client.
|
||||||
|
|
||||||
|
**Responsibilities**:
|
||||||
|
- Manage WebSocket connection to server
|
||||||
|
- Watch local filesystem for changes
|
||||||
|
- Upload and download files
|
||||||
|
- Apply remote changes locally
|
||||||
|
- Handle conflict resolution
|
||||||
|
- Maintain sync metadata
|
||||||
|
|
||||||
|
**Technology**:
|
||||||
|
- **Language**: TypeScript
|
||||||
|
- **Build**: Webpack
|
||||||
|
- **Protocol**: WebSocket client
|
||||||
|
- **File System**: Node.js `fs` API / Obsidian API
|
||||||
|
|
||||||
|
### Obsidian Plugin
|
||||||
|
|
||||||
|
Integration layer between sync client and Obsidian.
|
||||||
|
|
||||||
|
**Responsibilities**:
|
||||||
|
- Provide UI for configuration
|
||||||
|
- Bridge sync client with Obsidian's file system API
|
||||||
|
- Handle Obsidian lifecycle events
|
||||||
|
- Display sync status to users
|
||||||
|
|
||||||
|
**Technology**:
|
||||||
|
- **Platform**: Obsidian Plugin API
|
||||||
|
- **Core**: sync-client library
|
||||||
|
- **UI**: Obsidian settings UI
|
||||||
|
|
||||||
|
### CLI Client
|
||||||
|
|
||||||
|
Standalone executable for syncing vaults without Obsidian.
|
||||||
|
|
||||||
|
**Responsibilities**:
|
||||||
|
- Command-line interface
|
||||||
|
- File system access via Node.js
|
||||||
|
- Daemon mode for continuous sync
|
||||||
|
- Health check endpoint for monitoring
|
||||||
|
|
||||||
|
**Technology**:
|
||||||
|
- **Language**: TypeScript
|
||||||
|
- **Runtime**: Node.js
|
||||||
|
- **CLI**: Commander.js
|
||||||
|
- **Core**: sync-client library
|
||||||
|
|
||||||
|
## Data Flow
|
||||||
|
|
||||||
|
### Initial Connection
|
||||||
|
|
||||||
|
1. Client connects via WebSocket to server
|
||||||
|
2. Server authenticates using provided token
|
||||||
|
3. Server verifies user has access to requested vault
|
||||||
|
4. Connection established, sync begins
|
||||||
|
|
||||||
|
### File Upload Flow
|
||||||
|
|
||||||
|
```
|
||||||
|
Client Server
|
||||||
|
│ │
|
||||||
|
│ 1. File changed locally │
|
||||||
|
│ │
|
||||||
|
│ 2. Read file content │
|
||||||
|
│ │
|
||||||
|
│ 3. WebSocket: Upload file │
|
||||||
|
├──────────────────────────────►│
|
||||||
|
│ │ 4. Store in SQLite
|
||||||
|
│ │
|
||||||
|
│ │ 5. Broadcast to other clients
|
||||||
|
│ ├───────────────────────►
|
||||||
|
│ 6. Ack upload │
|
||||||
|
│◄──────────────────────────────┤
|
||||||
|
```
|
||||||
|
|
||||||
|
### File Download Flow
|
||||||
|
|
||||||
|
```
|
||||||
|
Client A Server Client B
|
||||||
|
│ │ │
|
||||||
|
│ │ 1. File uploaded │
|
||||||
|
│ │◄────────────────────────┤
|
||||||
|
│ │ │
|
||||||
|
│ │ 2. Store in DB │
|
||||||
|
│ │ │
|
||||||
|
│ 3. Push notification │ │
|
||||||
|
│◄────────────────────────┤ │
|
||||||
|
│ │ │
|
||||||
|
│ 4. Download file │ │
|
||||||
|
├────────────────────────►│ │
|
||||||
|
│ │ │
|
||||||
|
│ 5. Write locally │ │
|
||||||
|
│ │ │
|
||||||
|
```
|
||||||
|
|
||||||
|
### Conflict Resolution
|
||||||
|
|
||||||
|
When two clients edit the same file simultaneously:
|
||||||
|
|
||||||
|
```
|
||||||
|
Client A Server Client B
|
||||||
|
│ │ │
|
||||||
|
│ 1. Edit file │ │ 1. Edit same file
|
||||||
|
│ │ │
|
||||||
|
│ 2. Upload changes │ │ 2. Upload changes
|
||||||
|
├────────────────────────►│◄────────────────────────┤
|
||||||
|
│ │ │
|
||||||
|
│ │ 3. Apply OT algorithm │
|
||||||
|
│ │ - Merge both edits │
|
||||||
|
│ │ - Preserve all changes│
|
||||||
|
│ │ │
|
||||||
|
│ 4. Receive merged ver. │ 5. Receive merged ver. │
|
||||||
|
│◄────────────────────────┤────────────────────────►│
|
||||||
|
│ │ │
|
||||||
|
│ 6. Apply locally │ │ 6. Apply locally
|
||||||
|
```
|
||||||
|
|
||||||
|
## Storage Architecture
|
||||||
|
|
||||||
|
### Server Storage
|
||||||
|
|
||||||
|
Each vault has its own SQLite database:
|
||||||
|
|
||||||
|
```
|
||||||
|
databases/
|
||||||
|
├── vault-1.db
|
||||||
|
├── vault-2.db
|
||||||
|
└── shared-team.db
|
||||||
|
```
|
||||||
|
|
||||||
|
**Database Schema** (simplified):
|
||||||
|
- **documents**: File metadata (path, size, modified time)
|
||||||
|
- **versions**: Document content with version history
|
||||||
|
- **cursors**: Client sync state
|
||||||
|
|
||||||
|
### Client Storage
|
||||||
|
|
||||||
|
Clients maintain sync metadata:
|
||||||
|
|
||||||
|
```
|
||||||
|
.vaultlink/
|
||||||
|
├── metadata.json # Sync state
|
||||||
|
└── cache/ # Optional local cache
|
||||||
|
```
|
||||||
|
|
||||||
|
The `.vaultlink` directory tracks which files have been synced and their versions to enable efficient synchronization.
|
||||||
|
|
||||||
|
## Communication Protocol
|
||||||
|
|
||||||
|
### WebSocket Messages
|
||||||
|
|
||||||
|
Client-server communication uses JSON messages over WebSocket.
|
||||||
|
|
||||||
|
**Message Types**:
|
||||||
|
- `upload_file`: Client → Server (file upload)
|
||||||
|
- `download_file`: Client → Server (request file)
|
||||||
|
- `file_updated`: Server → Client (file changed notification)
|
||||||
|
- `file_deleted`: Server → Client (file deleted notification)
|
||||||
|
- `sync_complete`: Server → Client (initial sync finished)
|
||||||
|
|
||||||
|
### Authentication
|
||||||
|
|
||||||
|
Token-based authentication on connection:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
// Client sends token on connect
|
||||||
|
{
|
||||||
|
type: "auth",
|
||||||
|
token: "user-auth-token",
|
||||||
|
vault: "vault-name"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Server responds
|
||||||
|
{
|
||||||
|
type: "auth_success"
|
||||||
|
}
|
||||||
|
// or
|
||||||
|
{
|
||||||
|
type: "auth_error",
|
||||||
|
message: "Invalid token"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Scalability Considerations
|
||||||
|
|
||||||
|
### Current Architecture
|
||||||
|
|
||||||
|
- **SQLite per vault**: Simple, performant, limited to single server
|
||||||
|
- **WebSocket connections**: Stateful, requires sticky sessions for load balancing
|
||||||
|
- **Operational transformation**: Centralized on server
|
||||||
|
|
||||||
|
### Scaling Approaches
|
||||||
|
|
||||||
|
**Vertical Scaling**:
|
||||||
|
- Increase server resources (CPU, RAM, storage)
|
||||||
|
- Optimize database queries and indexing
|
||||||
|
- Tune connection limits
|
||||||
|
|
||||||
|
**Horizontal Scaling** (future):
|
||||||
|
- Separate vault servers (vault sharding)
|
||||||
|
- Load balancer with sticky sessions
|
||||||
|
- Shared storage layer for SQLite databases
|
||||||
|
- Consider alternative databases (PostgreSQL) for multi-server setups
|
||||||
|
|
||||||
|
### Performance Characteristics
|
||||||
|
|
||||||
|
- **Small vaults** (< 1000 files): Excellent performance
|
||||||
|
- **Medium vaults** (1000-10000 files): Good performance with tuning
|
||||||
|
- **Large vaults** (> 10000 files): May require optimization
|
||||||
|
- **Concurrent users**: Tested with dozens of simultaneous clients per vault
|
||||||
|
|
||||||
|
## Security Model
|
||||||
|
|
||||||
|
### Authentication
|
||||||
|
|
||||||
|
- Token-based authentication
|
||||||
|
- Tokens configured in server `config.yml`
|
||||||
|
- No password hashing (tokens are secrets)
|
||||||
|
|
||||||
|
### Authorization
|
||||||
|
|
||||||
|
- Per-user vault access control
|
||||||
|
- Allow-list or deny-list patterns
|
||||||
|
- Global access or vault-specific access
|
||||||
|
|
||||||
|
### Network Security
|
||||||
|
|
||||||
|
- WebSocket over TLS (WSS) for encrypted transport
|
||||||
|
- No built-in SSL (use reverse proxy)
|
||||||
|
- CORS configured for web clients
|
||||||
|
|
||||||
|
### Data Security
|
||||||
|
|
||||||
|
- No encryption at rest (use encrypted filesystems if needed)
|
||||||
|
- No end-to-end encryption (server sees all content)
|
||||||
|
- Self-hosted model: you control the data
|
||||||
|
|
||||||
|
## Technology Choices
|
||||||
|
|
||||||
|
### Why Rust for Server?
|
||||||
|
|
||||||
|
- **Performance**: Low latency for real-time sync
|
||||||
|
- **Memory safety**: No crashes from memory bugs
|
||||||
|
- **Concurrency**: Excellent async support with Tokio
|
||||||
|
- **Type safety**: Catch bugs at compile time
|
||||||
|
- **SQLx**: Compile-time SQL verification
|
||||||
|
|
||||||
|
### Why SQLite?
|
||||||
|
|
||||||
|
- **Simplicity**: No separate database server required
|
||||||
|
- **Performance**: Fast for read-heavy workloads
|
||||||
|
- **Reliability**: Battle-tested, ACID compliant
|
||||||
|
- **Portability**: Single file per vault
|
||||||
|
- **Backups**: Simple file copy
|
||||||
|
|
||||||
|
### Why WebSocket?
|
||||||
|
|
||||||
|
- **Real-time**: Bidirectional push for instant updates
|
||||||
|
- **Efficiency**: Persistent connection, no polling overhead
|
||||||
|
- **Simplicity**: Built-in browser/Node.js support
|
||||||
|
- **Standards**: Well-supported protocol
|
||||||
|
|
||||||
|
### Why Operational Transformation?
|
||||||
|
|
||||||
|
- **Automatic conflict resolution**: No manual merging required
|
||||||
|
- **Preserves intent**: All edits are kept
|
||||||
|
- **Real-time collaboration**: Users see changes as they happen
|
||||||
|
- **Proven algorithm**: Used by Google Docs, etc.
|
||||||
|
|
||||||
|
## Design Principles
|
||||||
|
|
||||||
|
1. **Self-hosted first**: Users control their data and infrastructure
|
||||||
|
2. **Simplicity**: Easy to deploy and operate
|
||||||
|
3. **Real-time**: Changes appear immediately
|
||||||
|
4. **Reliability**: Handle network failures gracefully
|
||||||
|
5. **Performance**: Fast sync for typical vault sizes
|
||||||
|
6. **Privacy**: No third-party services or telemetry
|
||||||
|
|
||||||
|
## Next Steps
|
||||||
|
|
||||||
|
- [Learn about the sync algorithm →](/architecture/sync-algorithm)
|
||||||
|
- [Understand data flow in detail →](/architecture/data-flow)
|
||||||
|
- [Deploy the server →](/guide/server-setup)
|
||||||
361
docs/architecture/sync-algorithm.md
Normal file
361
docs/architecture/sync-algorithm.md
Normal file
|
|
@ -0,0 +1,361 @@
|
||||||
|
# Sync Algorithm
|
||||||
|
|
||||||
|
VaultLink uses operational transformation (OT) to handle concurrent edits and maintain consistency across clients. This document explains how the algorithm works.
|
||||||
|
|
||||||
|
## Operational Transformation
|
||||||
|
|
||||||
|
Operational transformation is a technique for managing concurrent edits to the same document. It transforms operations (edits) so they can be applied in different orders while preserving user intent.
|
||||||
|
|
||||||
|
### Why OT?
|
||||||
|
|
||||||
|
Traditional conflict resolution approaches:
|
||||||
|
- **Last write wins**: Loses data, frustrating for users
|
||||||
|
- **Manual merging**: Interrupts workflow, requires user intervention
|
||||||
|
- **Version branching**: Complex, not suitable for real-time sync
|
||||||
|
|
||||||
|
Operational transformation:
|
||||||
|
- **Automatic**: No user intervention required
|
||||||
|
- **Preserves all edits**: No data loss
|
||||||
|
- **Real-time**: Changes appear immediately
|
||||||
|
- **Intuitive**: Behavior matches user expectations
|
||||||
|
|
||||||
|
## The reconcile-text Library
|
||||||
|
|
||||||
|
VaultLink uses the [`reconcile-text`](https://crates.io/crates/reconcile-text) Rust library for operational transformation on text documents.
|
||||||
|
|
||||||
|
### How It Works
|
||||||
|
|
||||||
|
Given a base document and two sets of changes, OT produces a merged result that includes both changes.
|
||||||
|
|
||||||
|
**Example**:
|
||||||
|
|
||||||
|
```
|
||||||
|
Base document: "Hello world"
|
||||||
|
|
||||||
|
User A: "Hello beautiful world" (inserts "beautiful ")
|
||||||
|
User B: "Hello world!" (inserts "!")
|
||||||
|
|
||||||
|
OT result: "Hello beautiful world!" (both changes applied)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Operation Types
|
||||||
|
|
||||||
|
The algorithm handles these operations:
|
||||||
|
- **Insert**: Add text at position
|
||||||
|
- **Delete**: Remove text from position
|
||||||
|
- **Retain**: Keep existing text unchanged
|
||||||
|
|
||||||
|
### Transformation Process
|
||||||
|
|
||||||
|
1. **Client A** makes edit and sends to server
|
||||||
|
2. **Client B** makes concurrent edit and sends to server
|
||||||
|
3. **Server** receives both edits
|
||||||
|
4. **Server** transforms operations to account for concurrent changes
|
||||||
|
5. **Server** applies merged result to database
|
||||||
|
6. **Server** sends transformed operations to both clients
|
||||||
|
7. **Clients** apply transformed operations locally
|
||||||
|
|
||||||
|
## Sync State Management
|
||||||
|
|
||||||
|
VaultLink maintains sync state to track which changes have been applied.
|
||||||
|
|
||||||
|
### Version Vectors
|
||||||
|
|
||||||
|
Each document has a version tracked by:
|
||||||
|
- **Server version**: Incremented on each change
|
||||||
|
- **Client cursors**: Track which version each client has seen
|
||||||
|
|
||||||
|
This enables:
|
||||||
|
- Efficient syncing (only send changes since last sync)
|
||||||
|
- Conflict detection (concurrent edits to same version)
|
||||||
|
- Ordering of operations
|
||||||
|
|
||||||
|
### Cursor Management
|
||||||
|
|
||||||
|
Clients maintain a cursor position:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
struct Cursor {
|
||||||
|
vault_id: String,
|
||||||
|
client_id: String,
|
||||||
|
last_version: u64,
|
||||||
|
last_updated: DateTime,
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
On sync:
|
||||||
|
1. Client sends cursor (last seen version)
|
||||||
|
2. Server returns all changes since that version
|
||||||
|
3. Client applies changes and updates cursor
|
||||||
|
|
||||||
|
## Conflict Resolution Flow
|
||||||
|
|
||||||
|
### Scenario: Concurrent Edits
|
||||||
|
|
||||||
|
Two users edit the same paragraph simultaneously.
|
||||||
|
|
||||||
|
**Initial state**:
|
||||||
|
```
|
||||||
|
Version 10: "The quick brown fox jumps over the lazy dog."
|
||||||
|
```
|
||||||
|
|
||||||
|
**User A's edit** (version 11):
|
||||||
|
```
|
||||||
|
"The quick brown fox jumps over the very lazy dog."
|
||||||
|
```
|
||||||
|
*Inserts "very " at position 40*
|
||||||
|
|
||||||
|
**User B's edit** (also from version 10):
|
||||||
|
```
|
||||||
|
"The quick red fox jumps over the lazy dog."
|
||||||
|
```
|
||||||
|
*Replaces "brown" with "red" at position 10*
|
||||||
|
|
||||||
|
### Server Processing
|
||||||
|
|
||||||
|
1. **Receive User A's operation**:
|
||||||
|
- Base: version 10
|
||||||
|
- Operation: Insert("very ", position=40)
|
||||||
|
- Apply to database → version 11
|
||||||
|
|
||||||
|
2. **Receive User B's operation**:
|
||||||
|
- Base: version 10
|
||||||
|
- Operation: Replace("brown"→"red", position=10)
|
||||||
|
- **Conflict detected**: Base is version 10, but current is version 11
|
||||||
|
|
||||||
|
3. **Transform User B's operation**:
|
||||||
|
- Transform against User A's operation
|
||||||
|
- Adjust positions/content as needed
|
||||||
|
- Apply transformed operation → version 12
|
||||||
|
|
||||||
|
4. **Broadcast updates**:
|
||||||
|
- Send User A's operation to User B
|
||||||
|
- Send transformed User B's operation to User A
|
||||||
|
|
||||||
|
### Final Result
|
||||||
|
|
||||||
|
```
|
||||||
|
Version 12: "The quick red fox jumps over the very lazy dog."
|
||||||
|
```
|
||||||
|
|
||||||
|
Both edits are preserved in the final document.
|
||||||
|
|
||||||
|
## Edge Cases
|
||||||
|
|
||||||
|
### 1. Delete vs Insert Conflict
|
||||||
|
|
||||||
|
**Scenario**: User A deletes a paragraph while User B edits it.
|
||||||
|
|
||||||
|
**Resolution**:
|
||||||
|
- OT algorithm prioritizes preservation of content
|
||||||
|
- Insert operation is transformed to account for deletion
|
||||||
|
- Typically results in inserted content appearing nearby
|
||||||
|
|
||||||
|
**Example**:
|
||||||
|
```
|
||||||
|
Base: "Line 1\nLine 2\nLine 3"
|
||||||
|
|
||||||
|
User A: Delete Line 2 → "Line 1\nLine 3"
|
||||||
|
User B: Edit Line 2 → "Line 1\nLine 2 modified\nLine 3"
|
||||||
|
|
||||||
|
Result: "Line 1\nLine 2 modified\nLine 3"
|
||||||
|
```
|
||||||
|
(Insert takes precedence, preserving user content)
|
||||||
|
|
||||||
|
### 2. Overlapping Edits
|
||||||
|
|
||||||
|
**Scenario**: Two users edit overlapping regions.
|
||||||
|
|
||||||
|
**Resolution**:
|
||||||
|
- OT splits operations into non-overlapping segments
|
||||||
|
- Applies each segment independently
|
||||||
|
- Merges results
|
||||||
|
|
||||||
|
### 3. Delete vs Delete
|
||||||
|
|
||||||
|
**Scenario**: Two users delete overlapping text.
|
||||||
|
|
||||||
|
**Resolution**:
|
||||||
|
- Deletes are merged
|
||||||
|
- Final result has the union of deleted ranges removed
|
||||||
|
|
||||||
|
### 4. Network Partitions
|
||||||
|
|
||||||
|
**Scenario**: Client loses connection, makes edits offline, reconnects.
|
||||||
|
|
||||||
|
**Resolution**:
|
||||||
|
1. Client queues edits locally
|
||||||
|
2. On reconnect, sends all queued operations
|
||||||
|
3. Server applies OT against all operations that happened during partition
|
||||||
|
4. Client receives transformed operations and applies
|
||||||
|
|
||||||
|
## Performance Characteristics
|
||||||
|
|
||||||
|
### Time Complexity
|
||||||
|
|
||||||
|
- **Single operation**: O(1) for most operations
|
||||||
|
- **Transformation**: O(n) where n is operation size
|
||||||
|
- **Conflict resolution**: O(m × n) where m is number of concurrent operations
|
||||||
|
|
||||||
|
### Space Complexity
|
||||||
|
|
||||||
|
- **Version history**: Grows with number of changes
|
||||||
|
- **Cursors**: O(clients × vaults)
|
||||||
|
- **Active operations**: Minimal (processed in real-time)
|
||||||
|
|
||||||
|
### Optimization
|
||||||
|
|
||||||
|
VaultLink optimizes for:
|
||||||
|
- Small, frequent edits (typical typing patterns)
|
||||||
|
- Text documents (not binary files)
|
||||||
|
- Real-time processing (no batching delay)
|
||||||
|
|
||||||
|
## Limitations
|
||||||
|
|
||||||
|
### Binary Files
|
||||||
|
|
||||||
|
OT works best for text files. Binary files:
|
||||||
|
- Cannot be meaningfully merged
|
||||||
|
- Use last-write-wins strategy
|
||||||
|
- May cause data loss on concurrent edits
|
||||||
|
|
||||||
|
**Workaround**: Avoid concurrent edits to binary files, or use versioning.
|
||||||
|
|
||||||
|
### Large Documents
|
||||||
|
|
||||||
|
Very large documents (> 1MB) may have:
|
||||||
|
- Higher transformation costs
|
||||||
|
- Slower sync times
|
||||||
|
- Increased memory usage
|
||||||
|
|
||||||
|
**Workaround**: Split large documents or increase timeout settings.
|
||||||
|
|
||||||
|
### Complex Formatting
|
||||||
|
|
||||||
|
Markdown with complex structures may occasionally produce unexpected results:
|
||||||
|
- Nested lists
|
||||||
|
- Tables
|
||||||
|
- Code blocks
|
||||||
|
|
||||||
|
**Workaround**: Manual cleanup if needed, or minimize concurrent edits to complex structures.
|
||||||
|
|
||||||
|
## Consistency Guarantees
|
||||||
|
|
||||||
|
### Strong Consistency
|
||||||
|
|
||||||
|
VaultLink provides **strong eventual consistency**:
|
||||||
|
- All clients eventually converge to the same state
|
||||||
|
- Operations applied in causal order
|
||||||
|
- No data loss under normal operation
|
||||||
|
|
||||||
|
### Ordering Guarantees
|
||||||
|
|
||||||
|
- Operations from the same client are applied in order
|
||||||
|
- Concurrent operations may be applied in any order
|
||||||
|
- Final result is independent of operation order (commutative)
|
||||||
|
|
||||||
|
### Durability
|
||||||
|
|
||||||
|
- Operations are written to SQLite before acknowledgment
|
||||||
|
- SQLite ACID guarantees protect against data loss
|
||||||
|
- Clients retry failed uploads
|
||||||
|
|
||||||
|
## Comparison with Other Approaches
|
||||||
|
|
||||||
|
### Git-style Merging
|
||||||
|
|
||||||
|
| Aspect | Git Merge | VaultLink OT |
|
||||||
|
|--------|-----------|--------------|
|
||||||
|
| Real-time | No | Yes |
|
||||||
|
| Manual conflict resolution | Yes | No |
|
||||||
|
| Branching | Yes | No |
|
||||||
|
| Automatic merge | Limited | Always |
|
||||||
|
| Use case | Code changes | Collaborative documents |
|
||||||
|
|
||||||
|
### CRDTs (Conflict-free Replicated Data Types)
|
||||||
|
|
||||||
|
| Aspect | CRDTs | VaultLink OT |
|
||||||
|
|--------|-------|--------------|
|
||||||
|
| Server required | No | Yes |
|
||||||
|
| Memory overhead | Higher | Lower |
|
||||||
|
| Complexity | Higher | Lower |
|
||||||
|
| Deletion handling | Complex (tombstones) | Simple |
|
||||||
|
| Best for | Distributed systems | Centralized sync |
|
||||||
|
|
||||||
|
### Last Write Wins
|
||||||
|
|
||||||
|
| Aspect | LWW | VaultLink OT |
|
||||||
|
|--------|-----|--------------|
|
||||||
|
| Data loss | Yes | No |
|
||||||
|
| Simplicity | High | Medium |
|
||||||
|
| User experience | Poor | Excellent |
|
||||||
|
| Performance | Best | Good |
|
||||||
|
|
||||||
|
## Algorithm Details
|
||||||
|
|
||||||
|
### Transformation Rules
|
||||||
|
|
||||||
|
When transforming operation `A` against operation `B`:
|
||||||
|
|
||||||
|
1. **Insert vs Insert**:
|
||||||
|
- If positions equal: Order by client ID
|
||||||
|
- If different positions: Adjust positions
|
||||||
|
|
||||||
|
2. **Insert vs Delete**:
|
||||||
|
- If insert in deleted range: Shift insert position
|
||||||
|
- If insert after delete: Adjust position by deleted length
|
||||||
|
|
||||||
|
3. **Delete vs Delete**:
|
||||||
|
- If ranges overlap: Merge delete ranges
|
||||||
|
- If ranges disjoint: Adjust positions
|
||||||
|
|
||||||
|
4. **Retain vs Any**:
|
||||||
|
- Retain operations don't conflict
|
||||||
|
- Simply adjust positions
|
||||||
|
|
||||||
|
### Transformation Example
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// Pseudo-code for transformation
|
||||||
|
fn transform(op_a: Operation, op_b: Operation) -> (Operation, Operation) {
|
||||||
|
match (op_a, op_b) {
|
||||||
|
(Insert(pos_a, text_a), Insert(pos_b, text_b)) => {
|
||||||
|
if pos_a < pos_b {
|
||||||
|
(op_a, Insert(pos_b + text_a.len(), text_b))
|
||||||
|
} else if pos_a > pos_b {
|
||||||
|
(Insert(pos_a + text_b.len(), text_a), op_b)
|
||||||
|
} else {
|
||||||
|
// Same position, use client ID to break tie
|
||||||
|
if client_id_a < client_id_b {
|
||||||
|
(op_a, Insert(pos_b + text_a.len(), text_b))
|
||||||
|
} else {
|
||||||
|
(Insert(pos_a + text_b.len(), text_a), op_b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// ... other cases
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Best Practices
|
||||||
|
|
||||||
|
### For Smooth Collaboration
|
||||||
|
|
||||||
|
1. **Small edits**: Make small, focused changes for easier merging
|
||||||
|
2. **Coordinate major changes**: Discuss large refactors with team
|
||||||
|
3. **Monitor sync status**: Ensure changes are uploaded before signing off
|
||||||
|
4. **Test conflict resolution**: Verify behavior matches expectations
|
||||||
|
|
||||||
|
### For Developers
|
||||||
|
|
||||||
|
1. **Text files preferred**: OT works best on text
|
||||||
|
2. **Limit file sizes**: Keep documents reasonably sized
|
||||||
|
3. **Binary files**: Use versioning or avoid concurrent edits
|
||||||
|
4. **Testing**: Test concurrent edit scenarios thoroughly
|
||||||
|
|
||||||
|
## Further Reading
|
||||||
|
|
||||||
|
- [reconcile-text library](https://crates.io/crates/reconcile-text)
|
||||||
|
- [Operational Transformation FAQ](https://en.wikipedia.org/wiki/Operational_transformation)
|
||||||
|
- [Data flow architecture →](/architecture/data-flow)
|
||||||
581
docs/config/advanced.md
Normal file
581
docs/config/advanced.md
Normal file
|
|
@ -0,0 +1,581 @@
|
||||||
|
# Advanced Configuration
|
||||||
|
|
||||||
|
Advanced topics for optimizing and customizing your VaultLink deployment.
|
||||||
|
|
||||||
|
## Database Optimization
|
||||||
|
|
||||||
|
### SQLite Tuning
|
||||||
|
|
||||||
|
While VaultLink handles most SQLite configuration automatically, you can optimize for specific workloads.
|
||||||
|
|
||||||
|
#### WAL Mode
|
||||||
|
|
||||||
|
VaultLink uses Write-Ahead Logging (WAL) mode by default for better concurrency.
|
||||||
|
|
||||||
|
**Benefits**:
|
||||||
|
- Readers don't block writers
|
||||||
|
- Writers don't block readers
|
||||||
|
- Better performance for concurrent access
|
||||||
|
|
||||||
|
**Maintenance**:
|
||||||
|
```bash
|
||||||
|
# Checkpoint WAL to main database (run periodically)
|
||||||
|
sqlite3 databases/vault.db "PRAGMA wal_checkpoint(TRUNCATE);"
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Database Size Management
|
||||||
|
|
||||||
|
Over time, databases can grow with version history:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Check database size
|
||||||
|
du -h databases/*.db
|
||||||
|
|
||||||
|
# Vacuum to reclaim space (offline only)
|
||||||
|
sqlite3 databases/vault.db "VACUUM;"
|
||||||
|
|
||||||
|
# Analyze for query optimization
|
||||||
|
sqlite3 databases/vault.db "ANALYZE;"
|
||||||
|
```
|
||||||
|
|
||||||
|
**Schedule maintenance**:
|
||||||
|
```bash
|
||||||
|
#!/bin/bash
|
||||||
|
# monthly-maintenance.sh
|
||||||
|
|
||||||
|
for db in databases/*.db; do
|
||||||
|
echo "Optimizing $db"
|
||||||
|
sqlite3 "$db" "PRAGMA optimize;"
|
||||||
|
sqlite3 "$db" "PRAGMA wal_checkpoint(TRUNCATE);"
|
||||||
|
done
|
||||||
|
```
|
||||||
|
|
||||||
|
### Version History Cleanup
|
||||||
|
|
||||||
|
To limit database growth, implement version history pruning (requires custom script):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
#!/bin/bash
|
||||||
|
# prune-old-versions.sh
|
||||||
|
# Keep only last 100 versions per document
|
||||||
|
|
||||||
|
for db in databases/*.db; do
|
||||||
|
sqlite3 "$db" <<EOF
|
||||||
|
DELETE FROM versions
|
||||||
|
WHERE id NOT IN (
|
||||||
|
SELECT id FROM versions
|
||||||
|
WHERE document_id = versions.document_id
|
||||||
|
ORDER BY version DESC
|
||||||
|
LIMIT 100
|
||||||
|
);
|
||||||
|
EOF
|
||||||
|
done
|
||||||
|
```
|
||||||
|
|
||||||
|
## Performance Tuning
|
||||||
|
|
||||||
|
### Connection Pool Sizing
|
||||||
|
|
||||||
|
Calculate optimal `max_connections_per_vault`:
|
||||||
|
|
||||||
|
```
|
||||||
|
max_connections = (concurrent_users × avg_operations_per_user) + buffer
|
||||||
|
```
|
||||||
|
|
||||||
|
**Example**:
|
||||||
|
- 20 concurrent users
|
||||||
|
- 2 operations per user on average
|
||||||
|
- 25% buffer
|
||||||
|
|
||||||
|
```
|
||||||
|
max_connections = (20 × 2) × 1.25 = 50
|
||||||
|
```
|
||||||
|
|
||||||
|
### Timeout Configuration
|
||||||
|
|
||||||
|
Adjust timeouts based on network characteristics:
|
||||||
|
|
||||||
|
**Fast local network**:
|
||||||
|
```yaml
|
||||||
|
database:
|
||||||
|
cursor_timeout_seconds: 30
|
||||||
|
|
||||||
|
server:
|
||||||
|
response_timeout_seconds: 30
|
||||||
|
```
|
||||||
|
|
||||||
|
**Slow or unreliable network**:
|
||||||
|
```yaml
|
||||||
|
database:
|
||||||
|
cursor_timeout_seconds: 180
|
||||||
|
|
||||||
|
server:
|
||||||
|
response_timeout_seconds: 120
|
||||||
|
```
|
||||||
|
|
||||||
|
**Mobile clients**:
|
||||||
|
```yaml
|
||||||
|
database:
|
||||||
|
cursor_timeout_seconds: 300 # Longer for intermittent connections
|
||||||
|
|
||||||
|
server:
|
||||||
|
response_timeout_seconds: 180
|
||||||
|
```
|
||||||
|
|
||||||
|
## Reverse Proxy Configuration
|
||||||
|
|
||||||
|
### Nginx with SSL
|
||||||
|
|
||||||
|
Complete Nginx configuration for production:
|
||||||
|
|
||||||
|
```nginx
|
||||||
|
# Rate limiting
|
||||||
|
limit_req_zone $binary_remote_addr zone=vaultlink:10m rate=10r/s;
|
||||||
|
|
||||||
|
upstream vaultlink {
|
||||||
|
server localhost:3000;
|
||||||
|
keepalive 32;
|
||||||
|
}
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 443 ssl http2;
|
||||||
|
server_name sync.example.com;
|
||||||
|
|
||||||
|
ssl_certificate /etc/letsencrypt/live/sync.example.com/fullchain.pem;
|
||||||
|
ssl_certificate_key /etc/letsencrypt/live/sync.example.com/privkey.pem;
|
||||||
|
|
||||||
|
# SSL security settings
|
||||||
|
ssl_protocols TLSv1.2 TLSv1.3;
|
||||||
|
ssl_ciphers HIGH:!aNULL:!MD5;
|
||||||
|
ssl_prefer_server_ciphers on;
|
||||||
|
|
||||||
|
# HSTS
|
||||||
|
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
|
||||||
|
|
||||||
|
# Rate limiting
|
||||||
|
limit_req zone=vaultlink burst=20 nodelay;
|
||||||
|
|
||||||
|
# Client body size (match server config)
|
||||||
|
client_max_body_size 512M;
|
||||||
|
|
||||||
|
# Timeouts
|
||||||
|
proxy_connect_timeout 90s;
|
||||||
|
proxy_send_timeout 90s;
|
||||||
|
proxy_read_timeout 3600s; # WebSocket long-lived connections
|
||||||
|
|
||||||
|
# WebSocket headers
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Upgrade $http_upgrade;
|
||||||
|
proxy_set_header Connection "upgrade";
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
|
||||||
|
# Disable buffering for WebSocket
|
||||||
|
proxy_buffering off;
|
||||||
|
|
||||||
|
location / {
|
||||||
|
proxy_pass http://vaultlink;
|
||||||
|
}
|
||||||
|
|
||||||
|
# Health check endpoint
|
||||||
|
location /health {
|
||||||
|
proxy_pass http://vaultlink/vaults/health/ping;
|
||||||
|
access_log off;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# Redirect HTTP to HTTPS
|
||||||
|
server {
|
||||||
|
listen 80;
|
||||||
|
server_name sync.example.com;
|
||||||
|
return 301 https://$server_name$request_uri;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Caddy with Auto SSL
|
||||||
|
|
||||||
|
Caddy handles SSL automatically:
|
||||||
|
|
||||||
|
```caddy
|
||||||
|
sync.example.com {
|
||||||
|
reverse_proxy localhost:3000 {
|
||||||
|
# WebSocket support
|
||||||
|
header_up X-Real-IP {remote_host}
|
||||||
|
header_up X-Forwarded-For {remote_host}
|
||||||
|
header_up X-Forwarded-Proto {scheme}
|
||||||
|
|
||||||
|
# Timeouts
|
||||||
|
transport http {
|
||||||
|
read_timeout 3600s
|
||||||
|
write_timeout 90s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# Rate limiting (requires caddy-rate-limit plugin)
|
||||||
|
rate_limit {
|
||||||
|
zone dynamic {
|
||||||
|
match {
|
||||||
|
remote_ip
|
||||||
|
}
|
||||||
|
rate 10r/s
|
||||||
|
burst 20
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Traefik Configuration
|
||||||
|
|
||||||
|
Using Docker labels:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
services:
|
||||||
|
vaultlink-server:
|
||||||
|
image: ghcr.io/schmelczer/vault-link-server:latest
|
||||||
|
labels:
|
||||||
|
- "traefik.enable=true"
|
||||||
|
- "traefik.http.routers.vaultlink.rule=Host(`sync.example.com`)"
|
||||||
|
- "traefik.http.routers.vaultlink.entrypoints=websecure"
|
||||||
|
- "traefik.http.routers.vaultlink.tls.certresolver=letsencrypt"
|
||||||
|
- "traefik.http.services.vaultlink.loadbalancer.server.port=3000"
|
||||||
|
# Middleware for timeouts
|
||||||
|
- "traefik.http.middlewares.vaultlink-timeout.timeout.request=3600s"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Docker Optimizations
|
||||||
|
|
||||||
|
### Resource Limits
|
||||||
|
|
||||||
|
Limit container resources:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
services:
|
||||||
|
vaultlink-server:
|
||||||
|
image: ghcr.io/schmelczer/vault-link-server:latest
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpus: '2.0'
|
||||||
|
memory: 4G
|
||||||
|
reservations:
|
||||||
|
cpus: '1.0'
|
||||||
|
memory: 2G
|
||||||
|
```
|
||||||
|
|
||||||
|
### Logging Configuration
|
||||||
|
|
||||||
|
Optimize Docker logging:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
services:
|
||||||
|
vaultlink-server:
|
||||||
|
image: ghcr.io/schmelczer/vault-link-server:latest
|
||||||
|
logging:
|
||||||
|
driver: "json-file"
|
||||||
|
options:
|
||||||
|
max-size: "50m"
|
||||||
|
max-file: "5"
|
||||||
|
```
|
||||||
|
|
||||||
|
### Volume Optimization
|
||||||
|
|
||||||
|
Use named volumes for better performance:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
services:
|
||||||
|
vaultlink-server:
|
||||||
|
image: ghcr.io/schmelczer/vault-link-server:latest
|
||||||
|
volumes:
|
||||||
|
- vaultlink-data:/data
|
||||||
|
- vaultlink-logs:/data/logs
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
vaultlink-data:
|
||||||
|
driver: local
|
||||||
|
driver_opts:
|
||||||
|
type: none
|
||||||
|
o: bind
|
||||||
|
device: /mnt/fast-ssd/vaultlink
|
||||||
|
vaultlink-logs:
|
||||||
|
driver: local
|
||||||
|
```
|
||||||
|
|
||||||
|
## High Availability
|
||||||
|
|
||||||
|
### Health Checks
|
||||||
|
|
||||||
|
Comprehensive health monitoring:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
services:
|
||||||
|
vaultlink-server:
|
||||||
|
image: ghcr.io/schmelczer/vault-link-server:latest
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "curl -f http://localhost:3000/vaults/health/ping || exit 1"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 3
|
||||||
|
start_period: 30s
|
||||||
|
```
|
||||||
|
|
||||||
|
Monitor health in production:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
#!/bin/bash
|
||||||
|
# health-monitor.sh
|
||||||
|
|
||||||
|
while true; do
|
||||||
|
if ! curl -sf http://localhost:3000/vaults/health/ping > /dev/null; then
|
||||||
|
echo "Health check failed at $(date)" | mail -s "VaultLink Down" admin@example.com
|
||||||
|
# Optionally restart
|
||||||
|
# docker restart vaultlink-server
|
||||||
|
fi
|
||||||
|
sleep 30
|
||||||
|
done
|
||||||
|
```
|
||||||
|
|
||||||
|
### Backup Automation
|
||||||
|
|
||||||
|
Automated backup script:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
#!/bin/bash
|
||||||
|
# backup-vaultlink.sh
|
||||||
|
|
||||||
|
BACKUP_DIR="/backup/vaultlink"
|
||||||
|
DATA_DIR="/data"
|
||||||
|
DATE=$(date +%Y%m%d-%H%M%S)
|
||||||
|
RETENTION_DAYS=30
|
||||||
|
|
||||||
|
# Create backup directory
|
||||||
|
mkdir -p "$BACKUP_DIR/$DATE"
|
||||||
|
|
||||||
|
# Backup databases (with WAL checkpoint)
|
||||||
|
for db in "$DATA_DIR"/databases/*.db; do
|
||||||
|
sqlite3 "$db" "PRAGMA wal_checkpoint(TRUNCATE);"
|
||||||
|
cp "$db" "$BACKUP_DIR/$DATE/"
|
||||||
|
[ -f "${db}-wal" ] && cp "${db}-wal" "$BACKUP_DIR/$DATE/"
|
||||||
|
[ -f "${db}-shm" ] && cp "${db}-shm" "$BACKUP_DIR/$DATE/"
|
||||||
|
done
|
||||||
|
|
||||||
|
# Backup configuration
|
||||||
|
cp "$DATA_DIR/config.yml" "$BACKUP_DIR/$DATE/"
|
||||||
|
|
||||||
|
# Compress backup
|
||||||
|
tar -czf "$BACKUP_DIR/vaultlink-$DATE.tar.gz" -C "$BACKUP_DIR" "$DATE"
|
||||||
|
rm -rf "$BACKUP_DIR/$DATE"
|
||||||
|
|
||||||
|
# Clean old backups
|
||||||
|
find "$BACKUP_DIR" -name "vaultlink-*.tar.gz" -mtime +$RETENTION_DAYS -delete
|
||||||
|
|
||||||
|
# Upload to remote storage (optional)
|
||||||
|
# rclone copy "$BACKUP_DIR/vaultlink-$DATE.tar.gz" remote:backups/
|
||||||
|
```
|
||||||
|
|
||||||
|
Schedule with cron:
|
||||||
|
```cron
|
||||||
|
0 2 * * * /opt/vaultlink/backup-vaultlink.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
### Restore from Backup
|
||||||
|
|
||||||
|
```bash
|
||||||
|
#!/bin/bash
|
||||||
|
# restore-vaultlink.sh
|
||||||
|
|
||||||
|
BACKUP_FILE="$1"
|
||||||
|
DATA_DIR="/data"
|
||||||
|
|
||||||
|
if [ -z "$BACKUP_FILE" ]; then
|
||||||
|
echo "Usage: $0 <backup-file.tar.gz>"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Stop server
|
||||||
|
docker stop vaultlink-server
|
||||||
|
|
||||||
|
# Extract backup
|
||||||
|
tar -xzf "$BACKUP_FILE" -C /tmp/
|
||||||
|
BACKUP_DATE=$(basename "$BACKUP_FILE" .tar.gz | cut -d- -f2-)
|
||||||
|
|
||||||
|
# Restore databases
|
||||||
|
cp /tmp/"$BACKUP_DATE"/databases/*.db "$DATA_DIR/databases/"
|
||||||
|
|
||||||
|
# Restore config (careful!)
|
||||||
|
# cp /tmp/$BACKUP_DATE/config.yml "$DATA_DIR/"
|
||||||
|
|
||||||
|
# Cleanup
|
||||||
|
rm -rf /tmp/"$BACKUP_DATE"
|
||||||
|
|
||||||
|
# Start server
|
||||||
|
docker start vaultlink-server
|
||||||
|
|
||||||
|
echo "Restore complete. Check server logs."
|
||||||
|
```
|
||||||
|
|
||||||
|
## Monitoring and Metrics
|
||||||
|
|
||||||
|
### Prometheus Metrics
|
||||||
|
|
||||||
|
While VaultLink doesn't expose metrics natively, monitor Docker:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
# docker-compose.yml
|
||||||
|
services:
|
||||||
|
vaultlink-server:
|
||||||
|
image: ghcr.io/schmelczer/vault-link-server:latest
|
||||||
|
labels:
|
||||||
|
- "prometheus.io/scrape=true"
|
||||||
|
- "prometheus.io/port=3000"
|
||||||
|
|
||||||
|
cadvisor:
|
||||||
|
image: gcr.io/cadvisor/cadvisor:latest
|
||||||
|
volumes:
|
||||||
|
- /:/rootfs:ro
|
||||||
|
- /var/run:/var/run:ro
|
||||||
|
- /sys:/sys:ro
|
||||||
|
- /var/lib/docker/:/var/lib/docker:ro
|
||||||
|
ports:
|
||||||
|
- 8080:8080
|
||||||
|
```
|
||||||
|
|
||||||
|
### Log Analysis
|
||||||
|
|
||||||
|
Analyze logs for insights:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Most active users
|
||||||
|
grep "authenticated" logs/*.log | cut -d"'" -f2 | sort | uniq -c | sort -rn
|
||||||
|
|
||||||
|
# Failed authentications by IP
|
||||||
|
grep "Authentication failed" logs/*.log | grep -oP '\d+\.\d+\.\d+\.\d+' | sort | uniq -c | sort -rn
|
||||||
|
|
||||||
|
# Upload activity
|
||||||
|
grep "Upload:" logs/*.log | wc -l
|
||||||
|
|
||||||
|
# Average files per vault
|
||||||
|
grep "Sync complete" logs/*.log | grep -oP '\d+ files' | cut -d' ' -f1 | awk '{sum+=$1; count++} END {print sum/count}'
|
||||||
|
```
|
||||||
|
|
||||||
|
### Alerting
|
||||||
|
|
||||||
|
Simple alerting with cron:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
#!/bin/bash
|
||||||
|
# alert-errors.sh
|
||||||
|
|
||||||
|
ERROR_THRESHOLD=10
|
||||||
|
ERROR_COUNT=$(grep -c "ERROR" logs/latest.log)
|
||||||
|
|
||||||
|
if [ "$ERROR_COUNT" -gt "$ERROR_THRESHOLD" ]; then
|
||||||
|
echo "VaultLink has $ERROR_COUNT errors in the last hour" | \
|
||||||
|
mail -s "VaultLink Alert" admin@example.com
|
||||||
|
fi
|
||||||
|
```
|
||||||
|
|
||||||
|
## Security Hardening
|
||||||
|
|
||||||
|
### Network Isolation
|
||||||
|
|
||||||
|
Run VaultLink in isolated network:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
services:
|
||||||
|
vaultlink-server:
|
||||||
|
image: ghcr.io/schmelczer/vault-link-server:latest
|
||||||
|
networks:
|
||||||
|
- vaultlink-internal
|
||||||
|
- proxy-external
|
||||||
|
|
||||||
|
networks:
|
||||||
|
vaultlink-internal:
|
||||||
|
internal: true
|
||||||
|
proxy-external:
|
||||||
|
driver: bridge
|
||||||
|
```
|
||||||
|
|
||||||
|
### Read-Only Root Filesystem
|
||||||
|
|
||||||
|
Run with read-only root (mount writable volumes for data):
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
services:
|
||||||
|
vaultlink-server:
|
||||||
|
image: ghcr.io/schmelczer/vault-link-server:latest
|
||||||
|
read_only: true
|
||||||
|
volumes:
|
||||||
|
- ./data:/data
|
||||||
|
- /tmp
|
||||||
|
```
|
||||||
|
|
||||||
|
### Drop Capabilities
|
||||||
|
|
||||||
|
Run with minimal privileges:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
services:
|
||||||
|
vaultlink-server:
|
||||||
|
image: ghcr.io/schmelczer/vault-link-server:latest
|
||||||
|
security_opt:
|
||||||
|
- no-new-privileges:true
|
||||||
|
cap_drop:
|
||||||
|
- ALL
|
||||||
|
```
|
||||||
|
|
||||||
|
## Migration
|
||||||
|
|
||||||
|
### Moving to New Server
|
||||||
|
|
||||||
|
1. **Backup on old server**:
|
||||||
|
```bash
|
||||||
|
./backup-vaultlink.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
2. **Transfer backup**:
|
||||||
|
```bash
|
||||||
|
scp vaultlink-backup.tar.gz new-server:/tmp/
|
||||||
|
```
|
||||||
|
|
||||||
|
3. **Restore on new server**:
|
||||||
|
```bash
|
||||||
|
./restore-vaultlink.sh /tmp/vaultlink-backup.tar.gz
|
||||||
|
```
|
||||||
|
|
||||||
|
4. **Update DNS/clients** to point to new server
|
||||||
|
|
||||||
|
5. **Verify sync** on all clients
|
||||||
|
|
||||||
|
### Version Upgrades
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Pull latest image
|
||||||
|
docker pull ghcr.io/schmelczer/vault-link-server:latest
|
||||||
|
|
||||||
|
# Backup first
|
||||||
|
./backup-vaultlink.sh
|
||||||
|
|
||||||
|
# Stop old container
|
||||||
|
docker stop vaultlink-server
|
||||||
|
docker rm vaultlink-server
|
||||||
|
|
||||||
|
# Start with new image
|
||||||
|
docker run -d \
|
||||||
|
--name vaultlink-server \
|
||||||
|
--restart unless-stopped \
|
||||||
|
-p 3000:3000 \
|
||||||
|
-v ./data:/data \
|
||||||
|
ghcr.io/schmelczer/vault-link-server:latest \
|
||||||
|
/app/sync_server /data/config.yml
|
||||||
|
|
||||||
|
# Check logs
|
||||||
|
docker logs -f vaultlink-server
|
||||||
|
```
|
||||||
|
|
||||||
|
## Next Steps
|
||||||
|
|
||||||
|
- [Understand the architecture →](/architecture/)
|
||||||
|
- [Deploy the server →](/guide/server-setup)
|
||||||
|
- [Configure clients →](/guide/obsidian-plugin)
|
||||||
530
docs/config/authentication.md
Normal file
530
docs/config/authentication.md
Normal file
|
|
@ -0,0 +1,530 @@
|
||||||
|
# Authentication Configuration
|
||||||
|
|
||||||
|
VaultLink uses token-based authentication with per-user vault access control. This guide covers all authentication and authorization options.
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
Authentication in VaultLink:
|
||||||
|
- **Token-based**: Users authenticate with secure tokens
|
||||||
|
- **Configured in YAML**: All users defined in `config.yml`
|
||||||
|
- **Vault-level access**: Control which vaults each user can access
|
||||||
|
- **No password hashing**: Tokens are treated as secrets
|
||||||
|
|
||||||
|
## Basic Configuration
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
users:
|
||||||
|
user_configs:
|
||||||
|
- name: alice
|
||||||
|
token: alice-secure-token-here
|
||||||
|
vault_access:
|
||||||
|
type: allow_access_to_all
|
||||||
|
```
|
||||||
|
|
||||||
|
## User Configuration Fields
|
||||||
|
|
||||||
|
### `name`
|
||||||
|
|
||||||
|
**Type**: String
|
||||||
|
**Required**: Yes
|
||||||
|
|
||||||
|
Human-readable identifier for the user. Used in logs and auditing.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
- name: alice
|
||||||
|
```
|
||||||
|
|
||||||
|
**Notes**:
|
||||||
|
- Must be unique across all users
|
||||||
|
- Used for identification only, not authentication
|
||||||
|
- Appears in server logs
|
||||||
|
- Can be any string (e.g., email, username)
|
||||||
|
|
||||||
|
### `token`
|
||||||
|
|
||||||
|
**Type**: String
|
||||||
|
**Required**: Yes
|
||||||
|
|
||||||
|
Authentication token for the user. Must be kept secret.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
- token: 1a2b3c4d5e6f7g8h9i0j...
|
||||||
|
```
|
||||||
|
|
||||||
|
**Best practices**:
|
||||||
|
- Generate with: `openssl rand -hex 32`
|
||||||
|
- Minimum length: 32 characters
|
||||||
|
- Use different token per user
|
||||||
|
- Never commit to version control
|
||||||
|
- Rotate periodically
|
||||||
|
|
||||||
|
**Example token generation**:
|
||||||
|
```bash
|
||||||
|
# Generate a secure token
|
||||||
|
openssl rand -hex 32
|
||||||
|
# Output: a7f3c9d1e8b2f4a6c3d9e1f7b8a4c2d6e9f1a3b7c5d8e2f4a6b9c3d1e8f7a4b2
|
||||||
|
```
|
||||||
|
|
||||||
|
### `vault_access`
|
||||||
|
|
||||||
|
**Type**: Object
|
||||||
|
**Required**: Yes
|
||||||
|
|
||||||
|
Defines which vaults the user can access.
|
||||||
|
|
||||||
|
**Three modes**:
|
||||||
|
1. `allow_access_to_all`: Access to all vaults
|
||||||
|
2. `allow_list`: Access to specific vaults only
|
||||||
|
3. `deny_list`: Access to all vaults except specific ones
|
||||||
|
|
||||||
|
## Access Control Modes
|
||||||
|
|
||||||
|
### Allow Access to All
|
||||||
|
|
||||||
|
Grant access to every vault:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
users:
|
||||||
|
user_configs:
|
||||||
|
- name: admin
|
||||||
|
token: admin-token
|
||||||
|
vault_access:
|
||||||
|
type: allow_access_to_all
|
||||||
|
```
|
||||||
|
|
||||||
|
**Use cases**:
|
||||||
|
- Administrator accounts
|
||||||
|
- Personal single-user deployments
|
||||||
|
- Development/testing
|
||||||
|
|
||||||
|
### Allow List
|
||||||
|
|
||||||
|
Grant access only to specific vaults:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
users:
|
||||||
|
user_configs:
|
||||||
|
- name: alice
|
||||||
|
token: alice-token
|
||||||
|
vault_access:
|
||||||
|
type: allow_list
|
||||||
|
allowed:
|
||||||
|
- personal
|
||||||
|
- shared-team
|
||||||
|
- project-alpha
|
||||||
|
```
|
||||||
|
|
||||||
|
**Use cases**:
|
||||||
|
- Multi-user deployments
|
||||||
|
- Restricted access scenarios
|
||||||
|
- Separation of concerns
|
||||||
|
|
||||||
|
**Notes**:
|
||||||
|
- User can only access listed vaults
|
||||||
|
- Attempting to access other vaults returns authentication error
|
||||||
|
- Empty list = no access to any vault
|
||||||
|
|
||||||
|
### Deny List
|
||||||
|
|
||||||
|
Grant access to all vaults except specific ones:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
users:
|
||||||
|
user_configs:
|
||||||
|
- name: bob
|
||||||
|
token: bob-token
|
||||||
|
vault_access:
|
||||||
|
type: deny_list
|
||||||
|
denied:
|
||||||
|
- restricted
|
||||||
|
- admin-only
|
||||||
|
```
|
||||||
|
|
||||||
|
**Use cases**:
|
||||||
|
- Users with broad access except sensitive vaults
|
||||||
|
- Simplify configuration when most vaults are accessible
|
||||||
|
|
||||||
|
**Notes**:
|
||||||
|
- User can access any vault not in the deny list
|
||||||
|
- Attempting to access denied vaults returns authentication error
|
||||||
|
|
||||||
|
## Multi-User Scenarios
|
||||||
|
|
||||||
|
### Personal Use (Single User)
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
users:
|
||||||
|
user_configs:
|
||||||
|
- name: me
|
||||||
|
token: my-super-secret-token
|
||||||
|
vault_access:
|
||||||
|
type: allow_access_to_all
|
||||||
|
```
|
||||||
|
|
||||||
|
### Small Team (Shared Vaults)
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
users:
|
||||||
|
user_configs:
|
||||||
|
- name: alice
|
||||||
|
token: alice-token
|
||||||
|
vault_access:
|
||||||
|
type: allow_list
|
||||||
|
allowed:
|
||||||
|
- personal-alice
|
||||||
|
- team-shared
|
||||||
|
- name: bob
|
||||||
|
token: bob-token
|
||||||
|
vault_access:
|
||||||
|
type: allow_list
|
||||||
|
allowed:
|
||||||
|
- personal-bob
|
||||||
|
- team-shared
|
||||||
|
- name: charlie
|
||||||
|
token: charlie-token
|
||||||
|
vault_access:
|
||||||
|
type: allow_list
|
||||||
|
allowed:
|
||||||
|
- personal-charlie
|
||||||
|
- team-shared
|
||||||
|
```
|
||||||
|
|
||||||
|
### Organization (Mixed Access)
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
users:
|
||||||
|
user_configs:
|
||||||
|
- name: admin
|
||||||
|
token: admin-token
|
||||||
|
vault_access:
|
||||||
|
type: allow_access_to_all
|
||||||
|
|
||||||
|
- name: developer
|
||||||
|
token: dev-token
|
||||||
|
vault_access:
|
||||||
|
type: allow_list
|
||||||
|
allowed:
|
||||||
|
- engineering-docs
|
||||||
|
- api-specs
|
||||||
|
- shared
|
||||||
|
|
||||||
|
- name: designer
|
||||||
|
token: design-token
|
||||||
|
vault_access:
|
||||||
|
type: allow_list
|
||||||
|
allowed:
|
||||||
|
- design-docs
|
||||||
|
- brand-assets
|
||||||
|
- shared
|
||||||
|
|
||||||
|
- name: readonly
|
||||||
|
token: readonly-token
|
||||||
|
vault_access:
|
||||||
|
type: allow_list
|
||||||
|
allowed:
|
||||||
|
- public-wiki
|
||||||
|
```
|
||||||
|
|
||||||
|
## Authentication Flow
|
||||||
|
|
||||||
|
### Connection
|
||||||
|
|
||||||
|
1. Client connects via WebSocket
|
||||||
|
2. Client sends authentication message:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"type": "auth",
|
||||||
|
"token": "user-token",
|
||||||
|
"vault": "vault-name"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
3. Server validates:
|
||||||
|
- Token exists in config
|
||||||
|
- User has access to requested vault
|
||||||
|
4. Server responds:
|
||||||
|
- Success: Connection established
|
||||||
|
- Failure: Connection closed with error
|
||||||
|
|
||||||
|
### Validation
|
||||||
|
|
||||||
|
Server checks:
|
||||||
|
1. **Token match**: Token exists in `user_configs`
|
||||||
|
2. **Vault access**: User has permission for vault
|
||||||
|
3. **Connection limits**: Not exceeding `max_clients_per_vault`
|
||||||
|
|
||||||
|
### Errors
|
||||||
|
|
||||||
|
**Invalid token**:
|
||||||
|
```
|
||||||
|
Authentication failed: Invalid token
|
||||||
|
```
|
||||||
|
|
||||||
|
**No vault access**:
|
||||||
|
```
|
||||||
|
Authentication failed: User does not have access to vault 'restricted'
|
||||||
|
```
|
||||||
|
|
||||||
|
**Connection limit**:
|
||||||
|
```
|
||||||
|
Connection rejected: Maximum clients reached for vault
|
||||||
|
```
|
||||||
|
|
||||||
|
## Security Best Practices
|
||||||
|
|
||||||
|
### Token Generation
|
||||||
|
|
||||||
|
Generate strong tokens:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 64 character hex token (256 bits)
|
||||||
|
openssl rand -hex 32
|
||||||
|
|
||||||
|
# Base64 encoded (256 bits)
|
||||||
|
openssl rand -base64 32
|
||||||
|
|
||||||
|
# UUID v4
|
||||||
|
uuidgen
|
||||||
|
```
|
||||||
|
|
||||||
|
### Token Storage
|
||||||
|
|
||||||
|
**In config file**:
|
||||||
|
```yaml
|
||||||
|
users:
|
||||||
|
user_configs:
|
||||||
|
- name: alice
|
||||||
|
token: !ENV ALICE_TOKEN # Read from environment variable
|
||||||
|
```
|
||||||
|
|
||||||
|
**Load from environment**:
|
||||||
|
```bash
|
||||||
|
export ALICE_TOKEN="$(openssl rand -hex 32)"
|
||||||
|
./sync_server config.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
### Token Rotation
|
||||||
|
|
||||||
|
Periodically change tokens:
|
||||||
|
|
||||||
|
1. Generate new token
|
||||||
|
2. Update `config.yml`
|
||||||
|
3. Restart server
|
||||||
|
4. Update clients with new token
|
||||||
|
|
||||||
|
### Token Revocation
|
||||||
|
|
||||||
|
To revoke access:
|
||||||
|
1. Remove user from `config.yml`
|
||||||
|
2. Restart server
|
||||||
|
3. User's connections will be rejected
|
||||||
|
|
||||||
|
For immediate revocation:
|
||||||
|
- Remove user from config
|
||||||
|
- Restart server
|
||||||
|
- Existing connections are terminated
|
||||||
|
|
||||||
|
## Access Patterns
|
||||||
|
|
||||||
|
### Read-Only Users
|
||||||
|
|
||||||
|
VaultLink doesn't distinguish read-only vs read-write. Implement via client:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
# Server: Grant access
|
||||||
|
users:
|
||||||
|
user_configs:
|
||||||
|
- name: readonly
|
||||||
|
token: readonly-token
|
||||||
|
vault_access:
|
||||||
|
type: allow_list
|
||||||
|
allowed:
|
||||||
|
- public
|
||||||
|
|
||||||
|
# Client: Use CLI in read-only mode (mount vault read-only)
|
||||||
|
docker run -v /vault:/vault:ro ...
|
||||||
|
```
|
||||||
|
|
||||||
|
### Temporary Access
|
||||||
|
|
||||||
|
Grant temporary access:
|
||||||
|
|
||||||
|
1. Add user to config
|
||||||
|
2. Set reminder to remove later
|
||||||
|
3. Remove user when no longer needed
|
||||||
|
4. Restart server
|
||||||
|
|
||||||
|
For automation:
|
||||||
|
```bash
|
||||||
|
# Add user with expiry comment
|
||||||
|
echo " - name: temp-user # EXPIRES: 2024-12-31" >> config.yml
|
||||||
|
echo " token: temp-token" >> config.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
### Shared Tokens (Not Recommended)
|
||||||
|
|
||||||
|
Multiple users sharing a token:
|
||||||
|
- All appear as same user in logs
|
||||||
|
- Can't revoke individual access
|
||||||
|
- Security risk if one person leaves
|
||||||
|
|
||||||
|
**Instead**: Create separate users with same vault access.
|
||||||
|
|
||||||
|
## Monitoring
|
||||||
|
|
||||||
|
### Server Logs
|
||||||
|
|
||||||
|
Authentication events are logged:
|
||||||
|
|
||||||
|
```
|
||||||
|
2024-01-01 12:00:00 INFO User 'alice' authenticated for vault 'personal'
|
||||||
|
2024-01-01 12:00:05 WARN Authentication failed: Invalid token from 192.168.1.100
|
||||||
|
2024-01-01 12:00:10 WARN User 'bob' denied access to vault 'restricted'
|
||||||
|
```
|
||||||
|
|
||||||
|
### Audit Trail
|
||||||
|
|
||||||
|
Monitor authentication:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# View authentication logs
|
||||||
|
grep "authenticated" logs/*.log
|
||||||
|
|
||||||
|
# View failed authentications
|
||||||
|
grep "Authentication failed" logs/*.log
|
||||||
|
|
||||||
|
# View access denials
|
||||||
|
grep "denied access" logs/*.log
|
||||||
|
```
|
||||||
|
|
||||||
|
## Advanced Scenarios
|
||||||
|
|
||||||
|
### Multiple Servers
|
||||||
|
|
||||||
|
Same user across multiple server instances:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
# Server 1 config.yml
|
||||||
|
users:
|
||||||
|
user_configs:
|
||||||
|
- name: alice
|
||||||
|
token: alice-global-token
|
||||||
|
vault_access:
|
||||||
|
type: allow_list
|
||||||
|
allowed:
|
||||||
|
- vault-1
|
||||||
|
- vault-2
|
||||||
|
|
||||||
|
# Server 2 config.yml
|
||||||
|
users:
|
||||||
|
user_configs:
|
||||||
|
- name: alice
|
||||||
|
token: alice-global-token # Same token
|
||||||
|
vault_access:
|
||||||
|
type: allow_list
|
||||||
|
allowed:
|
||||||
|
- vault-3
|
||||||
|
- vault-4
|
||||||
|
```
|
||||||
|
|
||||||
|
### Service Accounts
|
||||||
|
|
||||||
|
Tokens for automated systems:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
users:
|
||||||
|
user_configs:
|
||||||
|
- name: backup-service
|
||||||
|
token: backup-service-token
|
||||||
|
vault_access:
|
||||||
|
type: allow_access_to_all
|
||||||
|
|
||||||
|
- name: ci-pipeline
|
||||||
|
token: ci-token
|
||||||
|
vault_access:
|
||||||
|
type: allow_list
|
||||||
|
allowed:
|
||||||
|
- documentation
|
||||||
|
|
||||||
|
- name: monitoring
|
||||||
|
token: monitoring-token
|
||||||
|
vault_access:
|
||||||
|
type: allow_list
|
||||||
|
allowed:
|
||||||
|
- metrics
|
||||||
|
```
|
||||||
|
|
||||||
|
### Dynamic Vault Access
|
||||||
|
|
||||||
|
VaultLink doesn't support runtime user management. To change access:
|
||||||
|
|
||||||
|
1. Update `config.yml`
|
||||||
|
2. Restart server
|
||||||
|
3. Users reconnect automatically
|
||||||
|
|
||||||
|
For frequent changes, consider:
|
||||||
|
- Over-provision access (deny list)
|
||||||
|
- Use external authentication proxy
|
||||||
|
- Script config updates + reload
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
### Can't connect
|
||||||
|
|
||||||
|
**Check token**:
|
||||||
|
```bash
|
||||||
|
# Verify token in config matches client
|
||||||
|
grep "token:" config.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
**Check vault name**:
|
||||||
|
```bash
|
||||||
|
# Ensure vault is in allowed list
|
||||||
|
grep -A 5 "name: alice" config.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
**Check server logs**:
|
||||||
|
```bash
|
||||||
|
tail -f logs/*.log | grep -i auth
|
||||||
|
```
|
||||||
|
|
||||||
|
### Access denied
|
||||||
|
|
||||||
|
**Verify vault access**:
|
||||||
|
```yaml
|
||||||
|
# Check user's vault_access configuration
|
||||||
|
users:
|
||||||
|
user_configs:
|
||||||
|
- name: alice
|
||||||
|
vault_access:
|
||||||
|
type: allow_list
|
||||||
|
allowed:
|
||||||
|
- vault-name # Must match exactly
|
||||||
|
```
|
||||||
|
|
||||||
|
**Case sensitivity**:
|
||||||
|
- Vault names are case-sensitive
|
||||||
|
- `Vault` ≠ `vault`
|
||||||
|
- Ensure exact match in config and client
|
||||||
|
|
||||||
|
### Token not working
|
||||||
|
|
||||||
|
**Check for typos**:
|
||||||
|
- Extra spaces
|
||||||
|
- Hidden characters
|
||||||
|
- Wrong quotes in YAML
|
||||||
|
|
||||||
|
**Regenerate token**:
|
||||||
|
```bash
|
||||||
|
# Generate new token
|
||||||
|
openssl rand -hex 32
|
||||||
|
|
||||||
|
# Update config
|
||||||
|
# Restart server
|
||||||
|
# Update client
|
||||||
|
```
|
||||||
|
|
||||||
|
## Next Steps
|
||||||
|
|
||||||
|
- [Server configuration reference →](/config/server)
|
||||||
|
- [Advanced configuration →](/config/advanced)
|
||||||
|
- [Deploy the server →](/guide/server-setup)
|
||||||
470
docs/config/server.md
Normal file
470
docs/config/server.md
Normal file
|
|
@ -0,0 +1,470 @@
|
||||||
|
# Server Configuration
|
||||||
|
|
||||||
|
Complete reference for configuring the VaultLink sync server via `config.yml`.
|
||||||
|
|
||||||
|
## Configuration File Format
|
||||||
|
|
||||||
|
The server is configured using a YAML file passed as a command-line argument:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
/app/sync_server /path/to/config.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
## Complete Example
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
database:
|
||||||
|
databases_directory_path: databases
|
||||||
|
max_connections_per_vault: 12
|
||||||
|
cursor_timeout_seconds: 60
|
||||||
|
|
||||||
|
server:
|
||||||
|
host: 0.0.0.0
|
||||||
|
port: 3000
|
||||||
|
max_body_size_mb: 512
|
||||||
|
max_clients_per_vault: 256
|
||||||
|
response_timeout_seconds: 60
|
||||||
|
|
||||||
|
users:
|
||||||
|
user_configs:
|
||||||
|
- name: admin
|
||||||
|
token: your-secure-random-token
|
||||||
|
vault_access:
|
||||||
|
type: allow_access_to_all
|
||||||
|
- name: alice
|
||||||
|
token: alice-token
|
||||||
|
vault_access:
|
||||||
|
type: allow_list
|
||||||
|
allowed:
|
||||||
|
- personal
|
||||||
|
- shared
|
||||||
|
- name: bob
|
||||||
|
token: bob-token
|
||||||
|
vault_access:
|
||||||
|
type: deny_list
|
||||||
|
denied:
|
||||||
|
- restricted
|
||||||
|
|
||||||
|
logging:
|
||||||
|
log_directory: logs
|
||||||
|
log_rotation: 7days
|
||||||
|
```
|
||||||
|
|
||||||
|
## Database Section
|
||||||
|
|
||||||
|
### `databases_directory_path`
|
||||||
|
|
||||||
|
**Type**: String
|
||||||
|
**Required**: Yes
|
||||||
|
**Default**: None
|
||||||
|
|
||||||
|
Directory where SQLite database files are stored. One database file per vault.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
database:
|
||||||
|
databases_directory_path: /data/databases
|
||||||
|
```
|
||||||
|
|
||||||
|
The directory structure:
|
||||||
|
```
|
||||||
|
databases/
|
||||||
|
├── vault-1.db
|
||||||
|
├── vault-2.db
|
||||||
|
└── personal.db
|
||||||
|
```
|
||||||
|
|
||||||
|
**Notes**:
|
||||||
|
- Path is relative to working directory or absolute
|
||||||
|
- Directory must be writable by the server process
|
||||||
|
- Ensure adequate disk space for vault data
|
||||||
|
- Back up this directory regularly
|
||||||
|
|
||||||
|
### `max_connections_per_vault`
|
||||||
|
|
||||||
|
**Type**: Integer
|
||||||
|
**Required**: Yes
|
||||||
|
**Default**: None
|
||||||
|
**Recommended**: 12
|
||||||
|
|
||||||
|
Maximum concurrent database connections per vault.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
database:
|
||||||
|
max_connections_per_vault: 12
|
||||||
|
```
|
||||||
|
|
||||||
|
**Tuning**:
|
||||||
|
- Higher values: Better performance under load
|
||||||
|
- Lower values: Less memory usage
|
||||||
|
- Typical range: 8-20
|
||||||
|
- Consider: Number of concurrent users × average operations per user
|
||||||
|
|
||||||
|
### `cursor_timeout_seconds`
|
||||||
|
|
||||||
|
**Type**: Integer
|
||||||
|
**Required**: Yes
|
||||||
|
**Default**: None
|
||||||
|
**Recommended**: 60
|
||||||
|
|
||||||
|
How long to keep database cursors alive for inactive clients.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
database:
|
||||||
|
cursor_timeout_seconds: 60
|
||||||
|
```
|
||||||
|
|
||||||
|
**Notes**:
|
||||||
|
- Cursors track client sync state
|
||||||
|
- Timeout too short: Clients may need to re-sync frequently
|
||||||
|
- Timeout too long: More memory usage
|
||||||
|
- Typical range: 30-300 seconds
|
||||||
|
|
||||||
|
## Server Section
|
||||||
|
|
||||||
|
### `host`
|
||||||
|
|
||||||
|
**Type**: String
|
||||||
|
**Required**: Yes
|
||||||
|
**Default**: None
|
||||||
|
|
||||||
|
Network interface to bind the server to.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
server:
|
||||||
|
host: 0.0.0.0 # All interfaces
|
||||||
|
# OR
|
||||||
|
host: 127.0.0.1 # Localhost only
|
||||||
|
# OR
|
||||||
|
host: 192.168.1.100 # Specific interface
|
||||||
|
```
|
||||||
|
|
||||||
|
**Common values**:
|
||||||
|
- `0.0.0.0`: Listen on all network interfaces (production)
|
||||||
|
- `127.0.0.1`: Listen on localhost only (development/testing)
|
||||||
|
- Specific IP: Listen on specific interface
|
||||||
|
|
||||||
|
### `port`
|
||||||
|
|
||||||
|
**Type**: Integer
|
||||||
|
**Required**: Yes
|
||||||
|
**Default**: None
|
||||||
|
**Recommended**: 3000
|
||||||
|
|
||||||
|
TCP port to listen on.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
server:
|
||||||
|
port: 3000
|
||||||
|
```
|
||||||
|
|
||||||
|
**Notes**:
|
||||||
|
- Must be available (not in use)
|
||||||
|
- Privileged ports (< 1024) require root
|
||||||
|
- Common ports: 3000, 8080, 8888
|
||||||
|
- Configure firewall to allow this port
|
||||||
|
|
||||||
|
### `max_body_size_mb`
|
||||||
|
|
||||||
|
**Type**: Integer
|
||||||
|
**Required**: Yes
|
||||||
|
**Default**: None
|
||||||
|
**Recommended**: 512
|
||||||
|
|
||||||
|
Maximum size of HTTP request body in megabytes.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
server:
|
||||||
|
max_body_size_mb: 512
|
||||||
|
```
|
||||||
|
|
||||||
|
**Usage**:
|
||||||
|
- Limits file upload size
|
||||||
|
- Prevents memory exhaustion attacks
|
||||||
|
- Must be larger than largest expected file
|
||||||
|
- Consider client `max_file_size_mb` settings
|
||||||
|
|
||||||
|
**Tuning**:
|
||||||
|
- Small vaults (mostly text): 100 MB
|
||||||
|
- Medium vaults (some images): 512 MB
|
||||||
|
- Large vaults (many images/PDFs): 1024+ MB
|
||||||
|
|
||||||
|
### `max_clients_per_vault`
|
||||||
|
|
||||||
|
**Type**: Integer
|
||||||
|
**Required**: Yes
|
||||||
|
**Default**: None
|
||||||
|
**Recommended**: 256
|
||||||
|
|
||||||
|
Maximum concurrent clients per vault.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
server:
|
||||||
|
max_clients_per_vault: 256
|
||||||
|
```
|
||||||
|
|
||||||
|
**Notes**:
|
||||||
|
- Limits concurrent WebSocket connections
|
||||||
|
- Prevents resource exhaustion
|
||||||
|
- Consider expected number of users
|
||||||
|
- Each client uses memory and file descriptors
|
||||||
|
|
||||||
|
**Scaling**:
|
||||||
|
- Personal use: 10-50
|
||||||
|
- Small team: 50-100
|
||||||
|
- Large team: 100-500
|
||||||
|
|
||||||
|
### `response_timeout_seconds`
|
||||||
|
|
||||||
|
**Type**: Integer
|
||||||
|
**Required**: Yes
|
||||||
|
**Default**: None
|
||||||
|
**Recommended**: 60
|
||||||
|
|
||||||
|
Maximum time to wait for client responses.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
server:
|
||||||
|
response_timeout_seconds: 60
|
||||||
|
```
|
||||||
|
|
||||||
|
**Usage**:
|
||||||
|
- Timeout for HTTP requests
|
||||||
|
- Timeout for WebSocket operations
|
||||||
|
- Clients disconnected if unresponsive
|
||||||
|
|
||||||
|
**Tuning**:
|
||||||
|
- Fast networks: 30 seconds
|
||||||
|
- Slow networks: 90-120 seconds
|
||||||
|
- Large file uploads: Increase proportionally
|
||||||
|
|
||||||
|
## Users Section
|
||||||
|
|
||||||
|
See [Authentication Configuration →](/config/authentication) for detailed user configuration.
|
||||||
|
|
||||||
|
## Logging Section
|
||||||
|
|
||||||
|
### `log_directory`
|
||||||
|
|
||||||
|
**Type**: String
|
||||||
|
**Required**: Yes
|
||||||
|
**Default**: None
|
||||||
|
|
||||||
|
Directory where log files are written.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
logging:
|
||||||
|
log_directory: /data/logs
|
||||||
|
# OR
|
||||||
|
log_directory: logs # Relative to working directory
|
||||||
|
```
|
||||||
|
|
||||||
|
**Notes**:
|
||||||
|
- Path is relative to working directory or absolute
|
||||||
|
- Directory must be writable
|
||||||
|
- Logs are rotated based on `log_rotation`
|
||||||
|
- Monitor disk usage
|
||||||
|
|
||||||
|
### `log_rotation`
|
||||||
|
|
||||||
|
**Type**: String
|
||||||
|
**Required**: Yes
|
||||||
|
**Default**: None
|
||||||
|
|
||||||
|
How often to rotate log files.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
logging:
|
||||||
|
log_rotation: 7days
|
||||||
|
# OR
|
||||||
|
log_rotation: 24hours
|
||||||
|
# OR
|
||||||
|
log_rotation: 30days
|
||||||
|
```
|
||||||
|
|
||||||
|
**Format**: `<number><unit>`
|
||||||
|
|
||||||
|
**Units**:
|
||||||
|
- `hours`: Hours (e.g., `12hours`, `24hours`)
|
||||||
|
- `days`: Days (e.g., `7days`, `30days`)
|
||||||
|
|
||||||
|
**Recommendations**:
|
||||||
|
- Development: `24hours` or `7days`
|
||||||
|
- Production: `7days` or `30days`
|
||||||
|
- High traffic: `24hours` (logs can be large)
|
||||||
|
|
||||||
|
## Environment-Specific Configurations
|
||||||
|
|
||||||
|
### Development
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
database:
|
||||||
|
databases_directory_path: ./databases
|
||||||
|
max_connections_per_vault: 8
|
||||||
|
cursor_timeout_seconds: 30
|
||||||
|
|
||||||
|
server:
|
||||||
|
host: 127.0.0.1
|
||||||
|
port: 3000
|
||||||
|
max_body_size_mb: 100
|
||||||
|
max_clients_per_vault: 10
|
||||||
|
response_timeout_seconds: 30
|
||||||
|
|
||||||
|
users:
|
||||||
|
user_configs:
|
||||||
|
- name: dev
|
||||||
|
token: dev-token
|
||||||
|
vault_access:
|
||||||
|
type: allow_access_to_all
|
||||||
|
|
||||||
|
logging:
|
||||||
|
log_directory: logs
|
||||||
|
log_rotation: 24hours
|
||||||
|
```
|
||||||
|
|
||||||
|
### Production
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
database:
|
||||||
|
databases_directory_path: /data/databases
|
||||||
|
max_connections_per_vault: 16
|
||||||
|
cursor_timeout_seconds: 120
|
||||||
|
|
||||||
|
server:
|
||||||
|
host: 0.0.0.0
|
||||||
|
port: 3000
|
||||||
|
max_body_size_mb: 512
|
||||||
|
max_clients_per_vault: 256
|
||||||
|
response_timeout_seconds: 90
|
||||||
|
|
||||||
|
users:
|
||||||
|
user_configs:
|
||||||
|
- name: admin
|
||||||
|
token: <strong-random-token>
|
||||||
|
vault_access:
|
||||||
|
type: allow_access_to_all
|
||||||
|
# Additional users...
|
||||||
|
|
||||||
|
logging:
|
||||||
|
log_directory: /data/logs
|
||||||
|
log_rotation: 7days
|
||||||
|
```
|
||||||
|
|
||||||
|
## Validation
|
||||||
|
|
||||||
|
The server validates configuration on startup:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Start server
|
||||||
|
./sync_server config.yml
|
||||||
|
|
||||||
|
# Check for errors in logs
|
||||||
|
tail -f logs/latest.log
|
||||||
|
```
|
||||||
|
|
||||||
|
**Common errors**:
|
||||||
|
- Missing required fields
|
||||||
|
- Invalid YAML syntax
|
||||||
|
- Invalid values (negative numbers, etc.)
|
||||||
|
- Directory not writable
|
||||||
|
|
||||||
|
## Performance Tuning
|
||||||
|
|
||||||
|
### High Concurrency
|
||||||
|
|
||||||
|
For many concurrent users:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
database:
|
||||||
|
max_connections_per_vault: 20 # Increase
|
||||||
|
|
||||||
|
server:
|
||||||
|
max_clients_per_vault: 500 # Increase
|
||||||
|
response_timeout_seconds: 120 # Increase for slow clients
|
||||||
|
```
|
||||||
|
|
||||||
|
### Large Files
|
||||||
|
|
||||||
|
For vaults with large files:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
server:
|
||||||
|
max_body_size_mb: 1024 # Allow larger uploads
|
||||||
|
response_timeout_seconds: 180 # More time for uploads
|
||||||
|
```
|
||||||
|
|
||||||
|
### Resource-Constrained Systems
|
||||||
|
|
||||||
|
For limited CPU/memory:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
database:
|
||||||
|
max_connections_per_vault: 6 # Reduce
|
||||||
|
|
||||||
|
server:
|
||||||
|
max_clients_per_vault: 50 # Reduce
|
||||||
|
max_body_size_mb: 256 # Reduce
|
||||||
|
```
|
||||||
|
|
||||||
|
## Security Considerations
|
||||||
|
|
||||||
|
### Token Security
|
||||||
|
|
||||||
|
- Use strong random tokens: `openssl rand -hex 32`
|
||||||
|
- Never commit tokens to version control
|
||||||
|
- Rotate tokens periodically
|
||||||
|
- Use different tokens per user
|
||||||
|
|
||||||
|
### Network Security
|
||||||
|
|
||||||
|
- Bind to `127.0.0.1` if using reverse proxy on same host
|
||||||
|
- Use firewall to restrict access
|
||||||
|
- Enable SSL/TLS via reverse proxy
|
||||||
|
|
||||||
|
### Resource Limits
|
||||||
|
|
||||||
|
- Set `max_clients_per_vault` to prevent DoS
|
||||||
|
- Set `max_body_size_mb` to prevent memory exhaustion
|
||||||
|
- Configure `response_timeout_seconds` to prevent hanging connections
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
### Server won't start
|
||||||
|
|
||||||
|
**Check YAML syntax**:
|
||||||
|
```bash
|
||||||
|
# Use a YAML validator
|
||||||
|
python -c 'import yaml, sys; yaml.safe_load(open("config.yml"))'
|
||||||
|
```
|
||||||
|
|
||||||
|
**Check file paths**:
|
||||||
|
```bash
|
||||||
|
# Ensure directories exist and are writable
|
||||||
|
mkdir -p databases logs
|
||||||
|
chmod 755 databases logs
|
||||||
|
```
|
||||||
|
|
||||||
|
**Check port availability**:
|
||||||
|
```bash
|
||||||
|
# Verify port is not in use
|
||||||
|
lsof -i :3000
|
||||||
|
```
|
||||||
|
|
||||||
|
### High memory usage
|
||||||
|
|
||||||
|
- Reduce `max_connections_per_vault`
|
||||||
|
- Reduce `max_clients_per_vault`
|
||||||
|
- Reduce `max_body_size_mb`
|
||||||
|
- Check for large vaults or many concurrent users
|
||||||
|
|
||||||
|
### Slow performance
|
||||||
|
|
||||||
|
- Increase `max_connections_per_vault`
|
||||||
|
- Increase database connection pool
|
||||||
|
- Use SSD for database storage
|
||||||
|
- Monitor database size (vacuum if needed)
|
||||||
|
|
||||||
|
## Next Steps
|
||||||
|
|
||||||
|
- [Configure authentication →](/config/authentication)
|
||||||
|
- [Advanced configuration options →](/config/advanced)
|
||||||
|
- [Deploy the server →](/guide/server-setup)
|
||||||
516
docs/guide/cli-client.md
Normal file
516
docs/guide/cli-client.md
Normal file
|
|
@ -0,0 +1,516 @@
|
||||||
|
# CLI Client
|
||||||
|
|
||||||
|
The VaultLink CLI client provides standalone synchronization without requiring Obsidian. Perfect for servers, automation, backups, or syncing vaults on headless systems.
|
||||||
|
|
||||||
|
## Installation
|
||||||
|
|
||||||
|
### Docker (Recommended)
|
||||||
|
|
||||||
|
Pull the latest image:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker pull ghcr.io/schmelczer/vault-link-cli:latest
|
||||||
|
```
|
||||||
|
|
||||||
|
### npm
|
||||||
|
|
||||||
|
Install globally:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm install -g @schmelczer/local-client-cli
|
||||||
|
```
|
||||||
|
|
||||||
|
Verify installation:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
vaultlink --version
|
||||||
|
```
|
||||||
|
|
||||||
|
### From Source
|
||||||
|
|
||||||
|
Build from the repository:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git clone https://github.com/schmelczer/vault-link.git
|
||||||
|
cd vault-link/frontend/local-client-cli
|
||||||
|
npm install
|
||||||
|
npm run build
|
||||||
|
node dist/cli.js --help
|
||||||
|
```
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
### Basic Usage
|
||||||
|
|
||||||
|
```bash
|
||||||
|
vaultlink \
|
||||||
|
--local-path /path/to/vault \
|
||||||
|
--remote-uri wss://sync.example.com \
|
||||||
|
--token your-auth-token \
|
||||||
|
--vault-name default
|
||||||
|
```
|
||||||
|
|
||||||
|
### Docker Usage
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker run -v /path/to/vault:/vault \
|
||||||
|
ghcr.io/schmelczer/vault-link-cli:latest \
|
||||||
|
-l /vault \
|
||||||
|
-r wss://sync.example.com \
|
||||||
|
-t your-auth-token \
|
||||||
|
-v default
|
||||||
|
```
|
||||||
|
|
||||||
|
### Docker Compose
|
||||||
|
|
||||||
|
Create `docker-compose.yml`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
services:
|
||||||
|
vaultlink-cli:
|
||||||
|
image: ghcr.io/schmelczer/vault-link-cli:latest
|
||||||
|
restart: unless-stopped
|
||||||
|
volumes:
|
||||||
|
- ./vault:/vault
|
||||||
|
command:
|
||||||
|
- "-l"
|
||||||
|
- "/vault"
|
||||||
|
- "-r"
|
||||||
|
- "wss://sync.example.com"
|
||||||
|
- "-t"
|
||||||
|
- "your-token"
|
||||||
|
- "-v"
|
||||||
|
- "default"
|
||||||
|
```
|
||||||
|
|
||||||
|
Start the client:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
## Configuration Options
|
||||||
|
|
||||||
|
### Required Arguments
|
||||||
|
|
||||||
|
| Argument | Short | Description | Example |
|
||||||
|
|----------|-------|-------------|---------|
|
||||||
|
| `--local-path` | `-l` | Local directory to sync | `/vault` |
|
||||||
|
| `--remote-uri` | `-r` | Server WebSocket URI | `wss://sync.example.com` |
|
||||||
|
| `--token` | `-t` | Authentication token | `abc123...` |
|
||||||
|
| `--vault-name` | `-v` | Vault name on server | `default` |
|
||||||
|
|
||||||
|
### Optional Arguments
|
||||||
|
|
||||||
|
| Argument | Default | Description |
|
||||||
|
|----------|---------|-------------|
|
||||||
|
| `--sync-concurrency` | `1` | Concurrent file operations |
|
||||||
|
| `--max-file-size-mb` | `10` | Max file size in MB |
|
||||||
|
| `--ignore-pattern` | - | Glob pattern to ignore (repeatable) |
|
||||||
|
| `--websocket-retry-interval-ms` | `3500` | Reconnection interval |
|
||||||
|
| `--log-level` | `INFO` | Log level: DEBUG, INFO, WARNING, ERROR |
|
||||||
|
|
||||||
|
### Environment Variables
|
||||||
|
|
||||||
|
Alternative to command-line arguments:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export VAULTLINK_LOCAL_PATH="/vault"
|
||||||
|
export VAULTLINK_REMOTE_URI="wss://sync.example.com"
|
||||||
|
export VAULTLINK_TOKEN="your-token"
|
||||||
|
export VAULTLINK_VAULT_NAME="default"
|
||||||
|
|
||||||
|
vaultlink
|
||||||
|
```
|
||||||
|
|
||||||
|
## Examples
|
||||||
|
|
||||||
|
### Basic Sync
|
||||||
|
|
||||||
|
Sync a local directory to the server:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
vaultlink \
|
||||||
|
-l ./my-notes \
|
||||||
|
-r wss://sync.example.com \
|
||||||
|
-t my-secure-token \
|
||||||
|
-v personal
|
||||||
|
```
|
||||||
|
|
||||||
|
### With Ignore Patterns
|
||||||
|
|
||||||
|
Exclude specific files or directories:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
vaultlink \
|
||||||
|
-l ./vault \
|
||||||
|
-r wss://sync.example.com \
|
||||||
|
-t token123 \
|
||||||
|
-v default \
|
||||||
|
--ignore-pattern "*.tmp" \
|
||||||
|
--ignore-pattern ".DS_Store" \
|
||||||
|
--ignore-pattern "node_modules/**"
|
||||||
|
```
|
||||||
|
|
||||||
|
### Debug Logging
|
||||||
|
|
||||||
|
Enable verbose logging:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
vaultlink \
|
||||||
|
-l ./vault \
|
||||||
|
-r wss://sync.example.com \
|
||||||
|
-t token123 \
|
||||||
|
-v default \
|
||||||
|
--log-level DEBUG
|
||||||
|
```
|
||||||
|
|
||||||
|
### High Concurrency
|
||||||
|
|
||||||
|
Faster initial sync:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
vaultlink \
|
||||||
|
-l ./vault \
|
||||||
|
-r wss://sync.example.com \
|
||||||
|
-t token123 \
|
||||||
|
-v default \
|
||||||
|
--sync-concurrency 5
|
||||||
|
```
|
||||||
|
|
||||||
|
### Large Files
|
||||||
|
|
||||||
|
Allow larger file uploads:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
vaultlink \
|
||||||
|
-l ./vault \
|
||||||
|
-r wss://sync.example.com \
|
||||||
|
-t token123 \
|
||||||
|
-v default \
|
||||||
|
--max-file-size-mb 50
|
||||||
|
```
|
||||||
|
|
||||||
|
## Docker Deployment
|
||||||
|
|
||||||
|
### Long-Running Sync
|
||||||
|
|
||||||
|
Run as a daemon for continuous synchronization:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker run -d \
|
||||||
|
--name vaultlink-sync \
|
||||||
|
--restart unless-stopped \
|
||||||
|
-v $(pwd)/vault:/vault \
|
||||||
|
ghcr.io/schmelczer/vault-link-cli:latest \
|
||||||
|
-l /vault \
|
||||||
|
-r wss://sync.example.com \
|
||||||
|
-t your-token \
|
||||||
|
-v default
|
||||||
|
```
|
||||||
|
|
||||||
|
Monitor logs:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker logs -f vaultlink-sync
|
||||||
|
```
|
||||||
|
|
||||||
|
### Health Monitoring
|
||||||
|
|
||||||
|
The Docker image includes built-in health checks:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Check health status
|
||||||
|
docker ps
|
||||||
|
|
||||||
|
# View detailed health info
|
||||||
|
docker inspect --format='{{json .State.Health}}' vaultlink-sync | jq
|
||||||
|
```
|
||||||
|
|
||||||
|
Health check verifies:
|
||||||
|
- Health file exists
|
||||||
|
- Status updated within last 30 seconds
|
||||||
|
- WebSocket connection is active
|
||||||
|
|
||||||
|
Configure custom health check:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
services:
|
||||||
|
vaultlink-cli:
|
||||||
|
image: ghcr.io/schmelczer/vault-link-cli:latest
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "node", "/app/healthcheck.js"]
|
||||||
|
interval: 15s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 5
|
||||||
|
start_period: 20s
|
||||||
|
```
|
||||||
|
|
||||||
|
### Read-Only Vault
|
||||||
|
|
||||||
|
Mount vault as read-only to prevent local changes:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker run -d \
|
||||||
|
-v $(pwd)/vault:/vault:ro \
|
||||||
|
ghcr.io/schmelczer/vault-link-cli:latest \
|
||||||
|
-l /vault \
|
||||||
|
-r wss://sync.example.com \
|
||||||
|
-t token \
|
||||||
|
-v default
|
||||||
|
```
|
||||||
|
|
||||||
|
::: warning
|
||||||
|
The CLI needs write access to create `.vaultlink` metadata directory. Mount as read-write or provide a separate writeable directory.
|
||||||
|
:::
|
||||||
|
|
||||||
|
## How It Works
|
||||||
|
|
||||||
|
### Initial Sync
|
||||||
|
|
||||||
|
On startup:
|
||||||
|
|
||||||
|
1. Creates `.vaultlink/` directory for metadata
|
||||||
|
2. Scans local filesystem
|
||||||
|
3. Uploads all local files to server
|
||||||
|
4. Downloads files from server not present locally
|
||||||
|
5. Resolves conflicts using operational transformation
|
||||||
|
|
||||||
|
### Real-Time Synchronization
|
||||||
|
|
||||||
|
After initial sync:
|
||||||
|
|
||||||
|
1. Watches filesystem for changes using `fs.watch`
|
||||||
|
2. Uploads changed files immediately
|
||||||
|
3. Receives real-time updates from server via WebSocket
|
||||||
|
4. Handles bidirectional sync automatically
|
||||||
|
|
||||||
|
### Graceful Shutdown
|
||||||
|
|
||||||
|
On SIGINT (Ctrl+C) or SIGTERM:
|
||||||
|
|
||||||
|
1. Completes pending uploads
|
||||||
|
2. Closes WebSocket connection cleanly
|
||||||
|
3. Flushes metadata to disk
|
||||||
|
4. Exits gracefully
|
||||||
|
|
||||||
|
## Use Cases
|
||||||
|
|
||||||
|
### Automated Backups
|
||||||
|
|
||||||
|
Continuously backup vaults to a remote server:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker run -d \
|
||||||
|
--name vault-backup \
|
||||||
|
-v /important/notes:/vault:ro \
|
||||||
|
ghcr.io/schmelczer/vault-link-cli:latest \
|
||||||
|
-l /vault -r wss://backup.example.com -t backup-token -v backups
|
||||||
|
```
|
||||||
|
|
||||||
|
### CI/CD Documentation
|
||||||
|
|
||||||
|
Sync documentation in automated pipelines:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# In your CI pipeline
|
||||||
|
docker run \
|
||||||
|
-v $(pwd)/docs:/vault \
|
||||||
|
ghcr.io/schmelczer/vault-link-cli:latest \
|
||||||
|
-l /vault -r wss://docs.example.com -t ci-token -v prod-docs
|
||||||
|
```
|
||||||
|
|
||||||
|
### Multi-Location Sync
|
||||||
|
|
||||||
|
Sync between different geographic locations:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Location A
|
||||||
|
vaultlink -l /data/vault -r wss://hub.example.com -t token -v shared
|
||||||
|
|
||||||
|
# Location B
|
||||||
|
vaultlink -l /backup/vault -r wss://hub.example.com -t token -v shared
|
||||||
|
```
|
||||||
|
|
||||||
|
### Development Environment
|
||||||
|
|
||||||
|
Keep documentation in sync across dev environments:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# In docker-compose.yml for your dev stack
|
||||||
|
services:
|
||||||
|
docs-sync:
|
||||||
|
image: ghcr.io/schmelczer/vault-link-cli:latest
|
||||||
|
volumes:
|
||||||
|
- ./docs:/vault
|
||||||
|
command: ["-l", "/vault", "-r", "wss://docs-server", "-t", "dev-token", "-v", "dev"]
|
||||||
|
```
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
### Client won't connect
|
||||||
|
|
||||||
|
**Check server accessibility**:
|
||||||
|
```bash
|
||||||
|
curl https://sync.example.com/vaults/test/ping
|
||||||
|
```
|
||||||
|
|
||||||
|
**Verify WebSocket protocol**:
|
||||||
|
- Use `ws://` for HTTP servers
|
||||||
|
- Use `wss://` for HTTPS servers
|
||||||
|
|
||||||
|
**Check authentication**:
|
||||||
|
- Token must match server config
|
||||||
|
- User must have access to the vault
|
||||||
|
|
||||||
|
### Permission errors
|
||||||
|
|
||||||
|
**Docker volume permissions**:
|
||||||
|
```bash
|
||||||
|
# Ensure directory is writable
|
||||||
|
chmod 755 /path/to/vault
|
||||||
|
|
||||||
|
# Check Docker user ID
|
||||||
|
docker run --rm ghcr.io/schmelczer/vault-link-cli:latest id
|
||||||
|
```
|
||||||
|
|
||||||
|
**SELinux issues**:
|
||||||
|
```bash
|
||||||
|
# Add :z flag to volume mount
|
||||||
|
docker run -v /path/to/vault:/vault:z ...
|
||||||
|
```
|
||||||
|
|
||||||
|
### Files not syncing
|
||||||
|
|
||||||
|
**Check ignore patterns**:
|
||||||
|
- View logs to see which files are skipped
|
||||||
|
- Ensure patterns don't match unintentionally
|
||||||
|
|
||||||
|
**File size limits**:
|
||||||
|
- Check `--max-file-size-mb` setting
|
||||||
|
- Large files are skipped with a warning
|
||||||
|
|
||||||
|
**Check metadata**:
|
||||||
|
```bash
|
||||||
|
# View sync metadata
|
||||||
|
cat /path/to/vault/.vaultlink/metadata.json
|
||||||
|
```
|
||||||
|
|
||||||
|
### High memory usage
|
||||||
|
|
||||||
|
**Reduce concurrency**:
|
||||||
|
```bash
|
||||||
|
--sync-concurrency 1
|
||||||
|
```
|
||||||
|
|
||||||
|
**Limit file sizes**:
|
||||||
|
```bash
|
||||||
|
--max-file-size-mb 5
|
||||||
|
```
|
||||||
|
|
||||||
|
**Check vault size**:
|
||||||
|
- Very large vaults may need more resources
|
||||||
|
- Consider splitting into multiple vaults
|
||||||
|
|
||||||
|
### Connection keeps dropping
|
||||||
|
|
||||||
|
**Increase retry interval**:
|
||||||
|
```bash
|
||||||
|
--websocket-retry-interval-ms 5000
|
||||||
|
```
|
||||||
|
|
||||||
|
**Check network stability**:
|
||||||
|
```bash
|
||||||
|
# Monitor connection
|
||||||
|
docker logs -f vaultlink-sync | grep -i websocket
|
||||||
|
```
|
||||||
|
|
||||||
|
**Server timeout settings**:
|
||||||
|
- Verify reverse proxy WebSocket timeout
|
||||||
|
- Check server `response_timeout_seconds`
|
||||||
|
|
||||||
|
## Advanced Usage
|
||||||
|
|
||||||
|
### Custom Healthcheck Script
|
||||||
|
|
||||||
|
Create your own health monitoring:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
#!/bin/bash
|
||||||
|
HEALTH_FILE="/tmp/vaultlink-health.json"
|
||||||
|
|
||||||
|
if [ ! -f "$HEALTH_FILE" ]; then
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check file is recent (within 60 seconds)
|
||||||
|
if [ $(( $(date +%s) - $(stat -c %Y "$HEALTH_FILE") )) -gt 60 ]; then
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check WebSocket is connected
|
||||||
|
if ! jq -e '.connected == true' "$HEALTH_FILE" > /dev/null; then
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
exit 0
|
||||||
|
```
|
||||||
|
|
||||||
|
### Automated Recovery
|
||||||
|
|
||||||
|
Restart on failure with exponential backoff:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
#!/bin/bash
|
||||||
|
RETRY_DELAY=5
|
||||||
|
|
||||||
|
while true; do
|
||||||
|
vaultlink -l /vault -r wss://server -t token -v default
|
||||||
|
|
||||||
|
echo "Client exited, restarting in ${RETRY_DELAY}s..."
|
||||||
|
sleep $RETRY_DELAY
|
||||||
|
|
||||||
|
# Exponential backoff up to 5 minutes
|
||||||
|
RETRY_DELAY=$((RETRY_DELAY * 2))
|
||||||
|
if [ $RETRY_DELAY -gt 300 ]; then
|
||||||
|
RETRY_DELAY=300
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
```
|
||||||
|
|
||||||
|
### Integration with systemd
|
||||||
|
|
||||||
|
Create `/etc/systemd/system/vaultlink-cli.service`:
|
||||||
|
|
||||||
|
```ini
|
||||||
|
[Unit]
|
||||||
|
Description=VaultLink CLI Sync
|
||||||
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
Restart=always
|
||||||
|
RestartSec=10
|
||||||
|
Environment="VAULTLINK_LOCAL_PATH=/data/vault"
|
||||||
|
Environment="VAULTLINK_REMOTE_URI=wss://sync.example.com"
|
||||||
|
Environment="VAULTLINK_TOKEN=your-token"
|
||||||
|
Environment="VAULTLINK_VAULT_NAME=default"
|
||||||
|
ExecStart=/usr/local/bin/vaultlink
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
|
```
|
||||||
|
|
||||||
|
Enable and start:
|
||||||
|
```bash
|
||||||
|
sudo systemctl daemon-reload
|
||||||
|
sudo systemctl enable vaultlink-cli
|
||||||
|
sudo systemctl start vaultlink-cli
|
||||||
|
```
|
||||||
|
|
||||||
|
## Next Steps
|
||||||
|
|
||||||
|
- [Configure server authentication →](/config/authentication)
|
||||||
|
- [Learn about the sync algorithm →](/architecture/sync-algorithm)
|
||||||
|
- [Set up Obsidian plugin →](/guide/obsidian-plugin)
|
||||||
185
docs/guide/getting-started.md
Normal file
185
docs/guide/getting-started.md
Normal file
|
|
@ -0,0 +1,185 @@
|
||||||
|
# Getting Started
|
||||||
|
|
||||||
|
This guide will walk you through setting up VaultLink from scratch. You'll have a working sync server and connected client in under 10 minutes.
|
||||||
|
|
||||||
|
## Prerequisites
|
||||||
|
|
||||||
|
- Docker installed (recommended) or Rust toolchain for building from source
|
||||||
|
- Basic familiarity with command line
|
||||||
|
- A server or machine to host the sync server (can be localhost for testing)
|
||||||
|
|
||||||
|
## Quick Start
|
||||||
|
|
||||||
|
### Step 1: Deploy the Sync Server
|
||||||
|
|
||||||
|
The fastest way to get started is with Docker:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Create a directory for server data
|
||||||
|
mkdir -p ~/vaultlink-data
|
||||||
|
cd ~/vaultlink-data
|
||||||
|
|
||||||
|
# Create a basic configuration file
|
||||||
|
cat > config.yml << 'EOF'
|
||||||
|
database:
|
||||||
|
databases_directory_path: databases
|
||||||
|
max_connections_per_vault: 12
|
||||||
|
cursor_timeout_seconds: 60
|
||||||
|
server:
|
||||||
|
host: 0.0.0.0
|
||||||
|
port: 3000
|
||||||
|
max_body_size_mb: 512
|
||||||
|
max_clients_per_vault: 256
|
||||||
|
response_timeout_seconds: 60
|
||||||
|
users:
|
||||||
|
user_configs:
|
||||||
|
- name: admin
|
||||||
|
token: change-this-to-a-secure-random-token
|
||||||
|
vault_access:
|
||||||
|
type: allow_access_to_all
|
||||||
|
logging:
|
||||||
|
log_directory: logs
|
||||||
|
log_rotation: 7days
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# Run the server
|
||||||
|
docker run -d \
|
||||||
|
--name vaultlink-server \
|
||||||
|
--restart unless-stopped \
|
||||||
|
-p 3000:3000 \
|
||||||
|
-v $(pwd):/data \
|
||||||
|
ghcr.io/schmelczer/vault-link-server:latest \
|
||||||
|
/app/sync_server /data/config.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
::: warning
|
||||||
|
Change the token in `config.yml` to a secure random value before deploying to production!
|
||||||
|
:::
|
||||||
|
|
||||||
|
Verify the server is running:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl http://localhost:3000/vaults/test/ping
|
||||||
|
```
|
||||||
|
|
||||||
|
You should see: `pong`
|
||||||
|
|
||||||
|
### Step 2: Choose Your Client
|
||||||
|
|
||||||
|
You can connect to VaultLink using either the Obsidian plugin or the standalone CLI client.
|
||||||
|
|
||||||
|
#### Option A: Obsidian Plugin
|
||||||
|
|
||||||
|
1. Open Obsidian Settings → Community Plugins
|
||||||
|
2. Browse community plugins and search for "VaultLink"
|
||||||
|
3. Install and enable the plugin
|
||||||
|
4. Configure the plugin:
|
||||||
|
- **Server URL**: `ws://localhost:3000` (or your server address)
|
||||||
|
- **Token**: The token from your `config.yml`
|
||||||
|
- **Vault Name**: `default` (or any name you choose)
|
||||||
|
|
||||||
|
[Read the full Obsidian plugin guide →](/guide/obsidian-plugin)
|
||||||
|
|
||||||
|
#### Option B: CLI Client
|
||||||
|
|
||||||
|
Perfect for syncing vaults without Obsidian:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker run -d \
|
||||||
|
--name vaultlink-cli \
|
||||||
|
--restart unless-stopped \
|
||||||
|
-v /path/to/your/vault:/vault \
|
||||||
|
ghcr.io/schmelczer/vault-link-cli:latest \
|
||||||
|
-l /vault \
|
||||||
|
-r ws://localhost:3000 \
|
||||||
|
-t change-this-to-a-secure-random-token \
|
||||||
|
-v default
|
||||||
|
```
|
||||||
|
|
||||||
|
Replace `/path/to/your/vault` with the directory containing your files.
|
||||||
|
|
||||||
|
[Read the full CLI client guide →](/guide/cli-client)
|
||||||
|
|
||||||
|
## Next Steps
|
||||||
|
|
||||||
|
### Production Deployment
|
||||||
|
|
||||||
|
For production use, you should:
|
||||||
|
|
||||||
|
1. **Use HTTPS/WSS**: Put the sync server behind a reverse proxy with SSL
|
||||||
|
2. **Secure tokens**: Generate cryptographically random tokens
|
||||||
|
3. **Configure backups**: Back up the SQLite databases regularly
|
||||||
|
4. **Set up monitoring**: Use Docker health checks and logging
|
||||||
|
|
||||||
|
[Learn about production deployment →](/guide/server-setup#production-deployment)
|
||||||
|
|
||||||
|
### Multiple Users
|
||||||
|
|
||||||
|
To add more users or restrict vault access:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
users:
|
||||||
|
user_configs:
|
||||||
|
- name: alice
|
||||||
|
token: alice-secure-token
|
||||||
|
vault_access:
|
||||||
|
type: allow_list
|
||||||
|
allowed:
|
||||||
|
- personal
|
||||||
|
- shared
|
||||||
|
- name: bob
|
||||||
|
token: bob-secure-token
|
||||||
|
vault_access:
|
||||||
|
type: allow_list
|
||||||
|
allowed:
|
||||||
|
- shared
|
||||||
|
```
|
||||||
|
|
||||||
|
[Learn about authentication configuration →](/config/authentication)
|
||||||
|
|
||||||
|
### Advanced Configuration
|
||||||
|
|
||||||
|
Explore advanced server options:
|
||||||
|
|
||||||
|
- Database tuning for large vaults
|
||||||
|
- Rate limiting and connection limits
|
||||||
|
- Custom logging and log rotation
|
||||||
|
- Multi-vault setups
|
||||||
|
|
||||||
|
[View configuration reference →](/config/server)
|
||||||
|
|
||||||
|
## Architecture Overview
|
||||||
|
|
||||||
|
Want to understand how VaultLink works under the hood?
|
||||||
|
|
||||||
|
[Read the architecture documentation →](/architecture/)
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
### Server won't start
|
||||||
|
|
||||||
|
Check Docker logs:
|
||||||
|
```bash
|
||||||
|
docker logs vaultlink-server
|
||||||
|
```
|
||||||
|
|
||||||
|
Common issues:
|
||||||
|
- Port 3000 already in use: Change the port mapping `-p 3001:3000`
|
||||||
|
- Config file errors: Validate YAML syntax
|
||||||
|
- Permission issues: Ensure the volume mount is writable
|
||||||
|
|
||||||
|
### Client can't connect
|
||||||
|
|
||||||
|
1. Verify server is accessible: `curl http://your-server:3000/vaults/test/ping`
|
||||||
|
2. Check WebSocket connectivity (browser dev tools or wscat)
|
||||||
|
3. Verify token matches between client and server config
|
||||||
|
4. Check firewall rules allow port 3000
|
||||||
|
|
||||||
|
### Files not syncing
|
||||||
|
|
||||||
|
1. Check client logs for errors
|
||||||
|
2. Verify vault name matches on both server and client
|
||||||
|
3. Ensure user has access to the vault (check server config)
|
||||||
|
4. Check for file size limits (default 10MB for CLI)
|
||||||
|
|
||||||
|
For more help, [open an issue on GitHub](https://github.com/schmelczer/vault-link/issues).
|
||||||
262
docs/guide/obsidian-plugin.md
Normal file
262
docs/guide/obsidian-plugin.md
Normal file
|
|
@ -0,0 +1,262 @@
|
||||||
|
# Obsidian Plugin
|
||||||
|
|
||||||
|
The VaultLink Obsidian plugin provides native real-time synchronization directly within Obsidian.
|
||||||
|
|
||||||
|
## Installation
|
||||||
|
|
||||||
|
### From Obsidian Community Plugins
|
||||||
|
|
||||||
|
1. Open Obsidian Settings
|
||||||
|
2. Navigate to **Community Plugins**
|
||||||
|
3. Click **Browse** and search for "VaultLink"
|
||||||
|
4. Click **Install**
|
||||||
|
5. Enable the plugin
|
||||||
|
|
||||||
|
### Manual Installation
|
||||||
|
|
||||||
|
1. Download the latest release from [GitHub Releases](https://github.com/schmelczer/vault-link/releases)
|
||||||
|
2. Extract `main.js`, `manifest.json`, and `styles.css`
|
||||||
|
3. Copy to `.obsidian/plugins/vault-link/` in your vault
|
||||||
|
4. Reload Obsidian
|
||||||
|
5. Enable VaultLink in Community Plugins settings
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
After installation, configure the plugin in **Settings → VaultLink**.
|
||||||
|
|
||||||
|
### Required Settings
|
||||||
|
|
||||||
|
#### Server URL
|
||||||
|
The WebSocket URL of your sync server.
|
||||||
|
|
||||||
|
- **Development/Local**: `ws://localhost:3000`
|
||||||
|
- **Production (SSL)**: `wss://sync.example.com`
|
||||||
|
|
||||||
|
::: tip
|
||||||
|
Use `ws://` for unencrypted connections and `wss://` for SSL connections (production).
|
||||||
|
:::
|
||||||
|
|
||||||
|
#### Authentication Token
|
||||||
|
Your authentication token from the server's `config.yml`.
|
||||||
|
|
||||||
|
Generate a secure token:
|
||||||
|
```bash
|
||||||
|
openssl rand -hex 32
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Vault Name
|
||||||
|
The name of the vault on the server. Can be any string.
|
||||||
|
|
||||||
|
Multiple Obsidian vaults can sync to the same server vault name (for shared vaults), or use unique names for separate vaults.
|
||||||
|
|
||||||
|
### Optional Settings
|
||||||
|
|
||||||
|
#### Sync Concurrency
|
||||||
|
Number of files to sync simultaneously.
|
||||||
|
- **Default**: 1
|
||||||
|
- **Range**: 1-10
|
||||||
|
- Higher values = faster initial sync, more resource usage
|
||||||
|
|
||||||
|
#### Max File Size
|
||||||
|
Maximum file size to sync (in MB).
|
||||||
|
- **Default**: 10
|
||||||
|
- Files larger than this are skipped
|
||||||
|
|
||||||
|
#### Ignore Patterns
|
||||||
|
Glob patterns for files to exclude from sync.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
- `*.tmp` - Ignore temporary files
|
||||||
|
- `.trash/**` - Ignore trash folder
|
||||||
|
- `private/**` - Ignore private directory
|
||||||
|
|
||||||
|
#### WebSocket Retry Interval
|
||||||
|
Milliseconds between reconnection attempts when disconnected.
|
||||||
|
- **Default**: 3500ms
|
||||||
|
- Increase for flaky networks to avoid connection spam
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
### Initial Sync
|
||||||
|
|
||||||
|
When first connecting:
|
||||||
|
|
||||||
|
1. The plugin uploads all local files to the server
|
||||||
|
2. Downloads any missing files from the server
|
||||||
|
3. Resolves any conflicts using operational transformation
|
||||||
|
4. Begins real-time synchronization
|
||||||
|
|
||||||
|
Initial sync time depends on vault size and `sync_concurrency` setting.
|
||||||
|
|
||||||
|
### Real-Time Sync
|
||||||
|
|
||||||
|
Once connected:
|
||||||
|
|
||||||
|
- **File changes**: Automatically synced when saved
|
||||||
|
- **File creation**: New files immediately uploaded
|
||||||
|
- **File deletion**: Deletions propagated to other clients
|
||||||
|
- **File renames**: Tracked and synchronized
|
||||||
|
|
||||||
|
The plugin watches your vault filesystem and syncs changes in real-time via WebSocket.
|
||||||
|
|
||||||
|
### Status Indicators
|
||||||
|
|
||||||
|
The plugin provides visual feedback:
|
||||||
|
|
||||||
|
- **Connected**: Green status in settings
|
||||||
|
- **Syncing**: Progress indicator during uploads
|
||||||
|
- **Disconnected**: Red status, automatic reconnection attempts
|
||||||
|
- **Error**: Error message in settings and console
|
||||||
|
|
||||||
|
Check the Obsidian console (Ctrl+Shift+I / Cmd+Option+I) for detailed logs.
|
||||||
|
|
||||||
|
## Features
|
||||||
|
|
||||||
|
### Automatic Conflict Resolution
|
||||||
|
|
||||||
|
When multiple users edit the same file simultaneously, operational transformation merges changes automatically:
|
||||||
|
|
||||||
|
- All edits are preserved
|
||||||
|
- No manual conflict resolution required
|
||||||
|
- Changes appear in real-time as others type
|
||||||
|
|
||||||
|
### Mobile Support
|
||||||
|
|
||||||
|
VaultLink works on Obsidian mobile (iOS and Android):
|
||||||
|
|
||||||
|
- Same configuration as desktop
|
||||||
|
- Real-time sync across all devices
|
||||||
|
- Handle network changes gracefully
|
||||||
|
|
||||||
|
::: warning
|
||||||
|
Ensure your sync server is accessible from mobile networks (use WSS with a public domain or VPN).
|
||||||
|
:::
|
||||||
|
|
||||||
|
### Offline Support
|
||||||
|
|
||||||
|
The plugin handles offline scenarios:
|
||||||
|
|
||||||
|
- Continue working when disconnected
|
||||||
|
- Changes queue locally
|
||||||
|
- Automatic sync when connection restored
|
||||||
|
- Conflict resolution if others edited the same files
|
||||||
|
|
||||||
|
## Collaboration Workflows
|
||||||
|
|
||||||
|
### Personal Multi-Device Sync
|
||||||
|
|
||||||
|
Sync the same vault across devices:
|
||||||
|
|
||||||
|
1. Configure each Obsidian instance with the same vault name
|
||||||
|
2. Use the same authentication token
|
||||||
|
3. All devices stay in sync automatically
|
||||||
|
|
||||||
|
### Team Shared Vault
|
||||||
|
|
||||||
|
Multiple users collaborating:
|
||||||
|
|
||||||
|
1. Each user has their own token (configured in server `config.yml`)
|
||||||
|
2. All users connect to the same vault name
|
||||||
|
3. Real-time collaborative editing with automatic conflict resolution
|
||||||
|
|
||||||
|
### Selective Sharing
|
||||||
|
|
||||||
|
Share specific folders while keeping others private:
|
||||||
|
|
||||||
|
1. Use different vault names for shared vs. private content
|
||||||
|
2. Configure access control on the server per vault
|
||||||
|
3. Use ignore patterns to exclude sensitive directories
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
### Plugin won't connect
|
||||||
|
|
||||||
|
1. **Verify server is running**:
|
||||||
|
```bash
|
||||||
|
curl http://your-server:3000/vaults/test/ping
|
||||||
|
```
|
||||||
|
Should return `pong`
|
||||||
|
|
||||||
|
2. **Check URL format**:
|
||||||
|
- Local: `ws://localhost:3000`
|
||||||
|
- Remote (SSL): `wss://sync.example.com`
|
||||||
|
- Don't include `/vault/name` in the URL
|
||||||
|
|
||||||
|
3. **Verify token**:
|
||||||
|
- Must match server config exactly
|
||||||
|
- No extra spaces or quotes
|
||||||
|
- Check server logs for authentication errors
|
||||||
|
|
||||||
|
4. **Check firewall**:
|
||||||
|
- Ensure port is accessible from your network
|
||||||
|
- For mobile, server must be publicly accessible or use VPN
|
||||||
|
|
||||||
|
### Files not syncing
|
||||||
|
|
||||||
|
1. **Check ignore patterns**: File may match an exclusion pattern
|
||||||
|
2. **File size**: Check if file exceeds `max_file_size_mb`
|
||||||
|
3. **Permissions**: Ensure vault directory is readable/writable
|
||||||
|
4. **Console errors**: Open dev tools (Ctrl+Shift+I) and check console
|
||||||
|
|
||||||
|
### Slow initial sync
|
||||||
|
|
||||||
|
1. **Increase concurrency**: Set `sync_concurrency` higher (e.g., 5)
|
||||||
|
2. **Network speed**: Check internet connection
|
||||||
|
3. **Server resources**: Ensure server isn't overloaded
|
||||||
|
4. **Large files**: Consider increasing timeout settings
|
||||||
|
|
||||||
|
### Conflicts not resolving
|
||||||
|
|
||||||
|
Operational transformation should handle conflicts automatically. If issues persist:
|
||||||
|
|
||||||
|
1. Check console for sync errors
|
||||||
|
2. Verify both clients are connected
|
||||||
|
3. Check server logs for processing errors
|
||||||
|
4. Ensure files are text-based (binary files may not merge well)
|
||||||
|
|
||||||
|
### High CPU/Memory usage
|
||||||
|
|
||||||
|
1. **Reduce concurrency**: Lower `sync_concurrency`
|
||||||
|
2. **Add ignore patterns**: Exclude unnecessary files
|
||||||
|
3. **File watchers**: Large vaults may trigger many filesystem events
|
||||||
|
4. **Check for sync loops**: Ensure no circular dependencies
|
||||||
|
|
||||||
|
## Advanced Configuration
|
||||||
|
|
||||||
|
### Multiple Vaults
|
||||||
|
|
||||||
|
To sync multiple Obsidian vaults to different server vaults:
|
||||||
|
|
||||||
|
1. Each Obsidian vault has its own VaultLink plugin configuration
|
||||||
|
2. Use different vault names for each
|
||||||
|
3. Can use the same or different tokens (depending on access control)
|
||||||
|
|
||||||
|
### Custom Sync Patterns
|
||||||
|
|
||||||
|
Combine ignore patterns for fine-grained control:
|
||||||
|
|
||||||
|
```
|
||||||
|
# Ignore patterns
|
||||||
|
*.tmp
|
||||||
|
*.bak
|
||||||
|
.DS_Store
|
||||||
|
.trash/**
|
||||||
|
private/**
|
||||||
|
drafts/**/*.draft.md
|
||||||
|
```
|
||||||
|
|
||||||
|
### Development/Testing
|
||||||
|
|
||||||
|
For plugin development:
|
||||||
|
|
||||||
|
1. Clone the repository
|
||||||
|
2. `cd frontend && npm install`
|
||||||
|
3. `npm run dev` to build in watch mode
|
||||||
|
4. Plugin rebuilds automatically on changes
|
||||||
|
5. Reload Obsidian to test changes
|
||||||
|
|
||||||
|
## Next Steps
|
||||||
|
|
||||||
|
- [Learn about the sync algorithm →](/architecture/sync-algorithm)
|
||||||
|
- [Configure the server →](/config/server)
|
||||||
|
- [Set up the CLI client →](/guide/cli-client)
|
||||||
370
docs/guide/server-setup.md
Normal file
370
docs/guide/server-setup.md
Normal file
|
|
@ -0,0 +1,370 @@
|
||||||
|
# Server Setup
|
||||||
|
|
||||||
|
This guide covers deploying the VaultLink sync server in various environments, from local development to production infrastructure.
|
||||||
|
|
||||||
|
## Deployment Options
|
||||||
|
|
||||||
|
### Docker (Recommended)
|
||||||
|
|
||||||
|
Docker provides the easiest deployment path with built-in health checks and minimal dependencies.
|
||||||
|
|
||||||
|
#### Basic Docker Deployment
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Pull the latest image
|
||||||
|
docker pull ghcr.io/schmelczer/vault-link-server:latest
|
||||||
|
|
||||||
|
# Create data directory
|
||||||
|
mkdir -p ~/vaultlink-data
|
||||||
|
|
||||||
|
# Create config.yml (see Configuration section below)
|
||||||
|
|
||||||
|
# Run the container
|
||||||
|
docker run -d \
|
||||||
|
--name vaultlink-server \
|
||||||
|
--restart unless-stopped \
|
||||||
|
-p 3000:3000 \
|
||||||
|
-v ~/vaultlink-data:/data \
|
||||||
|
ghcr.io/schmelczer/vault-link-server:latest \
|
||||||
|
/app/sync_server /data/config.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Docker Compose
|
||||||
|
|
||||||
|
Create `docker-compose.yml`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
services:
|
||||||
|
vaultlink-server:
|
||||||
|
image: ghcr.io/schmelczer/vault-link-server:latest
|
||||||
|
container_name: vaultlink-server
|
||||||
|
restart: unless-stopped
|
||||||
|
ports:
|
||||||
|
- "3000:3000"
|
||||||
|
volumes:
|
||||||
|
- ./data:/data
|
||||||
|
command: ["/app/sync_server", "/data/config.yml"]
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "curl", "-f", "http://localhost:3000/vaults/fake/ping"]
|
||||||
|
interval: 30s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 3
|
||||||
|
start_period: 10s
|
||||||
|
```
|
||||||
|
|
||||||
|
Start the server:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
### Binary Installation
|
||||||
|
|
||||||
|
Download pre-built binaries from [GitHub Releases](https://github.com/schmelczer/vault-link/releases).
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Download the binary for your platform
|
||||||
|
wget https://github.com/schmelczer/vault-link/releases/latest/download/sync_server-linux-x86_64
|
||||||
|
|
||||||
|
# Make executable
|
||||||
|
chmod +x sync_server-linux-x86_64
|
||||||
|
|
||||||
|
# Run the server
|
||||||
|
./sync_server-linux-x86_64 config.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
### Build from Source
|
||||||
|
|
||||||
|
Requirements:
|
||||||
|
- Rust 1.89.0 or later
|
||||||
|
- SQLite development headers
|
||||||
|
- SQLx CLI
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Clone the repository
|
||||||
|
git clone https://github.com/schmelczer/vault-link.git
|
||||||
|
cd vault-link/sync-server
|
||||||
|
|
||||||
|
# Install SQLx CLI
|
||||||
|
cargo install sqlx-cli
|
||||||
|
|
||||||
|
# Set up the database
|
||||||
|
sqlx database create --database-url sqlite://db.sqlite3
|
||||||
|
sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3
|
||||||
|
cargo sqlx prepare --workspace
|
||||||
|
|
||||||
|
# Build in release mode
|
||||||
|
cargo build --release
|
||||||
|
|
||||||
|
# Run the server
|
||||||
|
./target/release/sync_server config.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
Create a `config.yml` file with your server configuration:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
database:
|
||||||
|
databases_directory_path: databases
|
||||||
|
max_connections_per_vault: 12
|
||||||
|
cursor_timeout_seconds: 60
|
||||||
|
|
||||||
|
server:
|
||||||
|
host: 0.0.0.0
|
||||||
|
port: 3000
|
||||||
|
max_body_size_mb: 512
|
||||||
|
max_clients_per_vault: 256
|
||||||
|
response_timeout_seconds: 60
|
||||||
|
|
||||||
|
users:
|
||||||
|
user_configs:
|
||||||
|
- name: admin
|
||||||
|
token: your-secure-random-token-here
|
||||||
|
vault_access:
|
||||||
|
type: allow_access_to_all
|
||||||
|
|
||||||
|
logging:
|
||||||
|
log_directory: logs
|
||||||
|
log_rotation: 7days
|
||||||
|
```
|
||||||
|
|
||||||
|
### Configuration Fields
|
||||||
|
|
||||||
|
#### Database
|
||||||
|
|
||||||
|
- `databases_directory_path`: Directory for SQLite databases (one per vault)
|
||||||
|
- `max_connections_per_vault`: Maximum concurrent database connections
|
||||||
|
- `cursor_timeout_seconds`: How long to keep database cursors alive
|
||||||
|
|
||||||
|
#### Server
|
||||||
|
|
||||||
|
- `host`: Bind address (use `0.0.0.0` for all interfaces)
|
||||||
|
- `port`: Port to listen on (default: 3000)
|
||||||
|
- `max_body_size_mb`: Maximum upload size
|
||||||
|
- `max_clients_per_vault`: Concurrent client limit per vault
|
||||||
|
- `response_timeout_seconds`: Request timeout
|
||||||
|
|
||||||
|
#### Users
|
||||||
|
|
||||||
|
See [Authentication Configuration →](/config/authentication) for detailed user setup.
|
||||||
|
|
||||||
|
#### Logging
|
||||||
|
|
||||||
|
- `log_directory`: Where to store log files
|
||||||
|
- `log_rotation`: How often to rotate logs (e.g., `7days`, `24hours`)
|
||||||
|
|
||||||
|
## Production Deployment
|
||||||
|
|
||||||
|
### SSL/TLS with Reverse Proxy
|
||||||
|
|
||||||
|
VaultLink doesn't handle SSL directly. Use a reverse proxy like Nginx or Caddy.
|
||||||
|
|
||||||
|
#### Nginx Configuration
|
||||||
|
|
||||||
|
```nginx
|
||||||
|
upstream vaultlink {
|
||||||
|
server localhost:3000;
|
||||||
|
}
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 443 ssl http2;
|
||||||
|
server_name sync.example.com;
|
||||||
|
|
||||||
|
ssl_certificate /path/to/cert.pem;
|
||||||
|
ssl_certificate_key /path/to/key.pem;
|
||||||
|
|
||||||
|
location / {
|
||||||
|
proxy_pass http://vaultlink;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Upgrade $http_upgrade;
|
||||||
|
proxy_set_header Connection "upgrade";
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
|
||||||
|
# WebSocket specific
|
||||||
|
proxy_read_timeout 3600s;
|
||||||
|
proxy_send_timeout 3600s;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Reload Nginx:
|
||||||
|
```bash
|
||||||
|
sudo nginx -t
|
||||||
|
sudo systemctl reload nginx
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Caddy Configuration
|
||||||
|
|
||||||
|
Caddy handles SSL automatically:
|
||||||
|
|
||||||
|
```caddy
|
||||||
|
sync.example.com {
|
||||||
|
reverse_proxy localhost:3000
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Start Caddy:
|
||||||
|
```bash
|
||||||
|
caddy run --config Caddyfile
|
||||||
|
```
|
||||||
|
|
||||||
|
### Systemd Service
|
||||||
|
|
||||||
|
Create `/etc/systemd/system/vaultlink.service`:
|
||||||
|
|
||||||
|
```ini
|
||||||
|
[Unit]
|
||||||
|
Description=VaultLink Sync Server
|
||||||
|
After=network.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=vaultlink
|
||||||
|
WorkingDirectory=/opt/vaultlink
|
||||||
|
ExecStart=/opt/vaultlink/sync_server /opt/vaultlink/config.yml
|
||||||
|
Restart=always
|
||||||
|
RestartSec=10
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
|
```
|
||||||
|
|
||||||
|
Enable and start:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo systemctl daemon-reload
|
||||||
|
sudo systemctl enable vaultlink
|
||||||
|
sudo systemctl start vaultlink
|
||||||
|
sudo systemctl status vaultlink
|
||||||
|
```
|
||||||
|
|
||||||
|
### Security Best Practices
|
||||||
|
|
||||||
|
1. **Use strong tokens**: Generate with `openssl rand -hex 32`
|
||||||
|
2. **Enable firewall**: Only expose port 3000 to reverse proxy
|
||||||
|
3. **Regular updates**: Keep Docker images and binaries updated
|
||||||
|
4. **Backup databases**: SQLite files in `databases_directory_path`
|
||||||
|
5. **Monitor logs**: Check log directory for errors and anomalies
|
||||||
|
6. **Limit access**: Use vault-specific access controls per user
|
||||||
|
|
||||||
|
### Backup Strategy
|
||||||
|
|
||||||
|
The SQLite databases contain all vault data and history:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Backup script
|
||||||
|
#!/bin/bash
|
||||||
|
BACKUP_DIR="/backup/vaultlink/$(date +%Y%m%d)"
|
||||||
|
DATA_DIR="/data/databases"
|
||||||
|
|
||||||
|
mkdir -p "$BACKUP_DIR"
|
||||||
|
cp -r "$DATA_DIR" "$BACKUP_DIR/"
|
||||||
|
|
||||||
|
# Keep 30 days of backups
|
||||||
|
find /backup/vaultlink -type d -mtime +30 -exec rm -rf {} +
|
||||||
|
```
|
||||||
|
|
||||||
|
Run daily via cron:
|
||||||
|
```cron
|
||||||
|
0 2 * * * /opt/vaultlink/backup.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
### Monitoring
|
||||||
|
|
||||||
|
#### Health Checks
|
||||||
|
|
||||||
|
The server exposes a ping endpoint:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl http://localhost:3000/vaults/fake/ping
|
||||||
|
# Returns: pong
|
||||||
|
```
|
||||||
|
|
||||||
|
Docker health check is built-in and checks this endpoint every 30 seconds.
|
||||||
|
|
||||||
|
#### Prometheus Metrics
|
||||||
|
|
||||||
|
For advanced monitoring, collect Docker stats or implement custom metrics.
|
||||||
|
|
||||||
|
#### Log Monitoring
|
||||||
|
|
||||||
|
Logs are written to the configured `log_directory`. Monitor for:
|
||||||
|
- Connection failures
|
||||||
|
- Authentication errors
|
||||||
|
- Database errors
|
||||||
|
- WebSocket disconnections
|
||||||
|
|
||||||
|
Example log watching:
|
||||||
|
```bash
|
||||||
|
tail -f /data/logs/*.log | grep -i error
|
||||||
|
```
|
||||||
|
|
||||||
|
## Scaling
|
||||||
|
|
||||||
|
### Horizontal Scaling
|
||||||
|
|
||||||
|
VaultLink currently uses SQLite, which limits horizontal scaling. For multiple servers:
|
||||||
|
|
||||||
|
1. Run separate instances for different vaults
|
||||||
|
2. Use load balancer with sticky sessions (same vault → same server)
|
||||||
|
3. Consider database architecture for your scale needs
|
||||||
|
|
||||||
|
### Vertical Scaling
|
||||||
|
|
||||||
|
Increase resources for the server:
|
||||||
|
- More CPU for handling concurrent connections
|
||||||
|
- More RAM for database caching
|
||||||
|
- Faster storage (SSD) for database operations
|
||||||
|
|
||||||
|
Tune configuration:
|
||||||
|
- Increase `max_clients_per_vault` for more concurrent users
|
||||||
|
- Increase `max_connections_per_vault` for database performance
|
||||||
|
- Adjust `max_body_size_mb` based on typical file sizes
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
### Server won't start
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Check Docker logs
|
||||||
|
docker logs vaultlink-server
|
||||||
|
|
||||||
|
# Common issues:
|
||||||
|
# - Port already in use: Change port mapping
|
||||||
|
# - Config syntax error: Validate YAML
|
||||||
|
# - Permission error: Check volume permissions
|
||||||
|
```
|
||||||
|
|
||||||
|
### High memory usage
|
||||||
|
|
||||||
|
- Reduce `max_connections_per_vault`
|
||||||
|
- Reduce `max_clients_per_vault`
|
||||||
|
- Check for large vaults (may need database optimization)
|
||||||
|
|
||||||
|
### Database corruption
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Verify database integrity
|
||||||
|
sqlite3 databases/your-vault.db "PRAGMA integrity_check;"
|
||||||
|
|
||||||
|
# If corrupted, restore from backup
|
||||||
|
cp /backup/databases/your-vault.db /data/databases/
|
||||||
|
```
|
||||||
|
|
||||||
|
### WebSocket connection drops
|
||||||
|
|
||||||
|
- Check reverse proxy timeout settings
|
||||||
|
- Verify firewall isn't closing connections
|
||||||
|
- Review client retry intervals
|
||||||
|
- Check server logs for errors
|
||||||
|
|
||||||
|
## Next Steps
|
||||||
|
|
||||||
|
- [Configure authentication and access control →](/config/authentication)
|
||||||
|
- [Set up Obsidian plugin →](/guide/obsidian-plugin)
|
||||||
|
- [Deploy CLI client →](/guide/cli-client)
|
||||||
|
- [Understand the architecture →](/architecture/)
|
||||||
115
docs/guide/what-is-vaultlink.md
Normal file
115
docs/guide/what-is-vaultlink.md
Normal file
|
|
@ -0,0 +1,115 @@
|
||||||
|
# What is VaultLink?
|
||||||
|
|
||||||
|
VaultLink is a self-hosted real-time synchronization system for Obsidian vaults. It provides collaborative file syncing with automatic conflict resolution, designed for users who want complete control over their data.
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
VaultLink consists of three main components:
|
||||||
|
|
||||||
|
### Sync Server
|
||||||
|
|
||||||
|
A Rust-based WebSocket server that handles:
|
||||||
|
- Real-time bidirectional synchronization
|
||||||
|
- Document versioning with SQLite
|
||||||
|
- User authentication and vault access control
|
||||||
|
- Operational transformation for conflict resolution
|
||||||
|
|
||||||
|
### Obsidian Plugin
|
||||||
|
|
||||||
|
A native Obsidian plugin that:
|
||||||
|
- Integrates sync directly into your Obsidian workflow
|
||||||
|
- Provides real-time updates as you edit
|
||||||
|
- Handles file watching and automatic synchronization
|
||||||
|
- Works across desktop and mobile platforms
|
||||||
|
|
||||||
|
### CLI Client
|
||||||
|
|
||||||
|
A standalone synchronization client that:
|
||||||
|
- Syncs vaults without requiring Obsidian
|
||||||
|
- Perfect for servers, automation, or backup systems
|
||||||
|
- Provides file watching and bidirectional sync
|
||||||
|
- Runs in Docker or as a standalone binary
|
||||||
|
|
||||||
|
## Key Features
|
||||||
|
|
||||||
|
### Real-Time Synchronization
|
||||||
|
|
||||||
|
Changes are synchronized immediately via WebSocket connections. When multiple users edit the same file, operational transformation ensures all edits are preserved without conflicts.
|
||||||
|
|
||||||
|
### Self-Hosted Architecture
|
||||||
|
|
||||||
|
Run the sync server on your own infrastructure:
|
||||||
|
- Full control over data storage and access
|
||||||
|
- No dependency on third-party services
|
||||||
|
- Configurable authentication and authorization
|
||||||
|
- Deploy anywhere: cloud VPS, home server, or localhost
|
||||||
|
|
||||||
|
### Operational Transformation
|
||||||
|
|
||||||
|
VaultLink uses the `reconcile-text` library for intelligent conflict resolution:
|
||||||
|
- Simultaneous edits are automatically merged
|
||||||
|
- No manual conflict resolution required
|
||||||
|
- Preserves intent of all contributors
|
||||||
|
- Works seamlessly in the background
|
||||||
|
|
||||||
|
### Flexible Authentication
|
||||||
|
|
||||||
|
Configure user access per vault:
|
||||||
|
- Token-based authentication
|
||||||
|
- Per-user vault access control
|
||||||
|
- Allow-list or deny-list patterns
|
||||||
|
- Support for multiple users and vaults
|
||||||
|
|
||||||
|
## Use Cases
|
||||||
|
|
||||||
|
### Personal Sync
|
||||||
|
|
||||||
|
Synchronize your Obsidian vault across multiple devices:
|
||||||
|
- Laptop, desktop, and mobile in real-time
|
||||||
|
- No cloud service subscription required
|
||||||
|
- Full privacy and data control
|
||||||
|
|
||||||
|
### Team Collaboration
|
||||||
|
|
||||||
|
Share knowledge bases with teammates:
|
||||||
|
- Real-time collaborative editing
|
||||||
|
- Granular access control per vault
|
||||||
|
- Self-hosted for enterprise security requirements
|
||||||
|
|
||||||
|
### Automated Backups
|
||||||
|
|
||||||
|
Use the CLI client for automated workflows:
|
||||||
|
- Scheduled backups to remote servers
|
||||||
|
- Integration with existing backup systems
|
||||||
|
- Headless operation without Obsidian
|
||||||
|
|
||||||
|
### Development & Testing
|
||||||
|
|
||||||
|
Synchronize documentation across environments:
|
||||||
|
- Keep docs in sync with development environments
|
||||||
|
- Automated deployment of documentation
|
||||||
|
- Version control integration
|
||||||
|
|
||||||
|
## How It Works
|
||||||
|
|
||||||
|
1. **Server Setup**: Deploy the sync server on your infrastructure
|
||||||
|
2. **Authentication**: Configure users and vault access in `config.yml`
|
||||||
|
3. **Client Connection**: Connect via Obsidian plugin or CLI client
|
||||||
|
4. **Initial Sync**: Client uploads local files to server
|
||||||
|
5. **Real-Time Updates**: Changes sync bidirectionally via WebSocket
|
||||||
|
6. **Conflict Resolution**: Operational transformation handles simultaneous edits
|
||||||
|
|
||||||
|
## Technology Stack
|
||||||
|
|
||||||
|
- **Server**: Rust with Axum framework, SQLite database, WebSocket protocol
|
||||||
|
- **Frontend**: TypeScript with WebSocket client, npm workspaces
|
||||||
|
- **Sync Algorithm**: reconcile-text operational transformation library
|
||||||
|
- **Deployment**: Docker images, binary releases, or source builds
|
||||||
|
|
||||||
|
## Next Steps
|
||||||
|
|
||||||
|
Ready to get started?
|
||||||
|
|
||||||
|
- [Getting Started Guide →](/guide/getting-started)
|
||||||
|
- [Server Setup →](/guide/server-setup)
|
||||||
|
- [Architecture Overview →](/architecture/)
|
||||||
72
docs/index.md
Normal file
72
docs/index.md
Normal file
|
|
@ -0,0 +1,72 @@
|
||||||
|
---
|
||||||
|
layout: home
|
||||||
|
|
||||||
|
hero:
|
||||||
|
name: VaultLink
|
||||||
|
text: Self-Hosted Sync for Obsidian
|
||||||
|
tagline: Real-time collaborative file synchronization for your knowledge base
|
||||||
|
image:
|
||||||
|
src: /logo.svg
|
||||||
|
alt: VaultLink
|
||||||
|
actions:
|
||||||
|
- theme: brand
|
||||||
|
text: Get Started
|
||||||
|
link: /guide/getting-started
|
||||||
|
- theme: alt
|
||||||
|
text: View on GitHub
|
||||||
|
link: https://github.com/schmelczer/vault-link
|
||||||
|
|
||||||
|
features:
|
||||||
|
- icon: 🚀
|
||||||
|
title: Real-Time Synchronization
|
||||||
|
details: Operational transformation-based conflict resolution ensures your files stay in sync across devices without data loss
|
||||||
|
- icon: 🔒
|
||||||
|
title: Self-Hosted & Private
|
||||||
|
details: Run your own sync server. Your data stays on your infrastructure with full control over access and privacy
|
||||||
|
- icon: 🎯
|
||||||
|
title: Obsidian Plugin
|
||||||
|
details: Native integration with Obsidian for seamless synchronization directly within your favorite note-taking app
|
||||||
|
- icon: 🖥️
|
||||||
|
title: CLI Client
|
||||||
|
details: Sync vaults to any system using the standalone CLI client. Perfect for servers, automation, or headless setups
|
||||||
|
- icon: ⚡
|
||||||
|
title: Built for Performance
|
||||||
|
details: Rust-powered WebSocket server with SQLite backend delivers blazing-fast sync performance
|
||||||
|
- icon: 🛠️
|
||||||
|
title: Flexible Deployment
|
||||||
|
details: Deploy via Docker, binary releases, or build from source. Configure authentication and access controls to fit your needs
|
||||||
|
---
|
||||||
|
|
||||||
|
## Quick Start
|
||||||
|
|
||||||
|
Deploy the sync server:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker run -d \
|
||||||
|
-p 3000:3000 \
|
||||||
|
-v $(pwd)/data:/data \
|
||||||
|
ghcr.io/schmelczer/vault-link-server:latest \
|
||||||
|
/app/sync_server config.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
Install the Obsidian plugin or use the CLI client:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker run -v /path/to/vault:/vault \
|
||||||
|
ghcr.io/schmelczer/vault-link-cli:latest \
|
||||||
|
-l /vault -r wss://your-server.com -t your-token -v default
|
||||||
|
```
|
||||||
|
|
||||||
|
[Learn more →](/guide/getting-started)
|
||||||
|
|
||||||
|
## Why VaultLink?
|
||||||
|
|
||||||
|
VaultLink provides a complete self-hosted synchronization solution for Obsidian:
|
||||||
|
|
||||||
|
- **No third-party services**: Your data never leaves your infrastructure
|
||||||
|
- **Operational transformation**: Smart conflict resolution that preserves all changes
|
||||||
|
- **Multi-platform**: Works with Obsidian plugin or standalone CLI on any system
|
||||||
|
- **Production-ready**: Docker images, health checks, and comprehensive logging
|
||||||
|
- **Open source**: MIT licensed with active development
|
||||||
|
|
||||||
|
[Read the architecture overview →](/architecture/)
|
||||||
18
docs/package.json
Normal file
18
docs/package.json
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
{
|
||||||
|
"name": "docs",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"description": "",
|
||||||
|
"main": "index.js",
|
||||||
|
"scripts": {
|
||||||
|
"dev": "vitepress dev",
|
||||||
|
"build": "vitepress build",
|
||||||
|
"preview": "vitepress preview"
|
||||||
|
},
|
||||||
|
"keywords": [],
|
||||||
|
"author": "",
|
||||||
|
"license": "ISC",
|
||||||
|
"devDependencies": {
|
||||||
|
"vitepress": "^1.6.4",
|
||||||
|
"vue": "^3.5.24"
|
||||||
|
}
|
||||||
|
}
|
||||||
34
docs/public/logo.svg
Normal file
34
docs/public/logo.svg
Normal file
|
|
@ -0,0 +1,34 @@
|
||||||
|
<svg width="200" height="200" viewBox="0 0 200 200" xmlns="http://www.w3.org/2000/svg">
|
||||||
|
<!-- Background circle -->
|
||||||
|
<circle cx="100" cy="100" r="95" fill="#4A90E2" opacity="0.1"/>
|
||||||
|
|
||||||
|
<!-- Link chain symbol -->
|
||||||
|
<g transform="translate(100, 100)">
|
||||||
|
<!-- Left link -->
|
||||||
|
<path d="M -60 -10 L -30 -10 C -20 -10 -15 -5 -15 5 L -15 5 C -15 15 -20 20 -30 20 L -60 20 C -70 20 -75 15 -75 5 L -75 -5 C -75 -15 -70 -20 -60 -20 Z"
|
||||||
|
fill="none" stroke="#4A90E2" stroke-width="8" stroke-linecap="round"/>
|
||||||
|
|
||||||
|
<!-- Right link -->
|
||||||
|
<path d="M 60 -10 L 30 -10 C 20 -10 15 -5 15 5 L 15 5 C 15 15 20 20 30 20 L 60 20 C 70 20 75 15 75 5 L 75 -5 C 75 -15 70 -20 60 -20 Z"
|
||||||
|
fill="none" stroke="#4A90E2" stroke-width="8" stroke-linecap="round"/>
|
||||||
|
|
||||||
|
<!-- Center connecting bar -->
|
||||||
|
<rect x="-15" y="-6" width="30" height="12" rx="6" fill="#4A90E2"/>
|
||||||
|
|
||||||
|
<!-- Vault door detail -->
|
||||||
|
<circle cx="0" cy="0" r="12" fill="none" stroke="#4A90E2" stroke-width="3"/>
|
||||||
|
<circle cx="0" cy="0" r="6" fill="#4A90E2"/>
|
||||||
|
|
||||||
|
<!-- Sync arrows -->
|
||||||
|
<g opacity="0.6">
|
||||||
|
<!-- Top arrow -->
|
||||||
|
<path d="M -5 -50 L 5 -50 L 0 -40 Z" fill="#4A90E2"/>
|
||||||
|
<!-- Bottom arrow -->
|
||||||
|
<path d="M 5 50 L -5 50 L 0 40 Z" fill="#4A90E2"/>
|
||||||
|
</g>
|
||||||
|
</g>
|
||||||
|
|
||||||
|
<!-- Text (optional) -->
|
||||||
|
<text x="100" y="175" font-family="Arial, sans-serif" font-size="24" font-weight="bold"
|
||||||
|
text-anchor="middle" fill="#4A90E2">VaultLink</text>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 1.4 KiB |
Loading…
Add table
Add a link
Reference in a new issue