Add docs
This commit is contained in:
parent
56c1f4d58b
commit
50a95b114d
19 changed files with 4663 additions and 1 deletions
65
.github/workflows/deploy-docs.yml
vendored
Normal file
65
.github/workflows/deploy-docs.yml
vendored
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
name: Deploy Documentation
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- 'docs/**'
|
||||
- '.github/workflows/deploy-docs.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pages: write
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: pages
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 20
|
||||
cache: npm
|
||||
cache-dependency-path: docs/package-lock.json
|
||||
|
||||
- name: Setup Pages
|
||||
uses: actions/configure-pages@v4
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
cd docs
|
||||
npm ci
|
||||
|
||||
- name: Build documentation
|
||||
run: |
|
||||
cd docs
|
||||
npm run build
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-pages-artifact@v3
|
||||
with:
|
||||
path: docs/.vitepress/dist
|
||||
|
||||
deploy:
|
||||
environment:
|
||||
name: github-pages
|
||||
url: ${{ steps.deployment.outputs.page_url }}
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
name: Deploy
|
||||
steps:
|
||||
- name: Deploy to GitHub Pages
|
||||
id: deployment
|
||||
uses: actions/deploy-pages@v4
|
||||
13
CLAUDE.md
13
CLAUDE.md
|
|
@ -29,7 +29,10 @@ cd sync-server
|
|||
cargo run config-e2e.yml # Start development server
|
||||
cargo test --verbose # Run Rust tests
|
||||
cargo clippy --all-targets --all-features # Lint Rust code
|
||||
cargo clippy --all-targets --all-features --fix --allow-dirty --allow-staged # Auto-fix clippy warnings
|
||||
cargo fmt --all -- --check # Check Rust formatting
|
||||
cargo fmt --all # Auto-format Rust code
|
||||
cargo machete --with-metadata # Detect unused dependencies
|
||||
```
|
||||
|
||||
### Frontend Development
|
||||
|
|
@ -49,8 +52,15 @@ sqlx migrate run --source src/app_state/database/migrations --database-url sqlit
|
|||
cargo sqlx prepare --workspace
|
||||
```
|
||||
|
||||
### Initial Setup
|
||||
```bash
|
||||
# Install required cargo tools
|
||||
cargo install sqlx-cli cargo-machete cargo-edit
|
||||
```
|
||||
|
||||
### Scripts
|
||||
- `scripts/check.sh`: Full CI check (builds, lints, tests both server and frontend)
|
||||
- `scripts/check.sh --fix`: Same as above but auto-fixes linting and formatting issues
|
||||
- `scripts/e2e.sh`: End-to-end testing
|
||||
- `scripts/clean-up.sh`: Clean logs and database files
|
||||
- `scripts/bump-version.sh patch`: Publish new version
|
||||
|
|
@ -59,10 +69,11 @@ cargo sqlx prepare --workspace
|
|||
## Code Structure
|
||||
|
||||
### Workspace Configuration
|
||||
The frontend uses npm workspaces with three packages:
|
||||
The frontend uses npm workspaces with four packages:
|
||||
- `sync-client`: Core synchronization logic
|
||||
- `obsidian-plugin`: Obsidian-specific integration
|
||||
- `test-client`: Testing utilities
|
||||
- `local-client-cli`: Standalone CLI for VaultLink sync client
|
||||
|
||||
### Type Generation
|
||||
Rust structs generate TypeScript types via ts-rs crate, stored in `sync-server/bindings/` and used by frontend packages.
|
||||
|
|
|
|||
4
docs/.gitignore
vendored
Normal file
4
docs/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
node_modules/
|
||||
.vitepress/dist/
|
||||
.vitepress/cache/
|
||||
package-lock.json
|
||||
62
docs/.vitepress/config.mts
Normal file
62
docs/.vitepress/config.mts
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
import { defineConfig } from 'vitepress'
|
||||
|
||||
export default defineConfig({
|
||||
title: 'VaultLink',
|
||||
description: 'Self-hosted real-time synchronization for Obsidian',
|
||||
base: '/vault-link/',
|
||||
themeConfig: {
|
||||
logo: '/logo.svg',
|
||||
nav: [
|
||||
{ text: 'Home', link: '/' },
|
||||
{ text: 'Guide', link: '/guide/getting-started' },
|
||||
{ text: 'Architecture', link: '/architecture/' },
|
||||
{ text: 'GitHub', link: 'https://github.com/schmelczer/vault-link' }
|
||||
],
|
||||
sidebar: [
|
||||
{
|
||||
text: 'Introduction',
|
||||
items: [
|
||||
{ text: 'What is VaultLink?', link: '/guide/what-is-vaultlink' },
|
||||
{ text: 'Getting Started', link: '/guide/getting-started' }
|
||||
]
|
||||
},
|
||||
{
|
||||
text: 'Setup',
|
||||
items: [
|
||||
{ text: 'Server Setup', link: '/guide/server-setup' },
|
||||
{ text: 'Obsidian Plugin', link: '/guide/obsidian-plugin' },
|
||||
{ text: 'CLI Client', link: '/guide/cli-client' }
|
||||
]
|
||||
},
|
||||
{
|
||||
text: 'Configuration',
|
||||
items: [
|
||||
{ text: 'Server Configuration', link: '/config/server' },
|
||||
{ text: 'Authentication', link: '/config/authentication' },
|
||||
{ text: 'Advanced Options', link: '/config/advanced' }
|
||||
]
|
||||
},
|
||||
{
|
||||
text: 'Architecture',
|
||||
items: [
|
||||
{ text: 'Overview', link: '/architecture/' },
|
||||
{ text: 'Sync Algorithm', link: '/architecture/sync-algorithm' },
|
||||
{ text: 'Data Flow', link: '/architecture/data-flow' }
|
||||
]
|
||||
}
|
||||
],
|
||||
socialLinks: [
|
||||
{ icon: 'github', link: 'https://github.com/schmelczer/vault-link' }
|
||||
],
|
||||
footer: {
|
||||
message: 'Released under the MIT License.',
|
||||
copyright: 'Copyright © 2024-present Andras Schmelczer'
|
||||
},
|
||||
search: {
|
||||
provider: 'local'
|
||||
}
|
||||
},
|
||||
head: [
|
||||
['link', { rel: 'icon', type: 'image/svg+xml', href: '/vault-link/logo.svg' }]
|
||||
]
|
||||
})
|
||||
130
docs/README.md
Normal file
130
docs/README.md
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
# VaultLink Documentation
|
||||
|
||||
This directory contains the VaultLink documentation site built with [VitePress](https://vitepress.dev/).
|
||||
|
||||
## Development
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- Node.js 18+
|
||||
- npm
|
||||
|
||||
### Setup
|
||||
|
||||
```bash
|
||||
cd docs
|
||||
npm install
|
||||
```
|
||||
|
||||
### Local Development
|
||||
|
||||
Start the development server with hot reload:
|
||||
|
||||
```bash
|
||||
npm run dev
|
||||
```
|
||||
|
||||
The site will be available at `http://localhost:5173/vault-link/`
|
||||
|
||||
### Build
|
||||
|
||||
Build the static site:
|
||||
|
||||
```bash
|
||||
npm run build
|
||||
```
|
||||
|
||||
Output will be in `.vitepress/dist/`
|
||||
|
||||
### Preview
|
||||
|
||||
Preview the built site:
|
||||
|
||||
```bash
|
||||
npm run preview
|
||||
```
|
||||
|
||||
## Deployment
|
||||
|
||||
The documentation is automatically deployed to GitHub Pages when changes are pushed to the `main` branch.
|
||||
|
||||
The deployment workflow is configured in `.github/workflows/deploy-docs.yml`.
|
||||
|
||||
## Structure
|
||||
|
||||
```
|
||||
docs/
|
||||
├── .vitepress/
|
||||
│ └── config.ts # VitePress configuration
|
||||
├── public/ # Static assets
|
||||
│ └── logo.svg # VaultLink logo
|
||||
├── guide/ # User guides
|
||||
│ ├── what-is-vaultlink.md
|
||||
│ ├── getting-started.md
|
||||
│ ├── server-setup.md
|
||||
│ ├── obsidian-plugin.md
|
||||
│ └── cli-client.md
|
||||
├── architecture/ # Architecture documentation
|
||||
│ ├── index.md
|
||||
│ ├── sync-algorithm.md
|
||||
│ └── data-flow.md
|
||||
├── config/ # Configuration reference
|
||||
│ ├── server.md
|
||||
│ ├── authentication.md
|
||||
│ └── advanced.md
|
||||
└── index.md # Home page
|
||||
|
||||
```
|
||||
|
||||
## Writing Documentation
|
||||
|
||||
### Markdown Features
|
||||
|
||||
VitePress supports:
|
||||
- GitHub Flavored Markdown
|
||||
- Custom containers (tip, warning, danger)
|
||||
- Code syntax highlighting
|
||||
- Mermaid diagrams
|
||||
- Emoji :rocket:
|
||||
|
||||
### Custom Containers
|
||||
|
||||
```markdown
|
||||
::: tip
|
||||
This is a tip
|
||||
:::
|
||||
|
||||
::: warning
|
||||
This is a warning
|
||||
:::
|
||||
|
||||
::: danger
|
||||
This is a danger message
|
||||
:::
|
||||
```
|
||||
|
||||
### Code Blocks
|
||||
|
||||
````markdown
|
||||
```bash
|
||||
npm install
|
||||
```
|
||||
|
||||
```yaml
|
||||
server:
|
||||
port: 3000
|
||||
```
|
||||
````
|
||||
|
||||
## Contributing
|
||||
|
||||
When adding new pages:
|
||||
|
||||
1. Create the markdown file in the appropriate directory
|
||||
2. Add it to the sidebar in `.vitepress/config.ts`
|
||||
3. Test locally with `npm run dev`
|
||||
4. Submit a pull request
|
||||
|
||||
## License
|
||||
|
||||
MIT - Same as VaultLink
|
||||
532
docs/architecture/data-flow.md
Normal file
532
docs/architecture/data-flow.md
Normal file
|
|
@ -0,0 +1,532 @@
|
|||
# Data Flow
|
||||
|
||||
This document provides a detailed look at how data flows through the VaultLink system, from client to server and back.
|
||||
|
||||
## Connection Lifecycle
|
||||
|
||||
### 1. Initial Connection
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant C as Client
|
||||
participant S as Server
|
||||
participant DB as Database
|
||||
|
||||
C->>S: WebSocket connect
|
||||
S->>S: Accept connection
|
||||
C->>S: Auth message (token + vault)
|
||||
S->>S: Validate token
|
||||
S->>S: Check vault access
|
||||
S-->>C: Auth success
|
||||
Note over C,S: Connection established
|
||||
```
|
||||
|
||||
**Steps**:
|
||||
1. Client initiates WebSocket connection to server
|
||||
2. Server accepts connection
|
||||
3. Client sends authentication message with token and vault name
|
||||
4. Server validates token against `config.yml`
|
||||
5. Server checks if user has access to requested vault
|
||||
6. Server responds with success or error
|
||||
7. Connection is ready for syncing
|
||||
|
||||
### 2. Initial Sync
|
||||
|
||||
After authentication, the client performs initial synchronization:
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant C as Client
|
||||
participant S as Server
|
||||
participant DB as SQLite
|
||||
|
||||
C->>C: Scan local filesystem
|
||||
C->>S: Request file list
|
||||
S->>DB: Query all files
|
||||
DB-->>S: File metadata
|
||||
S-->>C: File list with versions
|
||||
|
||||
loop For each local file
|
||||
C->>C: Check if file on server
|
||||
alt File not on server
|
||||
C->>S: Upload file
|
||||
S->>DB: Store file + metadata
|
||||
else File on server (different version)
|
||||
C->>C: Compare versions
|
||||
C->>S: Upload newer or merge
|
||||
end
|
||||
end
|
||||
|
||||
loop For each server file
|
||||
C->>C: Check if file local
|
||||
alt File not local
|
||||
C->>S: Download file
|
||||
S->>DB: Retrieve file
|
||||
DB-->>S: File content
|
||||
S-->>C: File content
|
||||
C->>C: Write to disk
|
||||
end
|
||||
end
|
||||
|
||||
S-->>C: Sync complete message
|
||||
```
|
||||
|
||||
**Process**:
|
||||
1. Client scans local filesystem
|
||||
2. Client requests file list from server
|
||||
3. Server queries database and returns metadata
|
||||
4. Client uploads missing or changed local files
|
||||
5. Client downloads missing files from server
|
||||
6. Server sends sync complete notification
|
||||
|
||||
### 3. Real-Time Synchronization
|
||||
|
||||
After initial sync, changes are pushed in real-time:
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant FS as Filesystem
|
||||
participant C1 as Client 1
|
||||
participant S as Server
|
||||
participant DB as Database
|
||||
participant C2 as Client 2
|
||||
|
||||
FS->>C1: File changed (fs.watch)
|
||||
C1->>C1: Read file content
|
||||
C1->>S: Upload file
|
||||
S->>DB: Store new version
|
||||
S->>S: Apply OT if needed
|
||||
S-->>C1: Upload ACK
|
||||
S->>C2: File update notification
|
||||
C2->>S: Download file
|
||||
S->>DB: Retrieve file
|
||||
DB-->>S: File content
|
||||
S-->>C2: File content
|
||||
C2->>FS: Write to disk
|
||||
```
|
||||
|
||||
**Flow**:
|
||||
1. Filesystem watcher detects local change
|
||||
2. Client reads file content
|
||||
3. Client uploads file via WebSocket
|
||||
4. Server stores in database
|
||||
5. Server applies operational transformation if concurrent edits
|
||||
6. Server acknowledges upload to sender
|
||||
7. Server broadcasts update to other clients
|
||||
8. Other clients download and apply changes
|
||||
|
||||
## File Operations
|
||||
|
||||
### Upload
|
||||
|
||||
```
|
||||
┌─────────┐
|
||||
│ Client │
|
||||
└────┬────┘
|
||||
│ 1. Detect file change
|
||||
│
|
||||
├─► 2. Read file content
|
||||
│
|
||||
├─► 3. Create upload message
|
||||
│ {
|
||||
│ type: "upload_file",
|
||||
│ path: "notes/daily.md",
|
||||
│ content: "...",
|
||||
│ version: 42,
|
||||
│ timestamp: "2024-01-01T12:00:00Z"
|
||||
│ }
|
||||
│
|
||||
▼
|
||||
┌─────────┐
|
||||
│ Server │
|
||||
└────┬────┘
|
||||
│ 4. Validate message
|
||||
│
|
||||
├─► 5. Check permissions
|
||||
│
|
||||
├─► 6. Apply OT (if conflicts)
|
||||
│
|
||||
├─► 7. Store in database
|
||||
│
|
||||
├─► 8. Update version
|
||||
│
|
||||
├─► 9. Broadcast to clients
|
||||
│
|
||||
└─► 10. Send ACK to uploader
|
||||
```
|
||||
|
||||
### Download
|
||||
|
||||
```
|
||||
┌─────────┐
|
||||
│ Server │
|
||||
└────┬────┘
|
||||
│ 1. File updated by another client
|
||||
│
|
||||
├─► 2. Broadcast notification
|
||||
│ {
|
||||
│ type: "file_updated",
|
||||
│ path: "notes/daily.md",
|
||||
│ version: 43
|
||||
│ }
|
||||
│
|
||||
▼
|
||||
┌─────────┐
|
||||
│ Client │
|
||||
└────┬────┘
|
||||
│ 3. Receive notification
|
||||
│
|
||||
├─► 4. Request file download
|
||||
│ {
|
||||
│ type: "download_file",
|
||||
│ path: "notes/daily.md",
|
||||
│ version: 43
|
||||
│ }
|
||||
│
|
||||
▼
|
||||
┌─────────┐
|
||||
│ Server │
|
||||
└────┬────┘
|
||||
│ 5. Retrieve from database
|
||||
│
|
||||
└─► 6. Send file content
|
||||
{
|
||||
type: "file_content",
|
||||
path: "notes/daily.md",
|
||||
content: "...",
|
||||
version: 43
|
||||
}
|
||||
│
|
||||
▼
|
||||
┌─────────┐
|
||||
│ Client │
|
||||
└────┬────┘
|
||||
│ 7. Write to filesystem
|
||||
│
|
||||
└─► 8. Update local metadata
|
||||
```
|
||||
|
||||
### Delete
|
||||
|
||||
```
|
||||
┌─────────┐
|
||||
│ Client │
|
||||
└────┬────┘
|
||||
│ 1. File deleted locally
|
||||
│
|
||||
├─► 2. Send delete message
|
||||
│ {
|
||||
│ type: "delete_file",
|
||||
│ path: "notes/old.md"
|
||||
│ }
|
||||
│
|
||||
▼
|
||||
┌─────────┐
|
||||
│ Server │
|
||||
└────┬────┘
|
||||
│ 3. Mark as deleted in DB
|
||||
│ (soft delete for history)
|
||||
│
|
||||
├─► 4. Broadcast deletion
|
||||
│
|
||||
└─► 5. ACK to sender
|
||||
│
|
||||
▼
|
||||
┌─────────┐
|
||||
│ Other │
|
||||
│ Clients │
|
||||
└────┬────┘
|
||||
│ 6. Delete local file
|
||||
│
|
||||
└─► 7. Update metadata
|
||||
```
|
||||
|
||||
## Conflict Resolution Flow
|
||||
|
||||
### Concurrent Edits Scenario
|
||||
|
||||
```
|
||||
Time →
|
||||
|
||||
Client A Server Client B
|
||||
│ │ │
|
||||
│ Edit file v10 │ │
|
||||
│ "Add line A" │ │ Edit file v10
|
||||
│ │ │ "Add line B"
|
||||
│ │ │
|
||||
├─── Upload @ t1 ─────────►│ │
|
||||
│ │◄────── Upload @ t2 ────────┤
|
||||
│ │ │
|
||||
│ │ 1. Receive both edits │
|
||||
│ │ (based on v10) │
|
||||
│ │ │
|
||||
│ │ 2. Apply first edit │
|
||||
│ │ → v11 (line A added) │
|
||||
│ │ │
|
||||
│ │ 3. Transform second edit │
|
||||
│ │ against first │
|
||||
│ │ │
|
||||
│ │ 4. Apply transformed edit │
|
||||
│ │ → v12 (both lines) │
|
||||
│ │ │
|
||||
│◄──── v12 content ────────┤ │
|
||||
│ ├───── v12 content ─────────►│
|
||||
│ │ │
|
||||
│ Apply v12 │ │ Apply v12
|
||||
│ (has both lines) │ │ (has both lines)
|
||||
│ │ │
|
||||
```
|
||||
|
||||
### Conflict Resolution Steps
|
||||
|
||||
1. **Detection**: Server receives two edits based on the same version
|
||||
2. **Ordering**: Determine which edit to apply first (by timestamp or client ID)
|
||||
3. **First edit**: Apply directly to database
|
||||
4. **Transformation**: Transform second edit against first using OT
|
||||
5. **Second edit**: Apply transformed edit to database
|
||||
6. **Broadcast**: Send merged result to all clients
|
||||
7. **Application**: Clients apply merged version locally
|
||||
|
||||
## Database Schema
|
||||
|
||||
### Core Tables
|
||||
|
||||
```sql
|
||||
-- Document metadata
|
||||
CREATE TABLE documents (
|
||||
id INTEGER PRIMARY KEY,
|
||||
path TEXT NOT NULL,
|
||||
version INTEGER NOT NULL,
|
||||
content_hash TEXT,
|
||||
size INTEGER,
|
||||
created_at TIMESTAMP,
|
||||
updated_at TIMESTAMP,
|
||||
deleted BOOLEAN DEFAULT FALSE
|
||||
);
|
||||
|
||||
-- Version history
|
||||
CREATE TABLE versions (
|
||||
id INTEGER PRIMARY KEY,
|
||||
document_id INTEGER,
|
||||
version INTEGER,
|
||||
content BLOB,
|
||||
created_at TIMESTAMP,
|
||||
FOREIGN KEY (document_id) REFERENCES documents(id)
|
||||
);
|
||||
|
||||
-- Client sync cursors
|
||||
CREATE TABLE cursors (
|
||||
client_id TEXT PRIMARY KEY,
|
||||
last_version INTEGER,
|
||||
last_updated TIMESTAMP
|
||||
);
|
||||
```
|
||||
|
||||
### Queries
|
||||
|
||||
**Get files since version**:
|
||||
```sql
|
||||
SELECT * FROM documents
|
||||
WHERE version > ? AND deleted = FALSE
|
||||
ORDER BY version ASC;
|
||||
```
|
||||
|
||||
**Store new version**:
|
||||
```sql
|
||||
INSERT INTO versions (document_id, version, content, created_at)
|
||||
VALUES (?, ?, ?, ?);
|
||||
|
||||
UPDATE documents
|
||||
SET version = ?, updated_at = ?
|
||||
WHERE id = ?;
|
||||
```
|
||||
|
||||
**Update cursor**:
|
||||
```sql
|
||||
INSERT OR REPLACE INTO cursors (client_id, last_version, last_updated)
|
||||
VALUES (?, ?, ?);
|
||||
```
|
||||
|
||||
## Message Protocol
|
||||
|
||||
### Client → Server Messages
|
||||
|
||||
**Upload File**:
|
||||
```json
|
||||
{
|
||||
"type": "upload_file",
|
||||
"path": "notes/example.md",
|
||||
"content": "File content here...",
|
||||
"base_version": 10,
|
||||
"timestamp": "2024-01-01T12:00:00Z"
|
||||
}
|
||||
```
|
||||
|
||||
**Download File**:
|
||||
```json
|
||||
{
|
||||
"type": "download_file",
|
||||
"path": "notes/example.md"
|
||||
}
|
||||
```
|
||||
|
||||
**Delete File**:
|
||||
```json
|
||||
{
|
||||
"type": "delete_file",
|
||||
"path": "notes/old.md"
|
||||
}
|
||||
```
|
||||
|
||||
**List Files**:
|
||||
```json
|
||||
{
|
||||
"type": "list_files",
|
||||
"since_version": 0
|
||||
}
|
||||
```
|
||||
|
||||
### Server → Client Messages
|
||||
|
||||
**File Updated**:
|
||||
```json
|
||||
{
|
||||
"type": "file_updated",
|
||||
"path": "notes/example.md",
|
||||
"version": 11,
|
||||
"size": 1024,
|
||||
"hash": "abc123..."
|
||||
}
|
||||
```
|
||||
|
||||
**File Content**:
|
||||
```json
|
||||
{
|
||||
"type": "file_content",
|
||||
"path": "notes/example.md",
|
||||
"content": "Updated content...",
|
||||
"version": 11
|
||||
}
|
||||
```
|
||||
|
||||
**File Deleted**:
|
||||
```json
|
||||
{
|
||||
"type": "file_deleted",
|
||||
"path": "notes/old.md",
|
||||
"version": 12
|
||||
}
|
||||
```
|
||||
|
||||
**Sync Complete**:
|
||||
```json
|
||||
{
|
||||
"type": "sync_complete",
|
||||
"total_files": 150,
|
||||
"current_version": 200
|
||||
}
|
||||
```
|
||||
|
||||
**Error**:
|
||||
```json
|
||||
{
|
||||
"type": "error",
|
||||
"message": "File too large",
|
||||
"code": "FILE_TOO_LARGE"
|
||||
}
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
### Client-Side Errors
|
||||
|
||||
**Network failure**:
|
||||
1. Detect WebSocket disconnect
|
||||
2. Queue pending operations
|
||||
3. Retry connection with exponential backoff
|
||||
4. Replay queued operations on reconnect
|
||||
|
||||
**File read error**:
|
||||
1. Log error
|
||||
2. Skip file
|
||||
3. Continue with other files
|
||||
4. Report to user
|
||||
|
||||
**Write conflict**:
|
||||
1. Receive updated version from server
|
||||
2. Apply OT merge locally
|
||||
3. Overwrite local file
|
||||
4. Continue syncing
|
||||
|
||||
### Server-Side Errors
|
||||
|
||||
**Database error**:
|
||||
1. Log error
|
||||
2. Return error to client
|
||||
3. Client retries operation
|
||||
|
||||
**Invalid operation**:
|
||||
1. Validate message format
|
||||
2. Return specific error code
|
||||
3. Client handles error appropriately
|
||||
|
||||
**Authentication failure**:
|
||||
1. Reject connection
|
||||
2. Send auth error
|
||||
3. Client prompts for new credentials
|
||||
|
||||
## Performance Optimizations
|
||||
|
||||
### Batching
|
||||
|
||||
- Small, rapid changes are batched together
|
||||
- Reduces message overhead
|
||||
- Applied as single atomic update
|
||||
|
||||
### Compression
|
||||
|
||||
- Large files compressed before transmission
|
||||
- Reduces bandwidth usage
|
||||
- Transparent to application layer
|
||||
|
||||
### Incremental Sync
|
||||
|
||||
- Only changed portions of files sent
|
||||
- Uses content-based diffing
|
||||
- Significantly reduces data transfer
|
||||
|
||||
### Caching
|
||||
|
||||
- Server caches recent file versions
|
||||
- Reduces database queries
|
||||
- Improves response time
|
||||
|
||||
## Monitoring Data Flow
|
||||
|
||||
### Server Logs
|
||||
|
||||
```
|
||||
2024-01-01 12:00:00 INFO WebSocket connection from 192.168.1.100
|
||||
2024-01-01 12:00:01 INFO User 'alice' authenticated for vault 'personal'
|
||||
2024-01-01 12:00:05 INFO Upload: notes/daily.md (v10 -> v11)
|
||||
2024-01-01 12:00:06 INFO Broadcast to 3 clients
|
||||
2024-01-01 12:00:10 INFO Conflict resolved: notes/shared.md (v12)
|
||||
```
|
||||
|
||||
### Client Logs
|
||||
|
||||
```
|
||||
2024-01-01 12:00:00 INFO Connecting to ws://sync.example.com
|
||||
2024-01-01 12:00:01 INFO Connected, authenticating...
|
||||
2024-01-01 12:00:01 INFO Authentication successful
|
||||
2024-01-01 12:00:02 INFO Starting initial sync
|
||||
2024-01-01 12:00:10 INFO Sync complete: 150 files, 200 MB
|
||||
2024-01-01 12:00:15 INFO Uploaded: notes/daily.md
|
||||
2024-01-01 12:00:20 INFO Downloaded: notes/shared.md (merged)
|
||||
```
|
||||
|
||||
## Next Steps
|
||||
|
||||
- [Understand the sync algorithm →](/architecture/sync-algorithm)
|
||||
- [Configure the server →](/config/server)
|
||||
- [Deploy VaultLink →](/guide/getting-started)
|
||||
344
docs/architecture/index.md
Normal file
344
docs/architecture/index.md
Normal file
|
|
@ -0,0 +1,344 @@
|
|||
# Architecture Overview
|
||||
|
||||
VaultLink is built as a distributed system with a central sync server and multiple clients. This document explains the high-level architecture and design decisions.
|
||||
|
||||
## System Components
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ Clients │
|
||||
├─────────────────────┬───────────────────┬───────────────────┤
|
||||
│ Obsidian Plugin │ Obsidian Plugin │ CLI Client │
|
||||
│ (User A - Device1) │ (User A - Device2│ (Server/Backup) │
|
||||
└──────────┬──────────┴─────────┬─────────┴──────────┬────────┘
|
||||
│ │ │
|
||||
│ WebSocket │ WebSocket │ WebSocket
|
||||
│ │ │
|
||||
└────────────────────┼────────────────────┘
|
||||
│
|
||||
┌───────────▼───────────┐
|
||||
│ Sync Server │
|
||||
│ (Rust + Axum) │
|
||||
│ │
|
||||
│ ┌─────────────────┐ │
|
||||
│ │ WebSocket Hub │ │
|
||||
│ └────────┬────────┘ │
|
||||
│ │ │
|
||||
│ ┌────────▼────────┐ │
|
||||
│ │ Sync Engine │ │
|
||||
│ │ (OT Algorithm) │ │
|
||||
│ └────────┬────────┘ │
|
||||
│ │ │
|
||||
│ ┌────────▼────────┐ │
|
||||
│ │ SQLite Database │ │
|
||||
│ │ (Per Vault) │ │
|
||||
│ └─────────────────┘ │
|
||||
└───────────────────────┘
|
||||
```
|
||||
|
||||
## Core Components
|
||||
|
||||
### Sync Server
|
||||
|
||||
The central authority for synchronization, written in Rust using Axum framework.
|
||||
|
||||
**Responsibilities**:
|
||||
- Accept WebSocket connections from clients
|
||||
- Authenticate users via token-based auth
|
||||
- Store document versions in SQLite
|
||||
- Coordinate real-time updates between clients
|
||||
- Apply operational transformation for conflict resolution
|
||||
- Manage vault access control
|
||||
|
||||
**Technology**:
|
||||
- **Language**: Rust 1.89+
|
||||
- **Framework**: Axum (async web framework)
|
||||
- **Database**: SQLite with SQLx
|
||||
- **Protocol**: WebSockets for real-time communication
|
||||
- **Sync Algorithm**: reconcile-text (operational transformation)
|
||||
|
||||
### Sync Client Library
|
||||
|
||||
TypeScript library providing core synchronization logic, used by both the Obsidian plugin and CLI client.
|
||||
|
||||
**Responsibilities**:
|
||||
- Manage WebSocket connection to server
|
||||
- Watch local filesystem for changes
|
||||
- Upload and download files
|
||||
- Apply remote changes locally
|
||||
- Handle conflict resolution
|
||||
- Maintain sync metadata
|
||||
|
||||
**Technology**:
|
||||
- **Language**: TypeScript
|
||||
- **Build**: Webpack
|
||||
- **Protocol**: WebSocket client
|
||||
- **File System**: Node.js `fs` API / Obsidian API
|
||||
|
||||
### Obsidian Plugin
|
||||
|
||||
Integration layer between sync client and Obsidian.
|
||||
|
||||
**Responsibilities**:
|
||||
- Provide UI for configuration
|
||||
- Bridge sync client with Obsidian's file system API
|
||||
- Handle Obsidian lifecycle events
|
||||
- Display sync status to users
|
||||
|
||||
**Technology**:
|
||||
- **Platform**: Obsidian Plugin API
|
||||
- **Core**: sync-client library
|
||||
- **UI**: Obsidian settings UI
|
||||
|
||||
### CLI Client
|
||||
|
||||
Standalone executable for syncing vaults without Obsidian.
|
||||
|
||||
**Responsibilities**:
|
||||
- Command-line interface
|
||||
- File system access via Node.js
|
||||
- Daemon mode for continuous sync
|
||||
- Health check endpoint for monitoring
|
||||
|
||||
**Technology**:
|
||||
- **Language**: TypeScript
|
||||
- **Runtime**: Node.js
|
||||
- **CLI**: Commander.js
|
||||
- **Core**: sync-client library
|
||||
|
||||
## Data Flow
|
||||
|
||||
### Initial Connection
|
||||
|
||||
1. Client connects via WebSocket to server
|
||||
2. Server authenticates using provided token
|
||||
3. Server verifies user has access to requested vault
|
||||
4. Connection established, sync begins
|
||||
|
||||
### File Upload Flow
|
||||
|
||||
```
|
||||
Client Server
|
||||
│ │
|
||||
│ 1. File changed locally │
|
||||
│ │
|
||||
│ 2. Read file content │
|
||||
│ │
|
||||
│ 3. WebSocket: Upload file │
|
||||
├──────────────────────────────►│
|
||||
│ │ 4. Store in SQLite
|
||||
│ │
|
||||
│ │ 5. Broadcast to other clients
|
||||
│ ├───────────────────────►
|
||||
│ 6. Ack upload │
|
||||
│◄──────────────────────────────┤
|
||||
```
|
||||
|
||||
### File Download Flow
|
||||
|
||||
```
|
||||
Client A Server Client B
|
||||
│ │ │
|
||||
│ │ 1. File uploaded │
|
||||
│ │◄────────────────────────┤
|
||||
│ │ │
|
||||
│ │ 2. Store in DB │
|
||||
│ │ │
|
||||
│ 3. Push notification │ │
|
||||
│◄────────────────────────┤ │
|
||||
│ │ │
|
||||
│ 4. Download file │ │
|
||||
├────────────────────────►│ │
|
||||
│ │ │
|
||||
│ 5. Write locally │ │
|
||||
│ │ │
|
||||
```
|
||||
|
||||
### Conflict Resolution
|
||||
|
||||
When two clients edit the same file simultaneously:
|
||||
|
||||
```
|
||||
Client A Server Client B
|
||||
│ │ │
|
||||
│ 1. Edit file │ │ 1. Edit same file
|
||||
│ │ │
|
||||
│ 2. Upload changes │ │ 2. Upload changes
|
||||
├────────────────────────►│◄────────────────────────┤
|
||||
│ │ │
|
||||
│ │ 3. Apply OT algorithm │
|
||||
│ │ - Merge both edits │
|
||||
│ │ - Preserve all changes│
|
||||
│ │ │
|
||||
│ 4. Receive merged ver. │ 5. Receive merged ver. │
|
||||
│◄────────────────────────┤────────────────────────►│
|
||||
│ │ │
|
||||
│ 6. Apply locally │ │ 6. Apply locally
|
||||
```
|
||||
|
||||
## Storage Architecture
|
||||
|
||||
### Server Storage
|
||||
|
||||
Each vault has its own SQLite database:
|
||||
|
||||
```
|
||||
databases/
|
||||
├── vault-1.db
|
||||
├── vault-2.db
|
||||
└── shared-team.db
|
||||
```
|
||||
|
||||
**Database Schema** (simplified):
|
||||
- **documents**: File metadata (path, size, modified time)
|
||||
- **versions**: Document content with version history
|
||||
- **cursors**: Client sync state
|
||||
|
||||
### Client Storage
|
||||
|
||||
Clients maintain sync metadata:
|
||||
|
||||
```
|
||||
.vaultlink/
|
||||
├── metadata.json # Sync state
|
||||
└── cache/ # Optional local cache
|
||||
```
|
||||
|
||||
The `.vaultlink` directory tracks which files have been synced and their versions to enable efficient synchronization.
|
||||
|
||||
## Communication Protocol
|
||||
|
||||
### WebSocket Messages
|
||||
|
||||
Client-server communication uses JSON messages over WebSocket.
|
||||
|
||||
**Message Types**:
|
||||
- `upload_file`: Client → Server (file upload)
|
||||
- `download_file`: Client → Server (request file)
|
||||
- `file_updated`: Server → Client (file changed notification)
|
||||
- `file_deleted`: Server → Client (file deleted notification)
|
||||
- `sync_complete`: Server → Client (initial sync finished)
|
||||
|
||||
### Authentication
|
||||
|
||||
Token-based authentication on connection:
|
||||
|
||||
```typescript
|
||||
// Client sends token on connect
|
||||
{
|
||||
type: "auth",
|
||||
token: "user-auth-token",
|
||||
vault: "vault-name"
|
||||
}
|
||||
|
||||
// Server responds
|
||||
{
|
||||
type: "auth_success"
|
||||
}
|
||||
// or
|
||||
{
|
||||
type: "auth_error",
|
||||
message: "Invalid token"
|
||||
}
|
||||
```
|
||||
|
||||
## Scalability Considerations
|
||||
|
||||
### Current Architecture
|
||||
|
||||
- **SQLite per vault**: Simple, performant, limited to single server
|
||||
- **WebSocket connections**: Stateful, requires sticky sessions for load balancing
|
||||
- **Operational transformation**: Centralized on server
|
||||
|
||||
### Scaling Approaches
|
||||
|
||||
**Vertical Scaling**:
|
||||
- Increase server resources (CPU, RAM, storage)
|
||||
- Optimize database queries and indexing
|
||||
- Tune connection limits
|
||||
|
||||
**Horizontal Scaling** (future):
|
||||
- Separate vault servers (vault sharding)
|
||||
- Load balancer with sticky sessions
|
||||
- Shared storage layer for SQLite databases
|
||||
- Consider alternative databases (PostgreSQL) for multi-server setups
|
||||
|
||||
### Performance Characteristics
|
||||
|
||||
- **Small vaults** (< 1000 files): Excellent performance
|
||||
- **Medium vaults** (1000-10000 files): Good performance with tuning
|
||||
- **Large vaults** (> 10000 files): May require optimization
|
||||
- **Concurrent users**: Tested with dozens of simultaneous clients per vault
|
||||
|
||||
## Security Model
|
||||
|
||||
### Authentication
|
||||
|
||||
- Token-based authentication
|
||||
- Tokens configured in server `config.yml`
|
||||
- No password hashing (tokens are secrets)
|
||||
|
||||
### Authorization
|
||||
|
||||
- Per-user vault access control
|
||||
- Allow-list or deny-list patterns
|
||||
- Global access or vault-specific access
|
||||
|
||||
### Network Security
|
||||
|
||||
- WebSocket over TLS (WSS) for encrypted transport
|
||||
- No built-in SSL (use reverse proxy)
|
||||
- CORS configured for web clients
|
||||
|
||||
### Data Security
|
||||
|
||||
- No encryption at rest (use encrypted filesystems if needed)
|
||||
- No end-to-end encryption (server sees all content)
|
||||
- Self-hosted model: you control the data
|
||||
|
||||
## Technology Choices
|
||||
|
||||
### Why Rust for Server?
|
||||
|
||||
- **Performance**: Low latency for real-time sync
|
||||
- **Memory safety**: No crashes from memory bugs
|
||||
- **Concurrency**: Excellent async support with Tokio
|
||||
- **Type safety**: Catch bugs at compile time
|
||||
- **SQLx**: Compile-time SQL verification
|
||||
|
||||
### Why SQLite?
|
||||
|
||||
- **Simplicity**: No separate database server required
|
||||
- **Performance**: Fast for read-heavy workloads
|
||||
- **Reliability**: Battle-tested, ACID compliant
|
||||
- **Portability**: Single file per vault
|
||||
- **Backups**: Simple file copy
|
||||
|
||||
### Why WebSocket?
|
||||
|
||||
- **Real-time**: Bidirectional push for instant updates
|
||||
- **Efficiency**: Persistent connection, no polling overhead
|
||||
- **Simplicity**: Built-in browser/Node.js support
|
||||
- **Standards**: Well-supported protocol
|
||||
|
||||
### Why Operational Transformation?
|
||||
|
||||
- **Automatic conflict resolution**: No manual merging required
|
||||
- **Preserves intent**: All edits are kept
|
||||
- **Real-time collaboration**: Users see changes as they happen
|
||||
- **Proven algorithm**: Used by Google Docs, etc.
|
||||
|
||||
## Design Principles
|
||||
|
||||
1. **Self-hosted first**: Users control their data and infrastructure
|
||||
2. **Simplicity**: Easy to deploy and operate
|
||||
3. **Real-time**: Changes appear immediately
|
||||
4. **Reliability**: Handle network failures gracefully
|
||||
5. **Performance**: Fast sync for typical vault sizes
|
||||
6. **Privacy**: No third-party services or telemetry
|
||||
|
||||
## Next Steps
|
||||
|
||||
- [Learn about the sync algorithm →](/architecture/sync-algorithm)
|
||||
- [Understand data flow in detail →](/architecture/data-flow)
|
||||
- [Deploy the server →](/guide/server-setup)
|
||||
361
docs/architecture/sync-algorithm.md
Normal file
361
docs/architecture/sync-algorithm.md
Normal file
|
|
@ -0,0 +1,361 @@
|
|||
# Sync Algorithm
|
||||
|
||||
VaultLink uses operational transformation (OT) to handle concurrent edits and maintain consistency across clients. This document explains how the algorithm works.
|
||||
|
||||
## Operational Transformation
|
||||
|
||||
Operational transformation is a technique for managing concurrent edits to the same document. It transforms operations (edits) so they can be applied in different orders while preserving user intent.
|
||||
|
||||
### Why OT?
|
||||
|
||||
Traditional conflict resolution approaches:
|
||||
- **Last write wins**: Loses data, frustrating for users
|
||||
- **Manual merging**: Interrupts workflow, requires user intervention
|
||||
- **Version branching**: Complex, not suitable for real-time sync
|
||||
|
||||
Operational transformation:
|
||||
- **Automatic**: No user intervention required
|
||||
- **Preserves all edits**: No data loss
|
||||
- **Real-time**: Changes appear immediately
|
||||
- **Intuitive**: Behavior matches user expectations
|
||||
|
||||
## The reconcile-text Library
|
||||
|
||||
VaultLink uses the [`reconcile-text`](https://crates.io/crates/reconcile-text) Rust library for operational transformation on text documents.
|
||||
|
||||
### How It Works
|
||||
|
||||
Given a base document and two sets of changes, OT produces a merged result that includes both changes.
|
||||
|
||||
**Example**:
|
||||
|
||||
```
|
||||
Base document: "Hello world"
|
||||
|
||||
User A: "Hello beautiful world" (inserts "beautiful ")
|
||||
User B: "Hello world!" (inserts "!")
|
||||
|
||||
OT result: "Hello beautiful world!" (both changes applied)
|
||||
```
|
||||
|
||||
### Operation Types
|
||||
|
||||
The algorithm handles these operations:
|
||||
- **Insert**: Add text at position
|
||||
- **Delete**: Remove text from position
|
||||
- **Retain**: Keep existing text unchanged
|
||||
|
||||
### Transformation Process
|
||||
|
||||
1. **Client A** makes edit and sends to server
|
||||
2. **Client B** makes concurrent edit and sends to server
|
||||
3. **Server** receives both edits
|
||||
4. **Server** transforms operations to account for concurrent changes
|
||||
5. **Server** applies merged result to database
|
||||
6. **Server** sends transformed operations to both clients
|
||||
7. **Clients** apply transformed operations locally
|
||||
|
||||
## Sync State Management
|
||||
|
||||
VaultLink maintains sync state to track which changes have been applied.
|
||||
|
||||
### Version Vectors
|
||||
|
||||
Each document has a version tracked by:
|
||||
- **Server version**: Incremented on each change
|
||||
- **Client cursors**: Track which version each client has seen
|
||||
|
||||
This enables:
|
||||
- Efficient syncing (only send changes since last sync)
|
||||
- Conflict detection (concurrent edits to same version)
|
||||
- Ordering of operations
|
||||
|
||||
### Cursor Management
|
||||
|
||||
Clients maintain a cursor position:
|
||||
|
||||
```rust
|
||||
struct Cursor {
|
||||
vault_id: String,
|
||||
client_id: String,
|
||||
last_version: u64,
|
||||
last_updated: DateTime,
|
||||
}
|
||||
```
|
||||
|
||||
On sync:
|
||||
1. Client sends cursor (last seen version)
|
||||
2. Server returns all changes since that version
|
||||
3. Client applies changes and updates cursor
|
||||
|
||||
## Conflict Resolution Flow
|
||||
|
||||
### Scenario: Concurrent Edits
|
||||
|
||||
Two users edit the same paragraph simultaneously.
|
||||
|
||||
**Initial state**:
|
||||
```
|
||||
Version 10: "The quick brown fox jumps over the lazy dog."
|
||||
```
|
||||
|
||||
**User A's edit** (version 11):
|
||||
```
|
||||
"The quick brown fox jumps over the very lazy dog."
|
||||
```
|
||||
*Inserts "very " at position 40*
|
||||
|
||||
**User B's edit** (also from version 10):
|
||||
```
|
||||
"The quick red fox jumps over the lazy dog."
|
||||
```
|
||||
*Replaces "brown" with "red" at position 10*
|
||||
|
||||
### Server Processing
|
||||
|
||||
1. **Receive User A's operation**:
|
||||
- Base: version 10
|
||||
- Operation: Insert("very ", position=40)
|
||||
- Apply to database → version 11
|
||||
|
||||
2. **Receive User B's operation**:
|
||||
- Base: version 10
|
||||
- Operation: Replace("brown"→"red", position=10)
|
||||
- **Conflict detected**: Base is version 10, but current is version 11
|
||||
|
||||
3. **Transform User B's operation**:
|
||||
- Transform against User A's operation
|
||||
- Adjust positions/content as needed
|
||||
- Apply transformed operation → version 12
|
||||
|
||||
4. **Broadcast updates**:
|
||||
- Send User A's operation to User B
|
||||
- Send transformed User B's operation to User A
|
||||
|
||||
### Final Result
|
||||
|
||||
```
|
||||
Version 12: "The quick red fox jumps over the very lazy dog."
|
||||
```
|
||||
|
||||
Both edits are preserved in the final document.
|
||||
|
||||
## Edge Cases
|
||||
|
||||
### 1. Delete vs Insert Conflict
|
||||
|
||||
**Scenario**: User A deletes a paragraph while User B edits it.
|
||||
|
||||
**Resolution**:
|
||||
- OT algorithm prioritizes preservation of content
|
||||
- Insert operation is transformed to account for deletion
|
||||
- Typically results in inserted content appearing nearby
|
||||
|
||||
**Example**:
|
||||
```
|
||||
Base: "Line 1\nLine 2\nLine 3"
|
||||
|
||||
User A: Delete Line 2 → "Line 1\nLine 3"
|
||||
User B: Edit Line 2 → "Line 1\nLine 2 modified\nLine 3"
|
||||
|
||||
Result: "Line 1\nLine 2 modified\nLine 3"
|
||||
```
|
||||
(Insert takes precedence, preserving user content)
|
||||
|
||||
### 2. Overlapping Edits
|
||||
|
||||
**Scenario**: Two users edit overlapping regions.
|
||||
|
||||
**Resolution**:
|
||||
- OT splits operations into non-overlapping segments
|
||||
- Applies each segment independently
|
||||
- Merges results
|
||||
|
||||
### 3. Delete vs Delete
|
||||
|
||||
**Scenario**: Two users delete overlapping text.
|
||||
|
||||
**Resolution**:
|
||||
- Deletes are merged
|
||||
- Final result has the union of deleted ranges removed
|
||||
|
||||
### 4. Network Partitions
|
||||
|
||||
**Scenario**: Client loses connection, makes edits offline, reconnects.
|
||||
|
||||
**Resolution**:
|
||||
1. Client queues edits locally
|
||||
2. On reconnect, sends all queued operations
|
||||
3. Server applies OT against all operations that happened during partition
|
||||
4. Client receives transformed operations and applies
|
||||
|
||||
## Performance Characteristics
|
||||
|
||||
### Time Complexity
|
||||
|
||||
- **Single operation**: O(1) for most operations
|
||||
- **Transformation**: O(n) where n is operation size
|
||||
- **Conflict resolution**: O(m × n) where m is number of concurrent operations
|
||||
|
||||
### Space Complexity
|
||||
|
||||
- **Version history**: Grows with number of changes
|
||||
- **Cursors**: O(clients × vaults)
|
||||
- **Active operations**: Minimal (processed in real-time)
|
||||
|
||||
### Optimization
|
||||
|
||||
VaultLink optimizes for:
|
||||
- Small, frequent edits (typical typing patterns)
|
||||
- Text documents (not binary files)
|
||||
- Real-time processing (no batching delay)
|
||||
|
||||
## Limitations
|
||||
|
||||
### Binary Files
|
||||
|
||||
OT works best for text files. Binary files:
|
||||
- Cannot be meaningfully merged
|
||||
- Use last-write-wins strategy
|
||||
- May cause data loss on concurrent edits
|
||||
|
||||
**Workaround**: Avoid concurrent edits to binary files, or use versioning.
|
||||
|
||||
### Large Documents
|
||||
|
||||
Very large documents (> 1MB) may have:
|
||||
- Higher transformation costs
|
||||
- Slower sync times
|
||||
- Increased memory usage
|
||||
|
||||
**Workaround**: Split large documents or increase timeout settings.
|
||||
|
||||
### Complex Formatting
|
||||
|
||||
Markdown with complex structures may occasionally produce unexpected results:
|
||||
- Nested lists
|
||||
- Tables
|
||||
- Code blocks
|
||||
|
||||
**Workaround**: Manual cleanup if needed, or minimize concurrent edits to complex structures.
|
||||
|
||||
## Consistency Guarantees
|
||||
|
||||
### Strong Consistency
|
||||
|
||||
VaultLink provides **strong eventual consistency**:
|
||||
- All clients eventually converge to the same state
|
||||
- Operations applied in causal order
|
||||
- No data loss under normal operation
|
||||
|
||||
### Ordering Guarantees
|
||||
|
||||
- Operations from the same client are applied in order
|
||||
- Concurrent operations may be applied in any order
|
||||
- Final result is independent of operation order (commutative)
|
||||
|
||||
### Durability
|
||||
|
||||
- Operations are written to SQLite before acknowledgment
|
||||
- SQLite ACID guarantees protect against data loss
|
||||
- Clients retry failed uploads
|
||||
|
||||
## Comparison with Other Approaches
|
||||
|
||||
### Git-style Merging
|
||||
|
||||
| Aspect | Git Merge | VaultLink OT |
|
||||
|--------|-----------|--------------|
|
||||
| Real-time | No | Yes |
|
||||
| Manual conflict resolution | Yes | No |
|
||||
| Branching | Yes | No |
|
||||
| Automatic merge | Limited | Always |
|
||||
| Use case | Code changes | Collaborative documents |
|
||||
|
||||
### CRDTs (Conflict-free Replicated Data Types)
|
||||
|
||||
| Aspect | CRDTs | VaultLink OT |
|
||||
|--------|-------|--------------|
|
||||
| Server required | No | Yes |
|
||||
| Memory overhead | Higher | Lower |
|
||||
| Complexity | Higher | Lower |
|
||||
| Deletion handling | Complex (tombstones) | Simple |
|
||||
| Best for | Distributed systems | Centralized sync |
|
||||
|
||||
### Last Write Wins
|
||||
|
||||
| Aspect | LWW | VaultLink OT |
|
||||
|--------|-----|--------------|
|
||||
| Data loss | Yes | No |
|
||||
| Simplicity | High | Medium |
|
||||
| User experience | Poor | Excellent |
|
||||
| Performance | Best | Good |
|
||||
|
||||
## Algorithm Details
|
||||
|
||||
### Transformation Rules
|
||||
|
||||
When transforming operation `A` against operation `B`:
|
||||
|
||||
1. **Insert vs Insert**:
|
||||
- If positions equal: Order by client ID
|
||||
- If different positions: Adjust positions
|
||||
|
||||
2. **Insert vs Delete**:
|
||||
- If insert in deleted range: Shift insert position
|
||||
- If insert after delete: Adjust position by deleted length
|
||||
|
||||
3. **Delete vs Delete**:
|
||||
- If ranges overlap: Merge delete ranges
|
||||
- If ranges disjoint: Adjust positions
|
||||
|
||||
4. **Retain vs Any**:
|
||||
- Retain operations don't conflict
|
||||
- Simply adjust positions
|
||||
|
||||
### Transformation Example
|
||||
|
||||
```rust
|
||||
// Pseudo-code for transformation
|
||||
fn transform(op_a: Operation, op_b: Operation) -> (Operation, Operation) {
|
||||
match (op_a, op_b) {
|
||||
(Insert(pos_a, text_a), Insert(pos_b, text_b)) => {
|
||||
if pos_a < pos_b {
|
||||
(op_a, Insert(pos_b + text_a.len(), text_b))
|
||||
} else if pos_a > pos_b {
|
||||
(Insert(pos_a + text_b.len(), text_a), op_b)
|
||||
} else {
|
||||
// Same position, use client ID to break tie
|
||||
if client_id_a < client_id_b {
|
||||
(op_a, Insert(pos_b + text_a.len(), text_b))
|
||||
} else {
|
||||
(Insert(pos_a + text_b.len(), text_a), op_b)
|
||||
}
|
||||
}
|
||||
}
|
||||
// ... other cases
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
### For Smooth Collaboration
|
||||
|
||||
1. **Small edits**: Make small, focused changes for easier merging
|
||||
2. **Coordinate major changes**: Discuss large refactors with team
|
||||
3. **Monitor sync status**: Ensure changes are uploaded before signing off
|
||||
4. **Test conflict resolution**: Verify behavior matches expectations
|
||||
|
||||
### For Developers
|
||||
|
||||
1. **Text files preferred**: OT works best on text
|
||||
2. **Limit file sizes**: Keep documents reasonably sized
|
||||
3. **Binary files**: Use versioning or avoid concurrent edits
|
||||
4. **Testing**: Test concurrent edit scenarios thoroughly
|
||||
|
||||
## Further Reading
|
||||
|
||||
- [reconcile-text library](https://crates.io/crates/reconcile-text)
|
||||
- [Operational Transformation FAQ](https://en.wikipedia.org/wiki/Operational_transformation)
|
||||
- [Data flow architecture →](/architecture/data-flow)
|
||||
581
docs/config/advanced.md
Normal file
581
docs/config/advanced.md
Normal file
|
|
@ -0,0 +1,581 @@
|
|||
# Advanced Configuration
|
||||
|
||||
Advanced topics for optimizing and customizing your VaultLink deployment.
|
||||
|
||||
## Database Optimization
|
||||
|
||||
### SQLite Tuning
|
||||
|
||||
While VaultLink handles most SQLite configuration automatically, you can optimize for specific workloads.
|
||||
|
||||
#### WAL Mode
|
||||
|
||||
VaultLink uses Write-Ahead Logging (WAL) mode by default for better concurrency.
|
||||
|
||||
**Benefits**:
|
||||
- Readers don't block writers
|
||||
- Writers don't block readers
|
||||
- Better performance for concurrent access
|
||||
|
||||
**Maintenance**:
|
||||
```bash
|
||||
# Checkpoint WAL to main database (run periodically)
|
||||
sqlite3 databases/vault.db "PRAGMA wal_checkpoint(TRUNCATE);"
|
||||
```
|
||||
|
||||
#### Database Size Management
|
||||
|
||||
Over time, databases can grow with version history:
|
||||
|
||||
```bash
|
||||
# Check database size
|
||||
du -h databases/*.db
|
||||
|
||||
# Vacuum to reclaim space (offline only)
|
||||
sqlite3 databases/vault.db "VACUUM;"
|
||||
|
||||
# Analyze for query optimization
|
||||
sqlite3 databases/vault.db "ANALYZE;"
|
||||
```
|
||||
|
||||
**Schedule maintenance**:
|
||||
```bash
|
||||
#!/bin/bash
|
||||
# monthly-maintenance.sh
|
||||
|
||||
for db in databases/*.db; do
|
||||
echo "Optimizing $db"
|
||||
sqlite3 "$db" "PRAGMA optimize;"
|
||||
sqlite3 "$db" "PRAGMA wal_checkpoint(TRUNCATE);"
|
||||
done
|
||||
```
|
||||
|
||||
### Version History Cleanup
|
||||
|
||||
To limit database growth, implement version history pruning (requires custom script):
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
# prune-old-versions.sh
|
||||
# Keep only last 100 versions per document
|
||||
|
||||
for db in databases/*.db; do
|
||||
sqlite3 "$db" <<EOF
|
||||
DELETE FROM versions
|
||||
WHERE id NOT IN (
|
||||
SELECT id FROM versions
|
||||
WHERE document_id = versions.document_id
|
||||
ORDER BY version DESC
|
||||
LIMIT 100
|
||||
);
|
||||
EOF
|
||||
done
|
||||
```
|
||||
|
||||
## Performance Tuning
|
||||
|
||||
### Connection Pool Sizing
|
||||
|
||||
Calculate optimal `max_connections_per_vault`:
|
||||
|
||||
```
|
||||
max_connections = (concurrent_users × avg_operations_per_user) + buffer
|
||||
```
|
||||
|
||||
**Example**:
|
||||
- 20 concurrent users
|
||||
- 2 operations per user on average
|
||||
- 25% buffer
|
||||
|
||||
```
|
||||
max_connections = (20 × 2) × 1.25 = 50
|
||||
```
|
||||
|
||||
### Timeout Configuration
|
||||
|
||||
Adjust timeouts based on network characteristics:
|
||||
|
||||
**Fast local network**:
|
||||
```yaml
|
||||
database:
|
||||
cursor_timeout_seconds: 30
|
||||
|
||||
server:
|
||||
response_timeout_seconds: 30
|
||||
```
|
||||
|
||||
**Slow or unreliable network**:
|
||||
```yaml
|
||||
database:
|
||||
cursor_timeout_seconds: 180
|
||||
|
||||
server:
|
||||
response_timeout_seconds: 120
|
||||
```
|
||||
|
||||
**Mobile clients**:
|
||||
```yaml
|
||||
database:
|
||||
cursor_timeout_seconds: 300 # Longer for intermittent connections
|
||||
|
||||
server:
|
||||
response_timeout_seconds: 180
|
||||
```
|
||||
|
||||
## Reverse Proxy Configuration
|
||||
|
||||
### Nginx with SSL
|
||||
|
||||
Complete Nginx configuration for production:
|
||||
|
||||
```nginx
|
||||
# Rate limiting
|
||||
limit_req_zone $binary_remote_addr zone=vaultlink:10m rate=10r/s;
|
||||
|
||||
upstream vaultlink {
|
||||
server localhost:3000;
|
||||
keepalive 32;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 443 ssl http2;
|
||||
server_name sync.example.com;
|
||||
|
||||
ssl_certificate /etc/letsencrypt/live/sync.example.com/fullchain.pem;
|
||||
ssl_certificate_key /etc/letsencrypt/live/sync.example.com/privkey.pem;
|
||||
|
||||
# SSL security settings
|
||||
ssl_protocols TLSv1.2 TLSv1.3;
|
||||
ssl_ciphers HIGH:!aNULL:!MD5;
|
||||
ssl_prefer_server_ciphers on;
|
||||
|
||||
# HSTS
|
||||
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
|
||||
|
||||
# Rate limiting
|
||||
limit_req zone=vaultlink burst=20 nodelay;
|
||||
|
||||
# Client body size (match server config)
|
||||
client_max_body_size 512M;
|
||||
|
||||
# Timeouts
|
||||
proxy_connect_timeout 90s;
|
||||
proxy_send_timeout 90s;
|
||||
proxy_read_timeout 3600s; # WebSocket long-lived connections
|
||||
|
||||
# WebSocket headers
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
# Disable buffering for WebSocket
|
||||
proxy_buffering off;
|
||||
|
||||
location / {
|
||||
proxy_pass http://vaultlink;
|
||||
}
|
||||
|
||||
# Health check endpoint
|
||||
location /health {
|
||||
proxy_pass http://vaultlink/vaults/health/ping;
|
||||
access_log off;
|
||||
}
|
||||
}
|
||||
|
||||
# Redirect HTTP to HTTPS
|
||||
server {
|
||||
listen 80;
|
||||
server_name sync.example.com;
|
||||
return 301 https://$server_name$request_uri;
|
||||
}
|
||||
```
|
||||
|
||||
### Caddy with Auto SSL
|
||||
|
||||
Caddy handles SSL automatically:
|
||||
|
||||
```caddy
|
||||
sync.example.com {
|
||||
reverse_proxy localhost:3000 {
|
||||
# WebSocket support
|
||||
header_up X-Real-IP {remote_host}
|
||||
header_up X-Forwarded-For {remote_host}
|
||||
header_up X-Forwarded-Proto {scheme}
|
||||
|
||||
# Timeouts
|
||||
transport http {
|
||||
read_timeout 3600s
|
||||
write_timeout 90s
|
||||
}
|
||||
}
|
||||
|
||||
# Rate limiting (requires caddy-rate-limit plugin)
|
||||
rate_limit {
|
||||
zone dynamic {
|
||||
match {
|
||||
remote_ip
|
||||
}
|
||||
rate 10r/s
|
||||
burst 20
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Traefik Configuration
|
||||
|
||||
Using Docker labels:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
vaultlink-server:
|
||||
image: ghcr.io/schmelczer/vault-link-server:latest
|
||||
labels:
|
||||
- "traefik.enable=true"
|
||||
- "traefik.http.routers.vaultlink.rule=Host(`sync.example.com`)"
|
||||
- "traefik.http.routers.vaultlink.entrypoints=websecure"
|
||||
- "traefik.http.routers.vaultlink.tls.certresolver=letsencrypt"
|
||||
- "traefik.http.services.vaultlink.loadbalancer.server.port=3000"
|
||||
# Middleware for timeouts
|
||||
- "traefik.http.middlewares.vaultlink-timeout.timeout.request=3600s"
|
||||
```
|
||||
|
||||
## Docker Optimizations
|
||||
|
||||
### Resource Limits
|
||||
|
||||
Limit container resources:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
vaultlink-server:
|
||||
image: ghcr.io/schmelczer/vault-link-server:latest
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '2.0'
|
||||
memory: 4G
|
||||
reservations:
|
||||
cpus: '1.0'
|
||||
memory: 2G
|
||||
```
|
||||
|
||||
### Logging Configuration
|
||||
|
||||
Optimize Docker logging:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
vaultlink-server:
|
||||
image: ghcr.io/schmelczer/vault-link-server:latest
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "50m"
|
||||
max-file: "5"
|
||||
```
|
||||
|
||||
### Volume Optimization
|
||||
|
||||
Use named volumes for better performance:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
vaultlink-server:
|
||||
image: ghcr.io/schmelczer/vault-link-server:latest
|
||||
volumes:
|
||||
- vaultlink-data:/data
|
||||
- vaultlink-logs:/data/logs
|
||||
|
||||
volumes:
|
||||
vaultlink-data:
|
||||
driver: local
|
||||
driver_opts:
|
||||
type: none
|
||||
o: bind
|
||||
device: /mnt/fast-ssd/vaultlink
|
||||
vaultlink-logs:
|
||||
driver: local
|
||||
```
|
||||
|
||||
## High Availability
|
||||
|
||||
### Health Checks
|
||||
|
||||
Comprehensive health monitoring:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
vaultlink-server:
|
||||
image: ghcr.io/schmelczer/vault-link-server:latest
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -f http://localhost:3000/vaults/health/ping || exit 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
```
|
||||
|
||||
Monitor health in production:
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
# health-monitor.sh
|
||||
|
||||
while true; do
|
||||
if ! curl -sf http://localhost:3000/vaults/health/ping > /dev/null; then
|
||||
echo "Health check failed at $(date)" | mail -s "VaultLink Down" admin@example.com
|
||||
# Optionally restart
|
||||
# docker restart vaultlink-server
|
||||
fi
|
||||
sleep 30
|
||||
done
|
||||
```
|
||||
|
||||
### Backup Automation
|
||||
|
||||
Automated backup script:
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
# backup-vaultlink.sh
|
||||
|
||||
BACKUP_DIR="/backup/vaultlink"
|
||||
DATA_DIR="/data"
|
||||
DATE=$(date +%Y%m%d-%H%M%S)
|
||||
RETENTION_DAYS=30
|
||||
|
||||
# Create backup directory
|
||||
mkdir -p "$BACKUP_DIR/$DATE"
|
||||
|
||||
# Backup databases (with WAL checkpoint)
|
||||
for db in "$DATA_DIR"/databases/*.db; do
|
||||
sqlite3 "$db" "PRAGMA wal_checkpoint(TRUNCATE);"
|
||||
cp "$db" "$BACKUP_DIR/$DATE/"
|
||||
[ -f "${db}-wal" ] && cp "${db}-wal" "$BACKUP_DIR/$DATE/"
|
||||
[ -f "${db}-shm" ] && cp "${db}-shm" "$BACKUP_DIR/$DATE/"
|
||||
done
|
||||
|
||||
# Backup configuration
|
||||
cp "$DATA_DIR/config.yml" "$BACKUP_DIR/$DATE/"
|
||||
|
||||
# Compress backup
|
||||
tar -czf "$BACKUP_DIR/vaultlink-$DATE.tar.gz" -C "$BACKUP_DIR" "$DATE"
|
||||
rm -rf "$BACKUP_DIR/$DATE"
|
||||
|
||||
# Clean old backups
|
||||
find "$BACKUP_DIR" -name "vaultlink-*.tar.gz" -mtime +$RETENTION_DAYS -delete
|
||||
|
||||
# Upload to remote storage (optional)
|
||||
# rclone copy "$BACKUP_DIR/vaultlink-$DATE.tar.gz" remote:backups/
|
||||
```
|
||||
|
||||
Schedule with cron:
|
||||
```cron
|
||||
0 2 * * * /opt/vaultlink/backup-vaultlink.sh
|
||||
```
|
||||
|
||||
### Restore from Backup
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
# restore-vaultlink.sh
|
||||
|
||||
BACKUP_FILE="$1"
|
||||
DATA_DIR="/data"
|
||||
|
||||
if [ -z "$BACKUP_FILE" ]; then
|
||||
echo "Usage: $0 <backup-file.tar.gz>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Stop server
|
||||
docker stop vaultlink-server
|
||||
|
||||
# Extract backup
|
||||
tar -xzf "$BACKUP_FILE" -C /tmp/
|
||||
BACKUP_DATE=$(basename "$BACKUP_FILE" .tar.gz | cut -d- -f2-)
|
||||
|
||||
# Restore databases
|
||||
cp /tmp/"$BACKUP_DATE"/databases/*.db "$DATA_DIR/databases/"
|
||||
|
||||
# Restore config (careful!)
|
||||
# cp /tmp/$BACKUP_DATE/config.yml "$DATA_DIR/"
|
||||
|
||||
# Cleanup
|
||||
rm -rf /tmp/"$BACKUP_DATE"
|
||||
|
||||
# Start server
|
||||
docker start vaultlink-server
|
||||
|
||||
echo "Restore complete. Check server logs."
|
||||
```
|
||||
|
||||
## Monitoring and Metrics
|
||||
|
||||
### Prometheus Metrics
|
||||
|
||||
While VaultLink doesn't expose metrics natively, monitor Docker:
|
||||
|
||||
```yaml
|
||||
# docker-compose.yml
|
||||
services:
|
||||
vaultlink-server:
|
||||
image: ghcr.io/schmelczer/vault-link-server:latest
|
||||
labels:
|
||||
- "prometheus.io/scrape=true"
|
||||
- "prometheus.io/port=3000"
|
||||
|
||||
cadvisor:
|
||||
image: gcr.io/cadvisor/cadvisor:latest
|
||||
volumes:
|
||||
- /:/rootfs:ro
|
||||
- /var/run:/var/run:ro
|
||||
- /sys:/sys:ro
|
||||
- /var/lib/docker/:/var/lib/docker:ro
|
||||
ports:
|
||||
- 8080:8080
|
||||
```
|
||||
|
||||
### Log Analysis
|
||||
|
||||
Analyze logs for insights:
|
||||
|
||||
```bash
|
||||
# Most active users
|
||||
grep "authenticated" logs/*.log | cut -d"'" -f2 | sort | uniq -c | sort -rn
|
||||
|
||||
# Failed authentications by IP
|
||||
grep "Authentication failed" logs/*.log | grep -oP '\d+\.\d+\.\d+\.\d+' | sort | uniq -c | sort -rn
|
||||
|
||||
# Upload activity
|
||||
grep "Upload:" logs/*.log | wc -l
|
||||
|
||||
# Average files per vault
|
||||
grep "Sync complete" logs/*.log | grep -oP '\d+ files' | cut -d' ' -f1 | awk '{sum+=$1; count++} END {print sum/count}'
|
||||
```
|
||||
|
||||
### Alerting
|
||||
|
||||
Simple alerting with cron:
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
# alert-errors.sh
|
||||
|
||||
ERROR_THRESHOLD=10
|
||||
ERROR_COUNT=$(grep -c "ERROR" logs/latest.log)
|
||||
|
||||
if [ "$ERROR_COUNT" -gt "$ERROR_THRESHOLD" ]; then
|
||||
echo "VaultLink has $ERROR_COUNT errors in the last hour" | \
|
||||
mail -s "VaultLink Alert" admin@example.com
|
||||
fi
|
||||
```
|
||||
|
||||
## Security Hardening
|
||||
|
||||
### Network Isolation
|
||||
|
||||
Run VaultLink in isolated network:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
vaultlink-server:
|
||||
image: ghcr.io/schmelczer/vault-link-server:latest
|
||||
networks:
|
||||
- vaultlink-internal
|
||||
- proxy-external
|
||||
|
||||
networks:
|
||||
vaultlink-internal:
|
||||
internal: true
|
||||
proxy-external:
|
||||
driver: bridge
|
||||
```
|
||||
|
||||
### Read-Only Root Filesystem
|
||||
|
||||
Run with read-only root (mount writable volumes for data):
|
||||
|
||||
```yaml
|
||||
services:
|
||||
vaultlink-server:
|
||||
image: ghcr.io/schmelczer/vault-link-server:latest
|
||||
read_only: true
|
||||
volumes:
|
||||
- ./data:/data
|
||||
- /tmp
|
||||
```
|
||||
|
||||
### Drop Capabilities
|
||||
|
||||
Run with minimal privileges:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
vaultlink-server:
|
||||
image: ghcr.io/schmelczer/vault-link-server:latest
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
cap_drop:
|
||||
- ALL
|
||||
```
|
||||
|
||||
## Migration
|
||||
|
||||
### Moving to New Server
|
||||
|
||||
1. **Backup on old server**:
|
||||
```bash
|
||||
./backup-vaultlink.sh
|
||||
```
|
||||
|
||||
2. **Transfer backup**:
|
||||
```bash
|
||||
scp vaultlink-backup.tar.gz new-server:/tmp/
|
||||
```
|
||||
|
||||
3. **Restore on new server**:
|
||||
```bash
|
||||
./restore-vaultlink.sh /tmp/vaultlink-backup.tar.gz
|
||||
```
|
||||
|
||||
4. **Update DNS/clients** to point to new server
|
||||
|
||||
5. **Verify sync** on all clients
|
||||
|
||||
### Version Upgrades
|
||||
|
||||
```bash
|
||||
# Pull latest image
|
||||
docker pull ghcr.io/schmelczer/vault-link-server:latest
|
||||
|
||||
# Backup first
|
||||
./backup-vaultlink.sh
|
||||
|
||||
# Stop old container
|
||||
docker stop vaultlink-server
|
||||
docker rm vaultlink-server
|
||||
|
||||
# Start with new image
|
||||
docker run -d \
|
||||
--name vaultlink-server \
|
||||
--restart unless-stopped \
|
||||
-p 3000:3000 \
|
||||
-v ./data:/data \
|
||||
ghcr.io/schmelczer/vault-link-server:latest \
|
||||
/app/sync_server /data/config.yml
|
||||
|
||||
# Check logs
|
||||
docker logs -f vaultlink-server
|
||||
```
|
||||
|
||||
## Next Steps
|
||||
|
||||
- [Understand the architecture →](/architecture/)
|
||||
- [Deploy the server →](/guide/server-setup)
|
||||
- [Configure clients →](/guide/obsidian-plugin)
|
||||
530
docs/config/authentication.md
Normal file
530
docs/config/authentication.md
Normal file
|
|
@ -0,0 +1,530 @@
|
|||
# Authentication Configuration
|
||||
|
||||
VaultLink uses token-based authentication with per-user vault access control. This guide covers all authentication and authorization options.
|
||||
|
||||
## Overview
|
||||
|
||||
Authentication in VaultLink:
|
||||
- **Token-based**: Users authenticate with secure tokens
|
||||
- **Configured in YAML**: All users defined in `config.yml`
|
||||
- **Vault-level access**: Control which vaults each user can access
|
||||
- **No password hashing**: Tokens are treated as secrets
|
||||
|
||||
## Basic Configuration
|
||||
|
||||
```yaml
|
||||
users:
|
||||
user_configs:
|
||||
- name: alice
|
||||
token: alice-secure-token-here
|
||||
vault_access:
|
||||
type: allow_access_to_all
|
||||
```
|
||||
|
||||
## User Configuration Fields
|
||||
|
||||
### `name`
|
||||
|
||||
**Type**: String
|
||||
**Required**: Yes
|
||||
|
||||
Human-readable identifier for the user. Used in logs and auditing.
|
||||
|
||||
```yaml
|
||||
- name: alice
|
||||
```
|
||||
|
||||
**Notes**:
|
||||
- Must be unique across all users
|
||||
- Used for identification only, not authentication
|
||||
- Appears in server logs
|
||||
- Can be any string (e.g., email, username)
|
||||
|
||||
### `token`
|
||||
|
||||
**Type**: String
|
||||
**Required**: Yes
|
||||
|
||||
Authentication token for the user. Must be kept secret.
|
||||
|
||||
```yaml
|
||||
- token: 1a2b3c4d5e6f7g8h9i0j...
|
||||
```
|
||||
|
||||
**Best practices**:
|
||||
- Generate with: `openssl rand -hex 32`
|
||||
- Minimum length: 32 characters
|
||||
- Use different token per user
|
||||
- Never commit to version control
|
||||
- Rotate periodically
|
||||
|
||||
**Example token generation**:
|
||||
```bash
|
||||
# Generate a secure token
|
||||
openssl rand -hex 32
|
||||
# Output: a7f3c9d1e8b2f4a6c3d9e1f7b8a4c2d6e9f1a3b7c5d8e2f4a6b9c3d1e8f7a4b2
|
||||
```
|
||||
|
||||
### `vault_access`
|
||||
|
||||
**Type**: Object
|
||||
**Required**: Yes
|
||||
|
||||
Defines which vaults the user can access.
|
||||
|
||||
**Three modes**:
|
||||
1. `allow_access_to_all`: Access to all vaults
|
||||
2. `allow_list`: Access to specific vaults only
|
||||
3. `deny_list`: Access to all vaults except specific ones
|
||||
|
||||
## Access Control Modes
|
||||
|
||||
### Allow Access to All
|
||||
|
||||
Grant access to every vault:
|
||||
|
||||
```yaml
|
||||
users:
|
||||
user_configs:
|
||||
- name: admin
|
||||
token: admin-token
|
||||
vault_access:
|
||||
type: allow_access_to_all
|
||||
```
|
||||
|
||||
**Use cases**:
|
||||
- Administrator accounts
|
||||
- Personal single-user deployments
|
||||
- Development/testing
|
||||
|
||||
### Allow List
|
||||
|
||||
Grant access only to specific vaults:
|
||||
|
||||
```yaml
|
||||
users:
|
||||
user_configs:
|
||||
- name: alice
|
||||
token: alice-token
|
||||
vault_access:
|
||||
type: allow_list
|
||||
allowed:
|
||||
- personal
|
||||
- shared-team
|
||||
- project-alpha
|
||||
```
|
||||
|
||||
**Use cases**:
|
||||
- Multi-user deployments
|
||||
- Restricted access scenarios
|
||||
- Separation of concerns
|
||||
|
||||
**Notes**:
|
||||
- User can only access listed vaults
|
||||
- Attempting to access other vaults returns authentication error
|
||||
- Empty list = no access to any vault
|
||||
|
||||
### Deny List
|
||||
|
||||
Grant access to all vaults except specific ones:
|
||||
|
||||
```yaml
|
||||
users:
|
||||
user_configs:
|
||||
- name: bob
|
||||
token: bob-token
|
||||
vault_access:
|
||||
type: deny_list
|
||||
denied:
|
||||
- restricted
|
||||
- admin-only
|
||||
```
|
||||
|
||||
**Use cases**:
|
||||
- Users with broad access except sensitive vaults
|
||||
- Simplify configuration when most vaults are accessible
|
||||
|
||||
**Notes**:
|
||||
- User can access any vault not in the deny list
|
||||
- Attempting to access denied vaults returns authentication error
|
||||
|
||||
## Multi-User Scenarios
|
||||
|
||||
### Personal Use (Single User)
|
||||
|
||||
```yaml
|
||||
users:
|
||||
user_configs:
|
||||
- name: me
|
||||
token: my-super-secret-token
|
||||
vault_access:
|
||||
type: allow_access_to_all
|
||||
```
|
||||
|
||||
### Small Team (Shared Vaults)
|
||||
|
||||
```yaml
|
||||
users:
|
||||
user_configs:
|
||||
- name: alice
|
||||
token: alice-token
|
||||
vault_access:
|
||||
type: allow_list
|
||||
allowed:
|
||||
- personal-alice
|
||||
- team-shared
|
||||
- name: bob
|
||||
token: bob-token
|
||||
vault_access:
|
||||
type: allow_list
|
||||
allowed:
|
||||
- personal-bob
|
||||
- team-shared
|
||||
- name: charlie
|
||||
token: charlie-token
|
||||
vault_access:
|
||||
type: allow_list
|
||||
allowed:
|
||||
- personal-charlie
|
||||
- team-shared
|
||||
```
|
||||
|
||||
### Organization (Mixed Access)
|
||||
|
||||
```yaml
|
||||
users:
|
||||
user_configs:
|
||||
- name: admin
|
||||
token: admin-token
|
||||
vault_access:
|
||||
type: allow_access_to_all
|
||||
|
||||
- name: developer
|
||||
token: dev-token
|
||||
vault_access:
|
||||
type: allow_list
|
||||
allowed:
|
||||
- engineering-docs
|
||||
- api-specs
|
||||
- shared
|
||||
|
||||
- name: designer
|
||||
token: design-token
|
||||
vault_access:
|
||||
type: allow_list
|
||||
allowed:
|
||||
- design-docs
|
||||
- brand-assets
|
||||
- shared
|
||||
|
||||
- name: readonly
|
||||
token: readonly-token
|
||||
vault_access:
|
||||
type: allow_list
|
||||
allowed:
|
||||
- public-wiki
|
||||
```
|
||||
|
||||
## Authentication Flow
|
||||
|
||||
### Connection
|
||||
|
||||
1. Client connects via WebSocket
|
||||
2. Client sends authentication message:
|
||||
```json
|
||||
{
|
||||
"type": "auth",
|
||||
"token": "user-token",
|
||||
"vault": "vault-name"
|
||||
}
|
||||
```
|
||||
3. Server validates:
|
||||
- Token exists in config
|
||||
- User has access to requested vault
|
||||
4. Server responds:
|
||||
- Success: Connection established
|
||||
- Failure: Connection closed with error
|
||||
|
||||
### Validation
|
||||
|
||||
Server checks:
|
||||
1. **Token match**: Token exists in `user_configs`
|
||||
2. **Vault access**: User has permission for vault
|
||||
3. **Connection limits**: Not exceeding `max_clients_per_vault`
|
||||
|
||||
### Errors
|
||||
|
||||
**Invalid token**:
|
||||
```
|
||||
Authentication failed: Invalid token
|
||||
```
|
||||
|
||||
**No vault access**:
|
||||
```
|
||||
Authentication failed: User does not have access to vault 'restricted'
|
||||
```
|
||||
|
||||
**Connection limit**:
|
||||
```
|
||||
Connection rejected: Maximum clients reached for vault
|
||||
```
|
||||
|
||||
## Security Best Practices
|
||||
|
||||
### Token Generation
|
||||
|
||||
Generate strong tokens:
|
||||
|
||||
```bash
|
||||
# 64 character hex token (256 bits)
|
||||
openssl rand -hex 32
|
||||
|
||||
# Base64 encoded (256 bits)
|
||||
openssl rand -base64 32
|
||||
|
||||
# UUID v4
|
||||
uuidgen
|
||||
```
|
||||
|
||||
### Token Storage
|
||||
|
||||
**In config file**:
|
||||
```yaml
|
||||
users:
|
||||
user_configs:
|
||||
- name: alice
|
||||
token: !ENV ALICE_TOKEN # Read from environment variable
|
||||
```
|
||||
|
||||
**Load from environment**:
|
||||
```bash
|
||||
export ALICE_TOKEN="$(openssl rand -hex 32)"
|
||||
./sync_server config.yml
|
||||
```
|
||||
|
||||
### Token Rotation
|
||||
|
||||
Periodically change tokens:
|
||||
|
||||
1. Generate new token
|
||||
2. Update `config.yml`
|
||||
3. Restart server
|
||||
4. Update clients with new token
|
||||
|
||||
### Token Revocation
|
||||
|
||||
To revoke access:
|
||||
1. Remove user from `config.yml`
|
||||
2. Restart server
|
||||
3. User's connections will be rejected
|
||||
|
||||
For immediate revocation:
|
||||
- Remove user from config
|
||||
- Restart server
|
||||
- Existing connections are terminated
|
||||
|
||||
## Access Patterns
|
||||
|
||||
### Read-Only Users
|
||||
|
||||
VaultLink doesn't distinguish read-only vs read-write. Implement via client:
|
||||
|
||||
```yaml
|
||||
# Server: Grant access
|
||||
users:
|
||||
user_configs:
|
||||
- name: readonly
|
||||
token: readonly-token
|
||||
vault_access:
|
||||
type: allow_list
|
||||
allowed:
|
||||
- public
|
||||
|
||||
# Client: Use CLI in read-only mode (mount vault read-only)
|
||||
docker run -v /vault:/vault:ro ...
|
||||
```
|
||||
|
||||
### Temporary Access
|
||||
|
||||
Grant temporary access:
|
||||
|
||||
1. Add user to config
|
||||
2. Set reminder to remove later
|
||||
3. Remove user when no longer needed
|
||||
4. Restart server
|
||||
|
||||
For automation:
|
||||
```bash
|
||||
# Add user with expiry comment
|
||||
echo " - name: temp-user # EXPIRES: 2024-12-31" >> config.yml
|
||||
echo " token: temp-token" >> config.yml
|
||||
```
|
||||
|
||||
### Shared Tokens (Not Recommended)
|
||||
|
||||
Multiple users sharing a token:
|
||||
- All appear as same user in logs
|
||||
- Can't revoke individual access
|
||||
- Security risk if one person leaves
|
||||
|
||||
**Instead**: Create separate users with same vault access.
|
||||
|
||||
## Monitoring
|
||||
|
||||
### Server Logs
|
||||
|
||||
Authentication events are logged:
|
||||
|
||||
```
|
||||
2024-01-01 12:00:00 INFO User 'alice' authenticated for vault 'personal'
|
||||
2024-01-01 12:00:05 WARN Authentication failed: Invalid token from 192.168.1.100
|
||||
2024-01-01 12:00:10 WARN User 'bob' denied access to vault 'restricted'
|
||||
```
|
||||
|
||||
### Audit Trail
|
||||
|
||||
Monitor authentication:
|
||||
|
||||
```bash
|
||||
# View authentication logs
|
||||
grep "authenticated" logs/*.log
|
||||
|
||||
# View failed authentications
|
||||
grep "Authentication failed" logs/*.log
|
||||
|
||||
# View access denials
|
||||
grep "denied access" logs/*.log
|
||||
```
|
||||
|
||||
## Advanced Scenarios
|
||||
|
||||
### Multiple Servers
|
||||
|
||||
Same user across multiple server instances:
|
||||
|
||||
```yaml
|
||||
# Server 1 config.yml
|
||||
users:
|
||||
user_configs:
|
||||
- name: alice
|
||||
token: alice-global-token
|
||||
vault_access:
|
||||
type: allow_list
|
||||
allowed:
|
||||
- vault-1
|
||||
- vault-2
|
||||
|
||||
# Server 2 config.yml
|
||||
users:
|
||||
user_configs:
|
||||
- name: alice
|
||||
token: alice-global-token # Same token
|
||||
vault_access:
|
||||
type: allow_list
|
||||
allowed:
|
||||
- vault-3
|
||||
- vault-4
|
||||
```
|
||||
|
||||
### Service Accounts
|
||||
|
||||
Tokens for automated systems:
|
||||
|
||||
```yaml
|
||||
users:
|
||||
user_configs:
|
||||
- name: backup-service
|
||||
token: backup-service-token
|
||||
vault_access:
|
||||
type: allow_access_to_all
|
||||
|
||||
- name: ci-pipeline
|
||||
token: ci-token
|
||||
vault_access:
|
||||
type: allow_list
|
||||
allowed:
|
||||
- documentation
|
||||
|
||||
- name: monitoring
|
||||
token: monitoring-token
|
||||
vault_access:
|
||||
type: allow_list
|
||||
allowed:
|
||||
- metrics
|
||||
```
|
||||
|
||||
### Dynamic Vault Access
|
||||
|
||||
VaultLink doesn't support runtime user management. To change access:
|
||||
|
||||
1. Update `config.yml`
|
||||
2. Restart server
|
||||
3. Users reconnect automatically
|
||||
|
||||
For frequent changes, consider:
|
||||
- Over-provision access (deny list)
|
||||
- Use external authentication proxy
|
||||
- Script config updates + reload
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Can't connect
|
||||
|
||||
**Check token**:
|
||||
```bash
|
||||
# Verify token in config matches client
|
||||
grep "token:" config.yml
|
||||
```
|
||||
|
||||
**Check vault name**:
|
||||
```bash
|
||||
# Ensure vault is in allowed list
|
||||
grep -A 5 "name: alice" config.yml
|
||||
```
|
||||
|
||||
**Check server logs**:
|
||||
```bash
|
||||
tail -f logs/*.log | grep -i auth
|
||||
```
|
||||
|
||||
### Access denied
|
||||
|
||||
**Verify vault access**:
|
||||
```yaml
|
||||
# Check user's vault_access configuration
|
||||
users:
|
||||
user_configs:
|
||||
- name: alice
|
||||
vault_access:
|
||||
type: allow_list
|
||||
allowed:
|
||||
- vault-name # Must match exactly
|
||||
```
|
||||
|
||||
**Case sensitivity**:
|
||||
- Vault names are case-sensitive
|
||||
- `Vault` ≠ `vault`
|
||||
- Ensure exact match in config and client
|
||||
|
||||
### Token not working
|
||||
|
||||
**Check for typos**:
|
||||
- Extra spaces
|
||||
- Hidden characters
|
||||
- Wrong quotes in YAML
|
||||
|
||||
**Regenerate token**:
|
||||
```bash
|
||||
# Generate new token
|
||||
openssl rand -hex 32
|
||||
|
||||
# Update config
|
||||
# Restart server
|
||||
# Update client
|
||||
```
|
||||
|
||||
## Next Steps
|
||||
|
||||
- [Server configuration reference →](/config/server)
|
||||
- [Advanced configuration →](/config/advanced)
|
||||
- [Deploy the server →](/guide/server-setup)
|
||||
470
docs/config/server.md
Normal file
470
docs/config/server.md
Normal file
|
|
@ -0,0 +1,470 @@
|
|||
# Server Configuration
|
||||
|
||||
Complete reference for configuring the VaultLink sync server via `config.yml`.
|
||||
|
||||
## Configuration File Format
|
||||
|
||||
The server is configured using a YAML file passed as a command-line argument:
|
||||
|
||||
```bash
|
||||
/app/sync_server /path/to/config.yml
|
||||
```
|
||||
|
||||
## Complete Example
|
||||
|
||||
```yaml
|
||||
database:
|
||||
databases_directory_path: databases
|
||||
max_connections_per_vault: 12
|
||||
cursor_timeout_seconds: 60
|
||||
|
||||
server:
|
||||
host: 0.0.0.0
|
||||
port: 3000
|
||||
max_body_size_mb: 512
|
||||
max_clients_per_vault: 256
|
||||
response_timeout_seconds: 60
|
||||
|
||||
users:
|
||||
user_configs:
|
||||
- name: admin
|
||||
token: your-secure-random-token
|
||||
vault_access:
|
||||
type: allow_access_to_all
|
||||
- name: alice
|
||||
token: alice-token
|
||||
vault_access:
|
||||
type: allow_list
|
||||
allowed:
|
||||
- personal
|
||||
- shared
|
||||
- name: bob
|
||||
token: bob-token
|
||||
vault_access:
|
||||
type: deny_list
|
||||
denied:
|
||||
- restricted
|
||||
|
||||
logging:
|
||||
log_directory: logs
|
||||
log_rotation: 7days
|
||||
```
|
||||
|
||||
## Database Section
|
||||
|
||||
### `databases_directory_path`
|
||||
|
||||
**Type**: String
|
||||
**Required**: Yes
|
||||
**Default**: None
|
||||
|
||||
Directory where SQLite database files are stored. One database file per vault.
|
||||
|
||||
```yaml
|
||||
database:
|
||||
databases_directory_path: /data/databases
|
||||
```
|
||||
|
||||
The directory structure:
|
||||
```
|
||||
databases/
|
||||
├── vault-1.db
|
||||
├── vault-2.db
|
||||
└── personal.db
|
||||
```
|
||||
|
||||
**Notes**:
|
||||
- Path is relative to working directory or absolute
|
||||
- Directory must be writable by the server process
|
||||
- Ensure adequate disk space for vault data
|
||||
- Back up this directory regularly
|
||||
|
||||
### `max_connections_per_vault`
|
||||
|
||||
**Type**: Integer
|
||||
**Required**: Yes
|
||||
**Default**: None
|
||||
**Recommended**: 12
|
||||
|
||||
Maximum concurrent database connections per vault.
|
||||
|
||||
```yaml
|
||||
database:
|
||||
max_connections_per_vault: 12
|
||||
```
|
||||
|
||||
**Tuning**:
|
||||
- Higher values: Better performance under load
|
||||
- Lower values: Less memory usage
|
||||
- Typical range: 8-20
|
||||
- Consider: Number of concurrent users × average operations per user
|
||||
|
||||
### `cursor_timeout_seconds`
|
||||
|
||||
**Type**: Integer
|
||||
**Required**: Yes
|
||||
**Default**: None
|
||||
**Recommended**: 60
|
||||
|
||||
How long to keep database cursors alive for inactive clients.
|
||||
|
||||
```yaml
|
||||
database:
|
||||
cursor_timeout_seconds: 60
|
||||
```
|
||||
|
||||
**Notes**:
|
||||
- Cursors track client sync state
|
||||
- Timeout too short: Clients may need to re-sync frequently
|
||||
- Timeout too long: More memory usage
|
||||
- Typical range: 30-300 seconds
|
||||
|
||||
## Server Section
|
||||
|
||||
### `host`
|
||||
|
||||
**Type**: String
|
||||
**Required**: Yes
|
||||
**Default**: None
|
||||
|
||||
Network interface to bind the server to.
|
||||
|
||||
```yaml
|
||||
server:
|
||||
host: 0.0.0.0 # All interfaces
|
||||
# OR
|
||||
host: 127.0.0.1 # Localhost only
|
||||
# OR
|
||||
host: 192.168.1.100 # Specific interface
|
||||
```
|
||||
|
||||
**Common values**:
|
||||
- `0.0.0.0`: Listen on all network interfaces (production)
|
||||
- `127.0.0.1`: Listen on localhost only (development/testing)
|
||||
- Specific IP: Listen on specific interface
|
||||
|
||||
### `port`
|
||||
|
||||
**Type**: Integer
|
||||
**Required**: Yes
|
||||
**Default**: None
|
||||
**Recommended**: 3000
|
||||
|
||||
TCP port to listen on.
|
||||
|
||||
```yaml
|
||||
server:
|
||||
port: 3000
|
||||
```
|
||||
|
||||
**Notes**:
|
||||
- Must be available (not in use)
|
||||
- Privileged ports (< 1024) require root
|
||||
- Common ports: 3000, 8080, 8888
|
||||
- Configure firewall to allow this port
|
||||
|
||||
### `max_body_size_mb`
|
||||
|
||||
**Type**: Integer
|
||||
**Required**: Yes
|
||||
**Default**: None
|
||||
**Recommended**: 512
|
||||
|
||||
Maximum size of HTTP request body in megabytes.
|
||||
|
||||
```yaml
|
||||
server:
|
||||
max_body_size_mb: 512
|
||||
```
|
||||
|
||||
**Usage**:
|
||||
- Limits file upload size
|
||||
- Prevents memory exhaustion attacks
|
||||
- Must be larger than largest expected file
|
||||
- Consider client `max_file_size_mb` settings
|
||||
|
||||
**Tuning**:
|
||||
- Small vaults (mostly text): 100 MB
|
||||
- Medium vaults (some images): 512 MB
|
||||
- Large vaults (many images/PDFs): 1024+ MB
|
||||
|
||||
### `max_clients_per_vault`
|
||||
|
||||
**Type**: Integer
|
||||
**Required**: Yes
|
||||
**Default**: None
|
||||
**Recommended**: 256
|
||||
|
||||
Maximum concurrent clients per vault.
|
||||
|
||||
```yaml
|
||||
server:
|
||||
max_clients_per_vault: 256
|
||||
```
|
||||
|
||||
**Notes**:
|
||||
- Limits concurrent WebSocket connections
|
||||
- Prevents resource exhaustion
|
||||
- Consider expected number of users
|
||||
- Each client uses memory and file descriptors
|
||||
|
||||
**Scaling**:
|
||||
- Personal use: 10-50
|
||||
- Small team: 50-100
|
||||
- Large team: 100-500
|
||||
|
||||
### `response_timeout_seconds`
|
||||
|
||||
**Type**: Integer
|
||||
**Required**: Yes
|
||||
**Default**: None
|
||||
**Recommended**: 60
|
||||
|
||||
Maximum time to wait for client responses.
|
||||
|
||||
```yaml
|
||||
server:
|
||||
response_timeout_seconds: 60
|
||||
```
|
||||
|
||||
**Usage**:
|
||||
- Timeout for HTTP requests
|
||||
- Timeout for WebSocket operations
|
||||
- Clients disconnected if unresponsive
|
||||
|
||||
**Tuning**:
|
||||
- Fast networks: 30 seconds
|
||||
- Slow networks: 90-120 seconds
|
||||
- Large file uploads: Increase proportionally
|
||||
|
||||
## Users Section
|
||||
|
||||
See [Authentication Configuration →](/config/authentication) for detailed user configuration.
|
||||
|
||||
## Logging Section
|
||||
|
||||
### `log_directory`
|
||||
|
||||
**Type**: String
|
||||
**Required**: Yes
|
||||
**Default**: None
|
||||
|
||||
Directory where log files are written.
|
||||
|
||||
```yaml
|
||||
logging:
|
||||
log_directory: /data/logs
|
||||
# OR
|
||||
log_directory: logs # Relative to working directory
|
||||
```
|
||||
|
||||
**Notes**:
|
||||
- Path is relative to working directory or absolute
|
||||
- Directory must be writable
|
||||
- Logs are rotated based on `log_rotation`
|
||||
- Monitor disk usage
|
||||
|
||||
### `log_rotation`
|
||||
|
||||
**Type**: String
|
||||
**Required**: Yes
|
||||
**Default**: None
|
||||
|
||||
How often to rotate log files.
|
||||
|
||||
```yaml
|
||||
logging:
|
||||
log_rotation: 7days
|
||||
# OR
|
||||
log_rotation: 24hours
|
||||
# OR
|
||||
log_rotation: 30days
|
||||
```
|
||||
|
||||
**Format**: `<number><unit>`
|
||||
|
||||
**Units**:
|
||||
- `hours`: Hours (e.g., `12hours`, `24hours`)
|
||||
- `days`: Days (e.g., `7days`, `30days`)
|
||||
|
||||
**Recommendations**:
|
||||
- Development: `24hours` or `7days`
|
||||
- Production: `7days` or `30days`
|
||||
- High traffic: `24hours` (logs can be large)
|
||||
|
||||
## Environment-Specific Configurations
|
||||
|
||||
### Development
|
||||
|
||||
```yaml
|
||||
database:
|
||||
databases_directory_path: ./databases
|
||||
max_connections_per_vault: 8
|
||||
cursor_timeout_seconds: 30
|
||||
|
||||
server:
|
||||
host: 127.0.0.1
|
||||
port: 3000
|
||||
max_body_size_mb: 100
|
||||
max_clients_per_vault: 10
|
||||
response_timeout_seconds: 30
|
||||
|
||||
users:
|
||||
user_configs:
|
||||
- name: dev
|
||||
token: dev-token
|
||||
vault_access:
|
||||
type: allow_access_to_all
|
||||
|
||||
logging:
|
||||
log_directory: logs
|
||||
log_rotation: 24hours
|
||||
```
|
||||
|
||||
### Production
|
||||
|
||||
```yaml
|
||||
database:
|
||||
databases_directory_path: /data/databases
|
||||
max_connections_per_vault: 16
|
||||
cursor_timeout_seconds: 120
|
||||
|
||||
server:
|
||||
host: 0.0.0.0
|
||||
port: 3000
|
||||
max_body_size_mb: 512
|
||||
max_clients_per_vault: 256
|
||||
response_timeout_seconds: 90
|
||||
|
||||
users:
|
||||
user_configs:
|
||||
- name: admin
|
||||
token: <strong-random-token>
|
||||
vault_access:
|
||||
type: allow_access_to_all
|
||||
# Additional users...
|
||||
|
||||
logging:
|
||||
log_directory: /data/logs
|
||||
log_rotation: 7days
|
||||
```
|
||||
|
||||
## Validation
|
||||
|
||||
The server validates configuration on startup:
|
||||
|
||||
```bash
|
||||
# Start server
|
||||
./sync_server config.yml
|
||||
|
||||
# Check for errors in logs
|
||||
tail -f logs/latest.log
|
||||
```
|
||||
|
||||
**Common errors**:
|
||||
- Missing required fields
|
||||
- Invalid YAML syntax
|
||||
- Invalid values (negative numbers, etc.)
|
||||
- Directory not writable
|
||||
|
||||
## Performance Tuning
|
||||
|
||||
### High Concurrency
|
||||
|
||||
For many concurrent users:
|
||||
|
||||
```yaml
|
||||
database:
|
||||
max_connections_per_vault: 20 # Increase
|
||||
|
||||
server:
|
||||
max_clients_per_vault: 500 # Increase
|
||||
response_timeout_seconds: 120 # Increase for slow clients
|
||||
```
|
||||
|
||||
### Large Files
|
||||
|
||||
For vaults with large files:
|
||||
|
||||
```yaml
|
||||
server:
|
||||
max_body_size_mb: 1024 # Allow larger uploads
|
||||
response_timeout_seconds: 180 # More time for uploads
|
||||
```
|
||||
|
||||
### Resource-Constrained Systems
|
||||
|
||||
For limited CPU/memory:
|
||||
|
||||
```yaml
|
||||
database:
|
||||
max_connections_per_vault: 6 # Reduce
|
||||
|
||||
server:
|
||||
max_clients_per_vault: 50 # Reduce
|
||||
max_body_size_mb: 256 # Reduce
|
||||
```
|
||||
|
||||
## Security Considerations
|
||||
|
||||
### Token Security
|
||||
|
||||
- Use strong random tokens: `openssl rand -hex 32`
|
||||
- Never commit tokens to version control
|
||||
- Rotate tokens periodically
|
||||
- Use different tokens per user
|
||||
|
||||
### Network Security
|
||||
|
||||
- Bind to `127.0.0.1` if using reverse proxy on same host
|
||||
- Use firewall to restrict access
|
||||
- Enable SSL/TLS via reverse proxy
|
||||
|
||||
### Resource Limits
|
||||
|
||||
- Set `max_clients_per_vault` to prevent DoS
|
||||
- Set `max_body_size_mb` to prevent memory exhaustion
|
||||
- Configure `response_timeout_seconds` to prevent hanging connections
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Server won't start
|
||||
|
||||
**Check YAML syntax**:
|
||||
```bash
|
||||
# Use a YAML validator
|
||||
python -c 'import yaml, sys; yaml.safe_load(open("config.yml"))'
|
||||
```
|
||||
|
||||
**Check file paths**:
|
||||
```bash
|
||||
# Ensure directories exist and are writable
|
||||
mkdir -p databases logs
|
||||
chmod 755 databases logs
|
||||
```
|
||||
|
||||
**Check port availability**:
|
||||
```bash
|
||||
# Verify port is not in use
|
||||
lsof -i :3000
|
||||
```
|
||||
|
||||
### High memory usage
|
||||
|
||||
- Reduce `max_connections_per_vault`
|
||||
- Reduce `max_clients_per_vault`
|
||||
- Reduce `max_body_size_mb`
|
||||
- Check for large vaults or many concurrent users
|
||||
|
||||
### Slow performance
|
||||
|
||||
- Increase `max_connections_per_vault`
|
||||
- Increase database connection pool
|
||||
- Use SSD for database storage
|
||||
- Monitor database size (vacuum if needed)
|
||||
|
||||
## Next Steps
|
||||
|
||||
- [Configure authentication →](/config/authentication)
|
||||
- [Advanced configuration options →](/config/advanced)
|
||||
- [Deploy the server →](/guide/server-setup)
|
||||
516
docs/guide/cli-client.md
Normal file
516
docs/guide/cli-client.md
Normal file
|
|
@ -0,0 +1,516 @@
|
|||
# CLI Client
|
||||
|
||||
The VaultLink CLI client provides standalone synchronization without requiring Obsidian. Perfect for servers, automation, backups, or syncing vaults on headless systems.
|
||||
|
||||
## Installation
|
||||
|
||||
### Docker (Recommended)
|
||||
|
||||
Pull the latest image:
|
||||
|
||||
```bash
|
||||
docker pull ghcr.io/schmelczer/vault-link-cli:latest
|
||||
```
|
||||
|
||||
### npm
|
||||
|
||||
Install globally:
|
||||
|
||||
```bash
|
||||
npm install -g @schmelczer/local-client-cli
|
||||
```
|
||||
|
||||
Verify installation:
|
||||
|
||||
```bash
|
||||
vaultlink --version
|
||||
```
|
||||
|
||||
### From Source
|
||||
|
||||
Build from the repository:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/schmelczer/vault-link.git
|
||||
cd vault-link/frontend/local-client-cli
|
||||
npm install
|
||||
npm run build
|
||||
node dist/cli.js --help
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
### Basic Usage
|
||||
|
||||
```bash
|
||||
vaultlink \
|
||||
--local-path /path/to/vault \
|
||||
--remote-uri wss://sync.example.com \
|
||||
--token your-auth-token \
|
||||
--vault-name default
|
||||
```
|
||||
|
||||
### Docker Usage
|
||||
|
||||
```bash
|
||||
docker run -v /path/to/vault:/vault \
|
||||
ghcr.io/schmelczer/vault-link-cli:latest \
|
||||
-l /vault \
|
||||
-r wss://sync.example.com \
|
||||
-t your-auth-token \
|
||||
-v default
|
||||
```
|
||||
|
||||
### Docker Compose
|
||||
|
||||
Create `docker-compose.yml`:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
vaultlink-cli:
|
||||
image: ghcr.io/schmelczer/vault-link-cli:latest
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- ./vault:/vault
|
||||
command:
|
||||
- "-l"
|
||||
- "/vault"
|
||||
- "-r"
|
||||
- "wss://sync.example.com"
|
||||
- "-t"
|
||||
- "your-token"
|
||||
- "-v"
|
||||
- "default"
|
||||
```
|
||||
|
||||
Start the client:
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
## Configuration Options
|
||||
|
||||
### Required Arguments
|
||||
|
||||
| Argument | Short | Description | Example |
|
||||
|----------|-------|-------------|---------|
|
||||
| `--local-path` | `-l` | Local directory to sync | `/vault` |
|
||||
| `--remote-uri` | `-r` | Server WebSocket URI | `wss://sync.example.com` |
|
||||
| `--token` | `-t` | Authentication token | `abc123...` |
|
||||
| `--vault-name` | `-v` | Vault name on server | `default` |
|
||||
|
||||
### Optional Arguments
|
||||
|
||||
| Argument | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `--sync-concurrency` | `1` | Concurrent file operations |
|
||||
| `--max-file-size-mb` | `10` | Max file size in MB |
|
||||
| `--ignore-pattern` | - | Glob pattern to ignore (repeatable) |
|
||||
| `--websocket-retry-interval-ms` | `3500` | Reconnection interval |
|
||||
| `--log-level` | `INFO` | Log level: DEBUG, INFO, WARNING, ERROR |
|
||||
|
||||
### Environment Variables
|
||||
|
||||
Alternative to command-line arguments:
|
||||
|
||||
```bash
|
||||
export VAULTLINK_LOCAL_PATH="/vault"
|
||||
export VAULTLINK_REMOTE_URI="wss://sync.example.com"
|
||||
export VAULTLINK_TOKEN="your-token"
|
||||
export VAULTLINK_VAULT_NAME="default"
|
||||
|
||||
vaultlink
|
||||
```
|
||||
|
||||
## Examples
|
||||
|
||||
### Basic Sync
|
||||
|
||||
Sync a local directory to the server:
|
||||
|
||||
```bash
|
||||
vaultlink \
|
||||
-l ./my-notes \
|
||||
-r wss://sync.example.com \
|
||||
-t my-secure-token \
|
||||
-v personal
|
||||
```
|
||||
|
||||
### With Ignore Patterns
|
||||
|
||||
Exclude specific files or directories:
|
||||
|
||||
```bash
|
||||
vaultlink \
|
||||
-l ./vault \
|
||||
-r wss://sync.example.com \
|
||||
-t token123 \
|
||||
-v default \
|
||||
--ignore-pattern "*.tmp" \
|
||||
--ignore-pattern ".DS_Store" \
|
||||
--ignore-pattern "node_modules/**"
|
||||
```
|
||||
|
||||
### Debug Logging
|
||||
|
||||
Enable verbose logging:
|
||||
|
||||
```bash
|
||||
vaultlink \
|
||||
-l ./vault \
|
||||
-r wss://sync.example.com \
|
||||
-t token123 \
|
||||
-v default \
|
||||
--log-level DEBUG
|
||||
```
|
||||
|
||||
### High Concurrency
|
||||
|
||||
Faster initial sync:
|
||||
|
||||
```bash
|
||||
vaultlink \
|
||||
-l ./vault \
|
||||
-r wss://sync.example.com \
|
||||
-t token123 \
|
||||
-v default \
|
||||
--sync-concurrency 5
|
||||
```
|
||||
|
||||
### Large Files
|
||||
|
||||
Allow larger file uploads:
|
||||
|
||||
```bash
|
||||
vaultlink \
|
||||
-l ./vault \
|
||||
-r wss://sync.example.com \
|
||||
-t token123 \
|
||||
-v default \
|
||||
--max-file-size-mb 50
|
||||
```
|
||||
|
||||
## Docker Deployment
|
||||
|
||||
### Long-Running Sync
|
||||
|
||||
Run as a daemon for continuous synchronization:
|
||||
|
||||
```bash
|
||||
docker run -d \
|
||||
--name vaultlink-sync \
|
||||
--restart unless-stopped \
|
||||
-v $(pwd)/vault:/vault \
|
||||
ghcr.io/schmelczer/vault-link-cli:latest \
|
||||
-l /vault \
|
||||
-r wss://sync.example.com \
|
||||
-t your-token \
|
||||
-v default
|
||||
```
|
||||
|
||||
Monitor logs:
|
||||
|
||||
```bash
|
||||
docker logs -f vaultlink-sync
|
||||
```
|
||||
|
||||
### Health Monitoring
|
||||
|
||||
The Docker image includes built-in health checks:
|
||||
|
||||
```bash
|
||||
# Check health status
|
||||
docker ps
|
||||
|
||||
# View detailed health info
|
||||
docker inspect --format='{{json .State.Health}}' vaultlink-sync | jq
|
||||
```
|
||||
|
||||
Health check verifies:
|
||||
- Health file exists
|
||||
- Status updated within last 30 seconds
|
||||
- WebSocket connection is active
|
||||
|
||||
Configure custom health check:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
vaultlink-cli:
|
||||
image: ghcr.io/schmelczer/vault-link-cli:latest
|
||||
healthcheck:
|
||||
test: ["CMD", "node", "/app/healthcheck.js"]
|
||||
interval: 15s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
start_period: 20s
|
||||
```
|
||||
|
||||
### Read-Only Vault
|
||||
|
||||
Mount vault as read-only to prevent local changes:
|
||||
|
||||
```bash
|
||||
docker run -d \
|
||||
-v $(pwd)/vault:/vault:ro \
|
||||
ghcr.io/schmelczer/vault-link-cli:latest \
|
||||
-l /vault \
|
||||
-r wss://sync.example.com \
|
||||
-t token \
|
||||
-v default
|
||||
```
|
||||
|
||||
::: warning
|
||||
The CLI needs write access to create `.vaultlink` metadata directory. Mount as read-write or provide a separate writeable directory.
|
||||
:::
|
||||
|
||||
## How It Works
|
||||
|
||||
### Initial Sync
|
||||
|
||||
On startup:
|
||||
|
||||
1. Creates `.vaultlink/` directory for metadata
|
||||
2. Scans local filesystem
|
||||
3. Uploads all local files to server
|
||||
4. Downloads files from server not present locally
|
||||
5. Resolves conflicts using operational transformation
|
||||
|
||||
### Real-Time Synchronization
|
||||
|
||||
After initial sync:
|
||||
|
||||
1. Watches filesystem for changes using `fs.watch`
|
||||
2. Uploads changed files immediately
|
||||
3. Receives real-time updates from server via WebSocket
|
||||
4. Handles bidirectional sync automatically
|
||||
|
||||
### Graceful Shutdown
|
||||
|
||||
On SIGINT (Ctrl+C) or SIGTERM:
|
||||
|
||||
1. Completes pending uploads
|
||||
2. Closes WebSocket connection cleanly
|
||||
3. Flushes metadata to disk
|
||||
4. Exits gracefully
|
||||
|
||||
## Use Cases
|
||||
|
||||
### Automated Backups
|
||||
|
||||
Continuously backup vaults to a remote server:
|
||||
|
||||
```bash
|
||||
docker run -d \
|
||||
--name vault-backup \
|
||||
-v /important/notes:/vault:ro \
|
||||
ghcr.io/schmelczer/vault-link-cli:latest \
|
||||
-l /vault -r wss://backup.example.com -t backup-token -v backups
|
||||
```
|
||||
|
||||
### CI/CD Documentation
|
||||
|
||||
Sync documentation in automated pipelines:
|
||||
|
||||
```bash
|
||||
# In your CI pipeline
|
||||
docker run \
|
||||
-v $(pwd)/docs:/vault \
|
||||
ghcr.io/schmelczer/vault-link-cli:latest \
|
||||
-l /vault -r wss://docs.example.com -t ci-token -v prod-docs
|
||||
```
|
||||
|
||||
### Multi-Location Sync
|
||||
|
||||
Sync between different geographic locations:
|
||||
|
||||
```bash
|
||||
# Location A
|
||||
vaultlink -l /data/vault -r wss://hub.example.com -t token -v shared
|
||||
|
||||
# Location B
|
||||
vaultlink -l /backup/vault -r wss://hub.example.com -t token -v shared
|
||||
```
|
||||
|
||||
### Development Environment
|
||||
|
||||
Keep documentation in sync across dev environments:
|
||||
|
||||
```bash
|
||||
# In docker-compose.yml for your dev stack
|
||||
services:
|
||||
docs-sync:
|
||||
image: ghcr.io/schmelczer/vault-link-cli:latest
|
||||
volumes:
|
||||
- ./docs:/vault
|
||||
command: ["-l", "/vault", "-r", "wss://docs-server", "-t", "dev-token", "-v", "dev"]
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Client won't connect
|
||||
|
||||
**Check server accessibility**:
|
||||
```bash
|
||||
curl https://sync.example.com/vaults/test/ping
|
||||
```
|
||||
|
||||
**Verify WebSocket protocol**:
|
||||
- Use `ws://` for HTTP servers
|
||||
- Use `wss://` for HTTPS servers
|
||||
|
||||
**Check authentication**:
|
||||
- Token must match server config
|
||||
- User must have access to the vault
|
||||
|
||||
### Permission errors
|
||||
|
||||
**Docker volume permissions**:
|
||||
```bash
|
||||
# Ensure directory is writable
|
||||
chmod 755 /path/to/vault
|
||||
|
||||
# Check Docker user ID
|
||||
docker run --rm ghcr.io/schmelczer/vault-link-cli:latest id
|
||||
```
|
||||
|
||||
**SELinux issues**:
|
||||
```bash
|
||||
# Add :z flag to volume mount
|
||||
docker run -v /path/to/vault:/vault:z ...
|
||||
```
|
||||
|
||||
### Files not syncing
|
||||
|
||||
**Check ignore patterns**:
|
||||
- View logs to see which files are skipped
|
||||
- Ensure patterns don't match unintentionally
|
||||
|
||||
**File size limits**:
|
||||
- Check `--max-file-size-mb` setting
|
||||
- Large files are skipped with a warning
|
||||
|
||||
**Check metadata**:
|
||||
```bash
|
||||
# View sync metadata
|
||||
cat /path/to/vault/.vaultlink/metadata.json
|
||||
```
|
||||
|
||||
### High memory usage
|
||||
|
||||
**Reduce concurrency**:
|
||||
```bash
|
||||
--sync-concurrency 1
|
||||
```
|
||||
|
||||
**Limit file sizes**:
|
||||
```bash
|
||||
--max-file-size-mb 5
|
||||
```
|
||||
|
||||
**Check vault size**:
|
||||
- Very large vaults may need more resources
|
||||
- Consider splitting into multiple vaults
|
||||
|
||||
### Connection keeps dropping
|
||||
|
||||
**Increase retry interval**:
|
||||
```bash
|
||||
--websocket-retry-interval-ms 5000
|
||||
```
|
||||
|
||||
**Check network stability**:
|
||||
```bash
|
||||
# Monitor connection
|
||||
docker logs -f vaultlink-sync | grep -i websocket
|
||||
```
|
||||
|
||||
**Server timeout settings**:
|
||||
- Verify reverse proxy WebSocket timeout
|
||||
- Check server `response_timeout_seconds`
|
||||
|
||||
## Advanced Usage
|
||||
|
||||
### Custom Healthcheck Script
|
||||
|
||||
Create your own health monitoring:
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
HEALTH_FILE="/tmp/vaultlink-health.json"
|
||||
|
||||
if [ ! -f "$HEALTH_FILE" ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check file is recent (within 60 seconds)
|
||||
if [ $(( $(date +%s) - $(stat -c %Y "$HEALTH_FILE") )) -gt 60 ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check WebSocket is connected
|
||||
if ! jq -e '.connected == true' "$HEALTH_FILE" > /dev/null; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exit 0
|
||||
```
|
||||
|
||||
### Automated Recovery
|
||||
|
||||
Restart on failure with exponential backoff:
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
RETRY_DELAY=5
|
||||
|
||||
while true; do
|
||||
vaultlink -l /vault -r wss://server -t token -v default
|
||||
|
||||
echo "Client exited, restarting in ${RETRY_DELAY}s..."
|
||||
sleep $RETRY_DELAY
|
||||
|
||||
# Exponential backoff up to 5 minutes
|
||||
RETRY_DELAY=$((RETRY_DELAY * 2))
|
||||
if [ $RETRY_DELAY -gt 300 ]; then
|
||||
RETRY_DELAY=300
|
||||
fi
|
||||
done
|
||||
```
|
||||
|
||||
### Integration with systemd
|
||||
|
||||
Create `/etc/systemd/system/vaultlink-cli.service`:
|
||||
|
||||
```ini
|
||||
[Unit]
|
||||
Description=VaultLink CLI Sync
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
Environment="VAULTLINK_LOCAL_PATH=/data/vault"
|
||||
Environment="VAULTLINK_REMOTE_URI=wss://sync.example.com"
|
||||
Environment="VAULTLINK_TOKEN=your-token"
|
||||
Environment="VAULTLINK_VAULT_NAME=default"
|
||||
ExecStart=/usr/local/bin/vaultlink
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
|
||||
Enable and start:
|
||||
```bash
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable vaultlink-cli
|
||||
sudo systemctl start vaultlink-cli
|
||||
```
|
||||
|
||||
## Next Steps
|
||||
|
||||
- [Configure server authentication →](/config/authentication)
|
||||
- [Learn about the sync algorithm →](/architecture/sync-algorithm)
|
||||
- [Set up Obsidian plugin →](/guide/obsidian-plugin)
|
||||
185
docs/guide/getting-started.md
Normal file
185
docs/guide/getting-started.md
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
# Getting Started
|
||||
|
||||
This guide will walk you through setting up VaultLink from scratch. You'll have a working sync server and connected client in under 10 minutes.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Docker installed (recommended) or Rust toolchain for building from source
|
||||
- Basic familiarity with command line
|
||||
- A server or machine to host the sync server (can be localhost for testing)
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Step 1: Deploy the Sync Server
|
||||
|
||||
The fastest way to get started is with Docker:
|
||||
|
||||
```bash
|
||||
# Create a directory for server data
|
||||
mkdir -p ~/vaultlink-data
|
||||
cd ~/vaultlink-data
|
||||
|
||||
# Create a basic configuration file
|
||||
cat > config.yml << 'EOF'
|
||||
database:
|
||||
databases_directory_path: databases
|
||||
max_connections_per_vault: 12
|
||||
cursor_timeout_seconds: 60
|
||||
server:
|
||||
host: 0.0.0.0
|
||||
port: 3000
|
||||
max_body_size_mb: 512
|
||||
max_clients_per_vault: 256
|
||||
response_timeout_seconds: 60
|
||||
users:
|
||||
user_configs:
|
||||
- name: admin
|
||||
token: change-this-to-a-secure-random-token
|
||||
vault_access:
|
||||
type: allow_access_to_all
|
||||
logging:
|
||||
log_directory: logs
|
||||
log_rotation: 7days
|
||||
EOF
|
||||
|
||||
# Run the server
|
||||
docker run -d \
|
||||
--name vaultlink-server \
|
||||
--restart unless-stopped \
|
||||
-p 3000:3000 \
|
||||
-v $(pwd):/data \
|
||||
ghcr.io/schmelczer/vault-link-server:latest \
|
||||
/app/sync_server /data/config.yml
|
||||
```
|
||||
|
||||
::: warning
|
||||
Change the token in `config.yml` to a secure random value before deploying to production!
|
||||
:::
|
||||
|
||||
Verify the server is running:
|
||||
|
||||
```bash
|
||||
curl http://localhost:3000/vaults/test/ping
|
||||
```
|
||||
|
||||
You should see: `pong`
|
||||
|
||||
### Step 2: Choose Your Client
|
||||
|
||||
You can connect to VaultLink using either the Obsidian plugin or the standalone CLI client.
|
||||
|
||||
#### Option A: Obsidian Plugin
|
||||
|
||||
1. Open Obsidian Settings → Community Plugins
|
||||
2. Browse community plugins and search for "VaultLink"
|
||||
3. Install and enable the plugin
|
||||
4. Configure the plugin:
|
||||
- **Server URL**: `ws://localhost:3000` (or your server address)
|
||||
- **Token**: The token from your `config.yml`
|
||||
- **Vault Name**: `default` (or any name you choose)
|
||||
|
||||
[Read the full Obsidian plugin guide →](/guide/obsidian-plugin)
|
||||
|
||||
#### Option B: CLI Client
|
||||
|
||||
Perfect for syncing vaults without Obsidian:
|
||||
|
||||
```bash
|
||||
docker run -d \
|
||||
--name vaultlink-cli \
|
||||
--restart unless-stopped \
|
||||
-v /path/to/your/vault:/vault \
|
||||
ghcr.io/schmelczer/vault-link-cli:latest \
|
||||
-l /vault \
|
||||
-r ws://localhost:3000 \
|
||||
-t change-this-to-a-secure-random-token \
|
||||
-v default
|
||||
```
|
||||
|
||||
Replace `/path/to/your/vault` with the directory containing your files.
|
||||
|
||||
[Read the full CLI client guide →](/guide/cli-client)
|
||||
|
||||
## Next Steps
|
||||
|
||||
### Production Deployment
|
||||
|
||||
For production use, you should:
|
||||
|
||||
1. **Use HTTPS/WSS**: Put the sync server behind a reverse proxy with SSL
|
||||
2. **Secure tokens**: Generate cryptographically random tokens
|
||||
3. **Configure backups**: Back up the SQLite databases regularly
|
||||
4. **Set up monitoring**: Use Docker health checks and logging
|
||||
|
||||
[Learn about production deployment →](/guide/server-setup#production-deployment)
|
||||
|
||||
### Multiple Users
|
||||
|
||||
To add more users or restrict vault access:
|
||||
|
||||
```yaml
|
||||
users:
|
||||
user_configs:
|
||||
- name: alice
|
||||
token: alice-secure-token
|
||||
vault_access:
|
||||
type: allow_list
|
||||
allowed:
|
||||
- personal
|
||||
- shared
|
||||
- name: bob
|
||||
token: bob-secure-token
|
||||
vault_access:
|
||||
type: allow_list
|
||||
allowed:
|
||||
- shared
|
||||
```
|
||||
|
||||
[Learn about authentication configuration →](/config/authentication)
|
||||
|
||||
### Advanced Configuration
|
||||
|
||||
Explore advanced server options:
|
||||
|
||||
- Database tuning for large vaults
|
||||
- Rate limiting and connection limits
|
||||
- Custom logging and log rotation
|
||||
- Multi-vault setups
|
||||
|
||||
[View configuration reference →](/config/server)
|
||||
|
||||
## Architecture Overview
|
||||
|
||||
Want to understand how VaultLink works under the hood?
|
||||
|
||||
[Read the architecture documentation →](/architecture/)
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Server won't start
|
||||
|
||||
Check Docker logs:
|
||||
```bash
|
||||
docker logs vaultlink-server
|
||||
```
|
||||
|
||||
Common issues:
|
||||
- Port 3000 already in use: Change the port mapping `-p 3001:3000`
|
||||
- Config file errors: Validate YAML syntax
|
||||
- Permission issues: Ensure the volume mount is writable
|
||||
|
||||
### Client can't connect
|
||||
|
||||
1. Verify server is accessible: `curl http://your-server:3000/vaults/test/ping`
|
||||
2. Check WebSocket connectivity (browser dev tools or wscat)
|
||||
3. Verify token matches between client and server config
|
||||
4. Check firewall rules allow port 3000
|
||||
|
||||
### Files not syncing
|
||||
|
||||
1. Check client logs for errors
|
||||
2. Verify vault name matches on both server and client
|
||||
3. Ensure user has access to the vault (check server config)
|
||||
4. Check for file size limits (default 10MB for CLI)
|
||||
|
||||
For more help, [open an issue on GitHub](https://github.com/schmelczer/vault-link/issues).
|
||||
262
docs/guide/obsidian-plugin.md
Normal file
262
docs/guide/obsidian-plugin.md
Normal file
|
|
@ -0,0 +1,262 @@
|
|||
# Obsidian Plugin
|
||||
|
||||
The VaultLink Obsidian plugin provides native real-time synchronization directly within Obsidian.
|
||||
|
||||
## Installation
|
||||
|
||||
### From Obsidian Community Plugins
|
||||
|
||||
1. Open Obsidian Settings
|
||||
2. Navigate to **Community Plugins**
|
||||
3. Click **Browse** and search for "VaultLink"
|
||||
4. Click **Install**
|
||||
5. Enable the plugin
|
||||
|
||||
### Manual Installation
|
||||
|
||||
1. Download the latest release from [GitHub Releases](https://github.com/schmelczer/vault-link/releases)
|
||||
2. Extract `main.js`, `manifest.json`, and `styles.css`
|
||||
3. Copy to `.obsidian/plugins/vault-link/` in your vault
|
||||
4. Reload Obsidian
|
||||
5. Enable VaultLink in Community Plugins settings
|
||||
|
||||
## Configuration
|
||||
|
||||
After installation, configure the plugin in **Settings → VaultLink**.
|
||||
|
||||
### Required Settings
|
||||
|
||||
#### Server URL
|
||||
The WebSocket URL of your sync server.
|
||||
|
||||
- **Development/Local**: `ws://localhost:3000`
|
||||
- **Production (SSL)**: `wss://sync.example.com`
|
||||
|
||||
::: tip
|
||||
Use `ws://` for unencrypted connections and `wss://` for SSL connections (production).
|
||||
:::
|
||||
|
||||
#### Authentication Token
|
||||
Your authentication token from the server's `config.yml`.
|
||||
|
||||
Generate a secure token:
|
||||
```bash
|
||||
openssl rand -hex 32
|
||||
```
|
||||
|
||||
#### Vault Name
|
||||
The name of the vault on the server. Can be any string.
|
||||
|
||||
Multiple Obsidian vaults can sync to the same server vault name (for shared vaults), or use unique names for separate vaults.
|
||||
|
||||
### Optional Settings
|
||||
|
||||
#### Sync Concurrency
|
||||
Number of files to sync simultaneously.
|
||||
- **Default**: 1
|
||||
- **Range**: 1-10
|
||||
- Higher values = faster initial sync, more resource usage
|
||||
|
||||
#### Max File Size
|
||||
Maximum file size to sync (in MB).
|
||||
- **Default**: 10
|
||||
- Files larger than this are skipped
|
||||
|
||||
#### Ignore Patterns
|
||||
Glob patterns for files to exclude from sync.
|
||||
|
||||
Examples:
|
||||
- `*.tmp` - Ignore temporary files
|
||||
- `.trash/**` - Ignore trash folder
|
||||
- `private/**` - Ignore private directory
|
||||
|
||||
#### WebSocket Retry Interval
|
||||
Milliseconds between reconnection attempts when disconnected.
|
||||
- **Default**: 3500ms
|
||||
- Increase for flaky networks to avoid connection spam
|
||||
|
||||
## Usage
|
||||
|
||||
### Initial Sync
|
||||
|
||||
When first connecting:
|
||||
|
||||
1. The plugin uploads all local files to the server
|
||||
2. Downloads any missing files from the server
|
||||
3. Resolves any conflicts using operational transformation
|
||||
4. Begins real-time synchronization
|
||||
|
||||
Initial sync time depends on vault size and `sync_concurrency` setting.
|
||||
|
||||
### Real-Time Sync
|
||||
|
||||
Once connected:
|
||||
|
||||
- **File changes**: Automatically synced when saved
|
||||
- **File creation**: New files immediately uploaded
|
||||
- **File deletion**: Deletions propagated to other clients
|
||||
- **File renames**: Tracked and synchronized
|
||||
|
||||
The plugin watches your vault filesystem and syncs changes in real-time via WebSocket.
|
||||
|
||||
### Status Indicators
|
||||
|
||||
The plugin provides visual feedback:
|
||||
|
||||
- **Connected**: Green status in settings
|
||||
- **Syncing**: Progress indicator during uploads
|
||||
- **Disconnected**: Red status, automatic reconnection attempts
|
||||
- **Error**: Error message in settings and console
|
||||
|
||||
Check the Obsidian console (Ctrl+Shift+I / Cmd+Option+I) for detailed logs.
|
||||
|
||||
## Features
|
||||
|
||||
### Automatic Conflict Resolution
|
||||
|
||||
When multiple users edit the same file simultaneously, operational transformation merges changes automatically:
|
||||
|
||||
- All edits are preserved
|
||||
- No manual conflict resolution required
|
||||
- Changes appear in real-time as others type
|
||||
|
||||
### Mobile Support
|
||||
|
||||
VaultLink works on Obsidian mobile (iOS and Android):
|
||||
|
||||
- Same configuration as desktop
|
||||
- Real-time sync across all devices
|
||||
- Handle network changes gracefully
|
||||
|
||||
::: warning
|
||||
Ensure your sync server is accessible from mobile networks (use WSS with a public domain or VPN).
|
||||
:::
|
||||
|
||||
### Offline Support
|
||||
|
||||
The plugin handles offline scenarios:
|
||||
|
||||
- Continue working when disconnected
|
||||
- Changes queue locally
|
||||
- Automatic sync when connection restored
|
||||
- Conflict resolution if others edited the same files
|
||||
|
||||
## Collaboration Workflows
|
||||
|
||||
### Personal Multi-Device Sync
|
||||
|
||||
Sync the same vault across devices:
|
||||
|
||||
1. Configure each Obsidian instance with the same vault name
|
||||
2. Use the same authentication token
|
||||
3. All devices stay in sync automatically
|
||||
|
||||
### Team Shared Vault
|
||||
|
||||
Multiple users collaborating:
|
||||
|
||||
1. Each user has their own token (configured in server `config.yml`)
|
||||
2. All users connect to the same vault name
|
||||
3. Real-time collaborative editing with automatic conflict resolution
|
||||
|
||||
### Selective Sharing
|
||||
|
||||
Share specific folders while keeping others private:
|
||||
|
||||
1. Use different vault names for shared vs. private content
|
||||
2. Configure access control on the server per vault
|
||||
3. Use ignore patterns to exclude sensitive directories
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Plugin won't connect
|
||||
|
||||
1. **Verify server is running**:
|
||||
```bash
|
||||
curl http://your-server:3000/vaults/test/ping
|
||||
```
|
||||
Should return `pong`
|
||||
|
||||
2. **Check URL format**:
|
||||
- Local: `ws://localhost:3000`
|
||||
- Remote (SSL): `wss://sync.example.com`
|
||||
- Don't include `/vault/name` in the URL
|
||||
|
||||
3. **Verify token**:
|
||||
- Must match server config exactly
|
||||
- No extra spaces or quotes
|
||||
- Check server logs for authentication errors
|
||||
|
||||
4. **Check firewall**:
|
||||
- Ensure port is accessible from your network
|
||||
- For mobile, server must be publicly accessible or use VPN
|
||||
|
||||
### Files not syncing
|
||||
|
||||
1. **Check ignore patterns**: File may match an exclusion pattern
|
||||
2. **File size**: Check if file exceeds `max_file_size_mb`
|
||||
3. **Permissions**: Ensure vault directory is readable/writable
|
||||
4. **Console errors**: Open dev tools (Ctrl+Shift+I) and check console
|
||||
|
||||
### Slow initial sync
|
||||
|
||||
1. **Increase concurrency**: Set `sync_concurrency` higher (e.g., 5)
|
||||
2. **Network speed**: Check internet connection
|
||||
3. **Server resources**: Ensure server isn't overloaded
|
||||
4. **Large files**: Consider increasing timeout settings
|
||||
|
||||
### Conflicts not resolving
|
||||
|
||||
Operational transformation should handle conflicts automatically. If issues persist:
|
||||
|
||||
1. Check console for sync errors
|
||||
2. Verify both clients are connected
|
||||
3. Check server logs for processing errors
|
||||
4. Ensure files are text-based (binary files may not merge well)
|
||||
|
||||
### High CPU/Memory usage
|
||||
|
||||
1. **Reduce concurrency**: Lower `sync_concurrency`
|
||||
2. **Add ignore patterns**: Exclude unnecessary files
|
||||
3. **File watchers**: Large vaults may trigger many filesystem events
|
||||
4. **Check for sync loops**: Ensure no circular dependencies
|
||||
|
||||
## Advanced Configuration
|
||||
|
||||
### Multiple Vaults
|
||||
|
||||
To sync multiple Obsidian vaults to different server vaults:
|
||||
|
||||
1. Each Obsidian vault has its own VaultLink plugin configuration
|
||||
2. Use different vault names for each
|
||||
3. Can use the same or different tokens (depending on access control)
|
||||
|
||||
### Custom Sync Patterns
|
||||
|
||||
Combine ignore patterns for fine-grained control:
|
||||
|
||||
```
|
||||
# Ignore patterns
|
||||
*.tmp
|
||||
*.bak
|
||||
.DS_Store
|
||||
.trash/**
|
||||
private/**
|
||||
drafts/**/*.draft.md
|
||||
```
|
||||
|
||||
### Development/Testing
|
||||
|
||||
For plugin development:
|
||||
|
||||
1. Clone the repository
|
||||
2. `cd frontend && npm install`
|
||||
3. `npm run dev` to build in watch mode
|
||||
4. Plugin rebuilds automatically on changes
|
||||
5. Reload Obsidian to test changes
|
||||
|
||||
## Next Steps
|
||||
|
||||
- [Learn about the sync algorithm →](/architecture/sync-algorithm)
|
||||
- [Configure the server →](/config/server)
|
||||
- [Set up the CLI client →](/guide/cli-client)
|
||||
370
docs/guide/server-setup.md
Normal file
370
docs/guide/server-setup.md
Normal file
|
|
@ -0,0 +1,370 @@
|
|||
# Server Setup
|
||||
|
||||
This guide covers deploying the VaultLink sync server in various environments, from local development to production infrastructure.
|
||||
|
||||
## Deployment Options
|
||||
|
||||
### Docker (Recommended)
|
||||
|
||||
Docker provides the easiest deployment path with built-in health checks and minimal dependencies.
|
||||
|
||||
#### Basic Docker Deployment
|
||||
|
||||
```bash
|
||||
# Pull the latest image
|
||||
docker pull ghcr.io/schmelczer/vault-link-server:latest
|
||||
|
||||
# Create data directory
|
||||
mkdir -p ~/vaultlink-data
|
||||
|
||||
# Create config.yml (see Configuration section below)
|
||||
|
||||
# Run the container
|
||||
docker run -d \
|
||||
--name vaultlink-server \
|
||||
--restart unless-stopped \
|
||||
-p 3000:3000 \
|
||||
-v ~/vaultlink-data:/data \
|
||||
ghcr.io/schmelczer/vault-link-server:latest \
|
||||
/app/sync_server /data/config.yml
|
||||
```
|
||||
|
||||
#### Docker Compose
|
||||
|
||||
Create `docker-compose.yml`:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
vaultlink-server:
|
||||
image: ghcr.io/schmelczer/vault-link-server:latest
|
||||
container_name: vaultlink-server
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "3000:3000"
|
||||
volumes:
|
||||
- ./data:/data
|
||||
command: ["/app/sync_server", "/data/config.yml"]
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:3000/vaults/fake/ping"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
```
|
||||
|
||||
Start the server:
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
### Binary Installation
|
||||
|
||||
Download pre-built binaries from [GitHub Releases](https://github.com/schmelczer/vault-link/releases).
|
||||
|
||||
```bash
|
||||
# Download the binary for your platform
|
||||
wget https://github.com/schmelczer/vault-link/releases/latest/download/sync_server-linux-x86_64
|
||||
|
||||
# Make executable
|
||||
chmod +x sync_server-linux-x86_64
|
||||
|
||||
# Run the server
|
||||
./sync_server-linux-x86_64 config.yml
|
||||
```
|
||||
|
||||
### Build from Source
|
||||
|
||||
Requirements:
|
||||
- Rust 1.89.0 or later
|
||||
- SQLite development headers
|
||||
- SQLx CLI
|
||||
|
||||
```bash
|
||||
# Clone the repository
|
||||
git clone https://github.com/schmelczer/vault-link.git
|
||||
cd vault-link/sync-server
|
||||
|
||||
# Install SQLx CLI
|
||||
cargo install sqlx-cli
|
||||
|
||||
# Set up the database
|
||||
sqlx database create --database-url sqlite://db.sqlite3
|
||||
sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3
|
||||
cargo sqlx prepare --workspace
|
||||
|
||||
# Build in release mode
|
||||
cargo build --release
|
||||
|
||||
# Run the server
|
||||
./target/release/sync_server config.yml
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
Create a `config.yml` file with your server configuration:
|
||||
|
||||
```yaml
|
||||
database:
|
||||
databases_directory_path: databases
|
||||
max_connections_per_vault: 12
|
||||
cursor_timeout_seconds: 60
|
||||
|
||||
server:
|
||||
host: 0.0.0.0
|
||||
port: 3000
|
||||
max_body_size_mb: 512
|
||||
max_clients_per_vault: 256
|
||||
response_timeout_seconds: 60
|
||||
|
||||
users:
|
||||
user_configs:
|
||||
- name: admin
|
||||
token: your-secure-random-token-here
|
||||
vault_access:
|
||||
type: allow_access_to_all
|
||||
|
||||
logging:
|
||||
log_directory: logs
|
||||
log_rotation: 7days
|
||||
```
|
||||
|
||||
### Configuration Fields
|
||||
|
||||
#### Database
|
||||
|
||||
- `databases_directory_path`: Directory for SQLite databases (one per vault)
|
||||
- `max_connections_per_vault`: Maximum concurrent database connections
|
||||
- `cursor_timeout_seconds`: How long to keep database cursors alive
|
||||
|
||||
#### Server
|
||||
|
||||
- `host`: Bind address (use `0.0.0.0` for all interfaces)
|
||||
- `port`: Port to listen on (default: 3000)
|
||||
- `max_body_size_mb`: Maximum upload size
|
||||
- `max_clients_per_vault`: Concurrent client limit per vault
|
||||
- `response_timeout_seconds`: Request timeout
|
||||
|
||||
#### Users
|
||||
|
||||
See [Authentication Configuration →](/config/authentication) for detailed user setup.
|
||||
|
||||
#### Logging
|
||||
|
||||
- `log_directory`: Where to store log files
|
||||
- `log_rotation`: How often to rotate logs (e.g., `7days`, `24hours`)
|
||||
|
||||
## Production Deployment
|
||||
|
||||
### SSL/TLS with Reverse Proxy
|
||||
|
||||
VaultLink doesn't handle SSL directly. Use a reverse proxy like Nginx or Caddy.
|
||||
|
||||
#### Nginx Configuration
|
||||
|
||||
```nginx
|
||||
upstream vaultlink {
|
||||
server localhost:3000;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 443 ssl http2;
|
||||
server_name sync.example.com;
|
||||
|
||||
ssl_certificate /path/to/cert.pem;
|
||||
ssl_certificate_key /path/to/key.pem;
|
||||
|
||||
location / {
|
||||
proxy_pass http://vaultlink;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
# WebSocket specific
|
||||
proxy_read_timeout 3600s;
|
||||
proxy_send_timeout 3600s;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Reload Nginx:
|
||||
```bash
|
||||
sudo nginx -t
|
||||
sudo systemctl reload nginx
|
||||
```
|
||||
|
||||
#### Caddy Configuration
|
||||
|
||||
Caddy handles SSL automatically:
|
||||
|
||||
```caddy
|
||||
sync.example.com {
|
||||
reverse_proxy localhost:3000
|
||||
}
|
||||
```
|
||||
|
||||
Start Caddy:
|
||||
```bash
|
||||
caddy run --config Caddyfile
|
||||
```
|
||||
|
||||
### Systemd Service
|
||||
|
||||
Create `/etc/systemd/system/vaultlink.service`:
|
||||
|
||||
```ini
|
||||
[Unit]
|
||||
Description=VaultLink Sync Server
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=vaultlink
|
||||
WorkingDirectory=/opt/vaultlink
|
||||
ExecStart=/opt/vaultlink/sync_server /opt/vaultlink/config.yml
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
|
||||
Enable and start:
|
||||
|
||||
```bash
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable vaultlink
|
||||
sudo systemctl start vaultlink
|
||||
sudo systemctl status vaultlink
|
||||
```
|
||||
|
||||
### Security Best Practices
|
||||
|
||||
1. **Use strong tokens**: Generate with `openssl rand -hex 32`
|
||||
2. **Enable firewall**: Only expose port 3000 to reverse proxy
|
||||
3. **Regular updates**: Keep Docker images and binaries updated
|
||||
4. **Backup databases**: SQLite files in `databases_directory_path`
|
||||
5. **Monitor logs**: Check log directory for errors and anomalies
|
||||
6. **Limit access**: Use vault-specific access controls per user
|
||||
|
||||
### Backup Strategy
|
||||
|
||||
The SQLite databases contain all vault data and history:
|
||||
|
||||
```bash
|
||||
# Backup script
|
||||
#!/bin/bash
|
||||
BACKUP_DIR="/backup/vaultlink/$(date +%Y%m%d)"
|
||||
DATA_DIR="/data/databases"
|
||||
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
cp -r "$DATA_DIR" "$BACKUP_DIR/"
|
||||
|
||||
# Keep 30 days of backups
|
||||
find /backup/vaultlink -type d -mtime +30 -exec rm -rf {} +
|
||||
```
|
||||
|
||||
Run daily via cron:
|
||||
```cron
|
||||
0 2 * * * /opt/vaultlink/backup.sh
|
||||
```
|
||||
|
||||
### Monitoring
|
||||
|
||||
#### Health Checks
|
||||
|
||||
The server exposes a ping endpoint:
|
||||
|
||||
```bash
|
||||
curl http://localhost:3000/vaults/fake/ping
|
||||
# Returns: pong
|
||||
```
|
||||
|
||||
Docker health check is built-in and checks this endpoint every 30 seconds.
|
||||
|
||||
#### Prometheus Metrics
|
||||
|
||||
For advanced monitoring, collect Docker stats or implement custom metrics.
|
||||
|
||||
#### Log Monitoring
|
||||
|
||||
Logs are written to the configured `log_directory`. Monitor for:
|
||||
- Connection failures
|
||||
- Authentication errors
|
||||
- Database errors
|
||||
- WebSocket disconnections
|
||||
|
||||
Example log watching:
|
||||
```bash
|
||||
tail -f /data/logs/*.log | grep -i error
|
||||
```
|
||||
|
||||
## Scaling
|
||||
|
||||
### Horizontal Scaling
|
||||
|
||||
VaultLink currently uses SQLite, which limits horizontal scaling. For multiple servers:
|
||||
|
||||
1. Run separate instances for different vaults
|
||||
2. Use load balancer with sticky sessions (same vault → same server)
|
||||
3. Consider database architecture for your scale needs
|
||||
|
||||
### Vertical Scaling
|
||||
|
||||
Increase resources for the server:
|
||||
- More CPU for handling concurrent connections
|
||||
- More RAM for database caching
|
||||
- Faster storage (SSD) for database operations
|
||||
|
||||
Tune configuration:
|
||||
- Increase `max_clients_per_vault` for more concurrent users
|
||||
- Increase `max_connections_per_vault` for database performance
|
||||
- Adjust `max_body_size_mb` based on typical file sizes
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Server won't start
|
||||
|
||||
```bash
|
||||
# Check Docker logs
|
||||
docker logs vaultlink-server
|
||||
|
||||
# Common issues:
|
||||
# - Port already in use: Change port mapping
|
||||
# - Config syntax error: Validate YAML
|
||||
# - Permission error: Check volume permissions
|
||||
```
|
||||
|
||||
### High memory usage
|
||||
|
||||
- Reduce `max_connections_per_vault`
|
||||
- Reduce `max_clients_per_vault`
|
||||
- Check for large vaults (may need database optimization)
|
||||
|
||||
### Database corruption
|
||||
|
||||
```bash
|
||||
# Verify database integrity
|
||||
sqlite3 databases/your-vault.db "PRAGMA integrity_check;"
|
||||
|
||||
# If corrupted, restore from backup
|
||||
cp /backup/databases/your-vault.db /data/databases/
|
||||
```
|
||||
|
||||
### WebSocket connection drops
|
||||
|
||||
- Check reverse proxy timeout settings
|
||||
- Verify firewall isn't closing connections
|
||||
- Review client retry intervals
|
||||
- Check server logs for errors
|
||||
|
||||
## Next Steps
|
||||
|
||||
- [Configure authentication and access control →](/config/authentication)
|
||||
- [Set up Obsidian plugin →](/guide/obsidian-plugin)
|
||||
- [Deploy CLI client →](/guide/cli-client)
|
||||
- [Understand the architecture →](/architecture/)
|
||||
115
docs/guide/what-is-vaultlink.md
Normal file
115
docs/guide/what-is-vaultlink.md
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
# What is VaultLink?
|
||||
|
||||
VaultLink is a self-hosted real-time synchronization system for Obsidian vaults. It provides collaborative file syncing with automatic conflict resolution, designed for users who want complete control over their data.
|
||||
|
||||
## Overview
|
||||
|
||||
VaultLink consists of three main components:
|
||||
|
||||
### Sync Server
|
||||
|
||||
A Rust-based WebSocket server that handles:
|
||||
- Real-time bidirectional synchronization
|
||||
- Document versioning with SQLite
|
||||
- User authentication and vault access control
|
||||
- Operational transformation for conflict resolution
|
||||
|
||||
### Obsidian Plugin
|
||||
|
||||
A native Obsidian plugin that:
|
||||
- Integrates sync directly into your Obsidian workflow
|
||||
- Provides real-time updates as you edit
|
||||
- Handles file watching and automatic synchronization
|
||||
- Works across desktop and mobile platforms
|
||||
|
||||
### CLI Client
|
||||
|
||||
A standalone synchronization client that:
|
||||
- Syncs vaults without requiring Obsidian
|
||||
- Perfect for servers, automation, or backup systems
|
||||
- Provides file watching and bidirectional sync
|
||||
- Runs in Docker or as a standalone binary
|
||||
|
||||
## Key Features
|
||||
|
||||
### Real-Time Synchronization
|
||||
|
||||
Changes are synchronized immediately via WebSocket connections. When multiple users edit the same file, operational transformation ensures all edits are preserved without conflicts.
|
||||
|
||||
### Self-Hosted Architecture
|
||||
|
||||
Run the sync server on your own infrastructure:
|
||||
- Full control over data storage and access
|
||||
- No dependency on third-party services
|
||||
- Configurable authentication and authorization
|
||||
- Deploy anywhere: cloud VPS, home server, or localhost
|
||||
|
||||
### Operational Transformation
|
||||
|
||||
VaultLink uses the `reconcile-text` library for intelligent conflict resolution:
|
||||
- Simultaneous edits are automatically merged
|
||||
- No manual conflict resolution required
|
||||
- Preserves intent of all contributors
|
||||
- Works seamlessly in the background
|
||||
|
||||
### Flexible Authentication
|
||||
|
||||
Configure user access per vault:
|
||||
- Token-based authentication
|
||||
- Per-user vault access control
|
||||
- Allow-list or deny-list patterns
|
||||
- Support for multiple users and vaults
|
||||
|
||||
## Use Cases
|
||||
|
||||
### Personal Sync
|
||||
|
||||
Synchronize your Obsidian vault across multiple devices:
|
||||
- Laptop, desktop, and mobile in real-time
|
||||
- No cloud service subscription required
|
||||
- Full privacy and data control
|
||||
|
||||
### Team Collaboration
|
||||
|
||||
Share knowledge bases with teammates:
|
||||
- Real-time collaborative editing
|
||||
- Granular access control per vault
|
||||
- Self-hosted for enterprise security requirements
|
||||
|
||||
### Automated Backups
|
||||
|
||||
Use the CLI client for automated workflows:
|
||||
- Scheduled backups to remote servers
|
||||
- Integration with existing backup systems
|
||||
- Headless operation without Obsidian
|
||||
|
||||
### Development & Testing
|
||||
|
||||
Synchronize documentation across environments:
|
||||
- Keep docs in sync with development environments
|
||||
- Automated deployment of documentation
|
||||
- Version control integration
|
||||
|
||||
## How It Works
|
||||
|
||||
1. **Server Setup**: Deploy the sync server on your infrastructure
|
||||
2. **Authentication**: Configure users and vault access in `config.yml`
|
||||
3. **Client Connection**: Connect via Obsidian plugin or CLI client
|
||||
4. **Initial Sync**: Client uploads local files to server
|
||||
5. **Real-Time Updates**: Changes sync bidirectionally via WebSocket
|
||||
6. **Conflict Resolution**: Operational transformation handles simultaneous edits
|
||||
|
||||
## Technology Stack
|
||||
|
||||
- **Server**: Rust with Axum framework, SQLite database, WebSocket protocol
|
||||
- **Frontend**: TypeScript with WebSocket client, npm workspaces
|
||||
- **Sync Algorithm**: reconcile-text operational transformation library
|
||||
- **Deployment**: Docker images, binary releases, or source builds
|
||||
|
||||
## Next Steps
|
||||
|
||||
Ready to get started?
|
||||
|
||||
- [Getting Started Guide →](/guide/getting-started)
|
||||
- [Server Setup →](/guide/server-setup)
|
||||
- [Architecture Overview →](/architecture/)
|
||||
72
docs/index.md
Normal file
72
docs/index.md
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
---
|
||||
layout: home
|
||||
|
||||
hero:
|
||||
name: VaultLink
|
||||
text: Self-Hosted Sync for Obsidian
|
||||
tagline: Real-time collaborative file synchronization for your knowledge base
|
||||
image:
|
||||
src: /logo.svg
|
||||
alt: VaultLink
|
||||
actions:
|
||||
- theme: brand
|
||||
text: Get Started
|
||||
link: /guide/getting-started
|
||||
- theme: alt
|
||||
text: View on GitHub
|
||||
link: https://github.com/schmelczer/vault-link
|
||||
|
||||
features:
|
||||
- icon: 🚀
|
||||
title: Real-Time Synchronization
|
||||
details: Operational transformation-based conflict resolution ensures your files stay in sync across devices without data loss
|
||||
- icon: 🔒
|
||||
title: Self-Hosted & Private
|
||||
details: Run your own sync server. Your data stays on your infrastructure with full control over access and privacy
|
||||
- icon: 🎯
|
||||
title: Obsidian Plugin
|
||||
details: Native integration with Obsidian for seamless synchronization directly within your favorite note-taking app
|
||||
- icon: 🖥️
|
||||
title: CLI Client
|
||||
details: Sync vaults to any system using the standalone CLI client. Perfect for servers, automation, or headless setups
|
||||
- icon: ⚡
|
||||
title: Built for Performance
|
||||
details: Rust-powered WebSocket server with SQLite backend delivers blazing-fast sync performance
|
||||
- icon: 🛠️
|
||||
title: Flexible Deployment
|
||||
details: Deploy via Docker, binary releases, or build from source. Configure authentication and access controls to fit your needs
|
||||
---
|
||||
|
||||
## Quick Start
|
||||
|
||||
Deploy the sync server:
|
||||
|
||||
```bash
|
||||
docker run -d \
|
||||
-p 3000:3000 \
|
||||
-v $(pwd)/data:/data \
|
||||
ghcr.io/schmelczer/vault-link-server:latest \
|
||||
/app/sync_server config.yml
|
||||
```
|
||||
|
||||
Install the Obsidian plugin or use the CLI client:
|
||||
|
||||
```bash
|
||||
docker run -v /path/to/vault:/vault \
|
||||
ghcr.io/schmelczer/vault-link-cli:latest \
|
||||
-l /vault -r wss://your-server.com -t your-token -v default
|
||||
```
|
||||
|
||||
[Learn more →](/guide/getting-started)
|
||||
|
||||
## Why VaultLink?
|
||||
|
||||
VaultLink provides a complete self-hosted synchronization solution for Obsidian:
|
||||
|
||||
- **No third-party services**: Your data never leaves your infrastructure
|
||||
- **Operational transformation**: Smart conflict resolution that preserves all changes
|
||||
- **Multi-platform**: Works with Obsidian plugin or standalone CLI on any system
|
||||
- **Production-ready**: Docker images, health checks, and comprehensive logging
|
||||
- **Open source**: MIT licensed with active development
|
||||
|
||||
[Read the architecture overview →](/architecture/)
|
||||
18
docs/package.json
Normal file
18
docs/package.json
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
{
|
||||
"name": "docs",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"dev": "vitepress dev",
|
||||
"build": "vitepress build",
|
||||
"preview": "vitepress preview"
|
||||
},
|
||||
"keywords": [],
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"devDependencies": {
|
||||
"vitepress": "^1.6.4",
|
||||
"vue": "^3.5.24"
|
||||
}
|
||||
}
|
||||
34
docs/public/logo.svg
Normal file
34
docs/public/logo.svg
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
<svg width="200" height="200" viewBox="0 0 200 200" xmlns="http://www.w3.org/2000/svg">
|
||||
<!-- Background circle -->
|
||||
<circle cx="100" cy="100" r="95" fill="#4A90E2" opacity="0.1"/>
|
||||
|
||||
<!-- Link chain symbol -->
|
||||
<g transform="translate(100, 100)">
|
||||
<!-- Left link -->
|
||||
<path d="M -60 -10 L -30 -10 C -20 -10 -15 -5 -15 5 L -15 5 C -15 15 -20 20 -30 20 L -60 20 C -70 20 -75 15 -75 5 L -75 -5 C -75 -15 -70 -20 -60 -20 Z"
|
||||
fill="none" stroke="#4A90E2" stroke-width="8" stroke-linecap="round"/>
|
||||
|
||||
<!-- Right link -->
|
||||
<path d="M 60 -10 L 30 -10 C 20 -10 15 -5 15 5 L 15 5 C 15 15 20 20 30 20 L 60 20 C 70 20 75 15 75 5 L 75 -5 C 75 -15 70 -20 60 -20 Z"
|
||||
fill="none" stroke="#4A90E2" stroke-width="8" stroke-linecap="round"/>
|
||||
|
||||
<!-- Center connecting bar -->
|
||||
<rect x="-15" y="-6" width="30" height="12" rx="6" fill="#4A90E2"/>
|
||||
|
||||
<!-- Vault door detail -->
|
||||
<circle cx="0" cy="0" r="12" fill="none" stroke="#4A90E2" stroke-width="3"/>
|
||||
<circle cx="0" cy="0" r="6" fill="#4A90E2"/>
|
||||
|
||||
<!-- Sync arrows -->
|
||||
<g opacity="0.6">
|
||||
<!-- Top arrow -->
|
||||
<path d="M -5 -50 L 5 -50 L 0 -40 Z" fill="#4A90E2"/>
|
||||
<!-- Bottom arrow -->
|
||||
<path d="M 5 50 L -5 50 L 0 40 Z" fill="#4A90E2"/>
|
||||
</g>
|
||||
</g>
|
||||
|
||||
<!-- Text (optional) -->
|
||||
<text x="100" y="175" font-family="Arial, sans-serif" font-size="24" font-weight="bold"
|
||||
text-anchor="middle" fill="#4A90E2">VaultLink</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 1.4 KiB |
Loading…
Add table
Add a link
Reference in a new issue