andras/reconcile
1
0
Fork 0
Code Issues Pull requests 5 Projects Releases Packages Wiki Activity Actions

Add WebSocket support (#12)

Browse source
This commit is contained in:
Andras Schmelczer 2025-03-29 10:17:46 +00:00 • committed by GitHub
parent 3d27b7f313
commit 1aad0fce31
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
68 changed files with 2578 additions and 993 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/check.yml vendored
View file

@ -28,7 +28,7 @@ jobs:
cargo install sqlx-cli wasm-pack
cd backend
sqlx database create --database-url sqlite://db.sqlite3
sqlx migrate run --source sync_server/src/database/migrations --database-url sqlite://db.sqlite3
sqlx migrate run --source sync_server/src/app_state/database/migrations --database-url sqlite://db.sqlite3
- name: Build wasm
run: |

4
.github/workflows/e2e.yml vendored
View file

@ -28,7 +28,7 @@ jobs:
cargo install sqlx-cli wasm-pack
cd backend
sqlx database create --database-url sqlite://db.sqlite3
sqlx migrate run --source sync_server/src/database/migrations --database-url sqlite://db.sqlite3
sqlx migrate run --source sync_server/src/app_state/database/migrations --database-url sqlite://db.sqlite3
- name: Build wasm
run: |
@ -38,7 +38,7 @@ jobs:
- name: E2E tests
run: |
cd backend
RUST_BACKTRACE=1 cargo run -p sync_server config-e2e.yml &
cargo run -p sync_server config-e2e.yml --color never &
cd ..
scripts/update-api-types.sh

2
.gitignore vendored
View file

@ -14,4 +14,4 @@ backend/databases
*.log
plugin/coverage
*.sqlx

17
backend/Cargo.lock generated
View file

@ -459,6 +459,16 @@ dependencies = [
"clap_derive",
]
[[package]]
name = "clap-verbosity-flag"
version = "3.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2678fade3b77aa3a8ff3aae87e9c008d3fb00473a41c71fbf74e91c8c7b37e84"
dependencies = [
"clap",
"log",
]
[[package]]
name = "clap_builder"
version = "4.5.32"
@ -2069,9 +2079,9 @@ dependencies = [
[[package]]
name = "serde_json"
version = "1.0.133"
version = "1.0.140"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377"
checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373"
dependencies = [
"itoa",
"memchr",
@ -2524,6 +2534,8 @@ dependencies = [
"axum_typed_multipart",
"chrono",
"clap",
"clap-verbosity-flag",
"futures",
"log",
"rand",
"reconcile",
@ -2531,6 +2543,7 @@ dependencies = [
"sanitize-filename",
"schemars",
"serde",
"serde_json",
"serde_yaml",
"sqlx",
"sync_lib",

1
backend/Cargo.toml
View file

@ -22,6 +22,7 @@ thiserror = { version = "1.0.66", default-features = false }
codegen-units = 1
lto = true
opt-level = 3
strip="debuginfo" # Keep some info for better panics
[workspace.lints.rust]
unsafe_code = "forbid"

2
backend/Dockerfile
View file

@ -8,7 +8,7 @@ RUN cargo install sqlx-cli
COPY . .
RUN sqlx database create --database-url sqlite://db.sqlite3
RUN sqlx migrate run --source sync_server/src/database/migrations --database-url sqlite://db.sqlite3
RUN sqlx migrate run --source sync_server/src/app_state/database/migrations --database-url sqlite://db.sqlite3
RUN cargo build --package sync_server --release --target x86_64-unknown-linux-musl

3
backend/config-e2e.yml
View file

@ -1,10 +1,13 @@
database:
databases_directory_path: databases
max_connections: 12
server:
host: 0.0.0.0
port: 3000
max_body_size_mb: 512
max_clients_per_vault: 256
users:
user_tokens:
- name: admin

3
backend/sync_server/Cargo.toml
View file

@ -34,6 +34,9 @@ sanitize-filename = "0.6.0"
axum-jsonschema = { version = "0.8.0", features = ["aide"] }
regex = "1.11.1"
clap = { version = "4.5.32", features = ["derive"] }
futures = "0.3.31"
serde_json = "1.0.140"
clap-verbosity-flag = "3.0.2"
[lints]
workspace = true

2
backend/sync_server/README.md
View file

@ -1,4 +1,4 @@
cargo install sqlx-cli
rm db.sqlite3; sqlx database create --database-url sqlite://db.sqlite3
sqlx migrate run --source sync_server/src/database/migrations --database-url sqlite://db.sqlite3
sqlx migrate run --source sync_server/src/app_state/database/migrations --database-url sqlite://db.sqlite3

15
backend/sync_server/src/server/app_state.rs → backend/sync_server/src/app_state.rs
View file

@ -1,13 +1,19 @@
pub mod broadcasts;
pub mod database;
use std::ffi::OsString;
use anyhow::Result;
use broadcasts::Broadcasts;
use database::Database;
use crate::{config::Config, consts::DEFAULT_CONFIG_PATH, database::Database};
use crate::{config::Config, consts::DEFAULT_CONFIG_PATH};
#[derive(Clone, Debug)]
pub struct AppState {
pub config: Config,
pub database: Database,
pub broadcasts: Broadcasts,
}
impl AppState {
@ -17,7 +23,12 @@ impl AppState {
let config = Config::read_or_create(&path).await?;
let database = Database::try_new(&config.database).await?;
let broadcasts = Broadcasts::new(&config.server);
Ok(Self { config, database })
Ok(Self {
config,
database,
broadcasts,
})
}
}

57
backend/sync_server/src/app_state/broadcasts.rs Normal file
View file

@ -0,0 +1,57 @@
use std::{collections::HashMap, sync::Arc};
use anyhow::Context;
use tokio::sync::{Mutex, broadcast};
use super::database::models::{DocumentVersionWithoutContent, VaultId};
use crate::{config::server_config::ServerConfig, errors::server_error};
#[derive(Debug, Clone)]
pub struct Broadcasts {
max_clients_per_vault: usize,
tx: Arc<Mutex<HashMap<VaultId, broadcast::Sender<DocumentVersionWithoutContent>>>>,
}
impl Broadcasts {
pub fn new(server_config: &ServerConfig) -> Self {
Self {
max_clients_per_vault: server_config.max_clients_per_vault,
tx: Arc::new(Mutex::new(HashMap::new())),
}
}
pub async fn get_receiver(
&self,
vault: VaultId,
) -> broadcast::Receiver<DocumentVersionWithoutContent> {
let tx = self.get_or_create(vault).await;
tx.subscribe()
}
/// Sent a document update to all clients subscribed to the vault.
/// We ignore & log failures.
pub async fn send(&self, vault: VaultId, document: DocumentVersionWithoutContent) {
let tx = self.get_or_create(vault).await;
let result = tx
.send(document)
.context("Cannot broadcast update message to websocket listeners")
.map_err(server_error);
if result.is_err() {
log::debug!("Failed to send message: {result:?}");
}
}
async fn get_or_create(
&self,
vault: VaultId,
) -> broadcast::Sender<DocumentVersionWithoutContent> {
let mut tx = self.tx.lock().await;
tx.entry(vault)
.or_insert_with(|| broadcast::channel(self.max_clients_per_vault).0.clone())
.clone()
}
}

25
backend/sync_server/src/database.rs → backend/sync_server/src/app_state/database.rs
View file

@ -85,13 +85,13 @@ impl Database {
}
async fn run_migrations(pool: &Pool<Sqlite>) -> Result<()> {
sqlx::migrate!("src/database/migrations")
sqlx::migrate!("src/app_state/database/migrations")
.run(pool)
.await
.context("Cannot check for pending migrations")
}
async fn get_connection_pool(&mut self, vault: &VaultId) -> Result<Pool<Sqlite>> {
async fn get_connection_pool(&self, vault: &VaultId) -> Result<Pool<Sqlite>> {
let mut pools = self.connection_pools.lock().await;
if !pools.contains_key(vault) {
let pool = Self::create_vault_database(&self.config, vault).await?;
@ -108,7 +108,7 @@ impl Database {
/// Attempting to write from this transaction might result in a
/// database locked error. Use this transaction for read-only operations.
pub async fn create_readonly_transaction(
&mut self,
&self,
vault: &VaultId,
) -> Result<Transaction<'static>> {
self.get_connection_pool(vault)
@ -118,10 +118,7 @@ impl Database {
.context("Cannot create transaction")
}
pub async fn create_write_transaction(
&mut self,
vault: &VaultId,
) -> Result<Transaction<'static>> {
pub async fn create_write_transaction(&self, vault: &VaultId) -> Result<Transaction<'static>> {
let mut transaction = self.create_readonly_transaction(vault).await?;
// sqlx doesn't support immediate transactions for sqlite: https://github.com/launchbadge/sqlx/issues/481
@ -134,7 +131,7 @@ impl Database {
/// Return the latest state of all documents in the vault
pub async fn get_latest_documents(
&mut self,
&self,
vault: &VaultId,
transaction: Option<&mut Transaction<'_>>,
) -> Result<Vec<DocumentVersionWithoutContent>> {
@ -165,7 +162,7 @@ impl Database {
/// Return the latest state of all documents (including deleted) in the
/// vault which have changed since the given update id
pub async fn get_latest_documents_since(
&mut self,
&self,
vault: &VaultId,
vault_update_id: VaultUpdateId,
transaction: Option<&mut Transaction<'_>>,
@ -199,7 +196,7 @@ impl Database {
}
pub async fn get_max_update_id_in_vault(
&mut self,
&self,
vault: &VaultId,
transaction: Option<&mut Transaction<'_>>,
) -> Result<i64> {
@ -222,7 +219,7 @@ impl Database {
}
pub async fn get_latest_document_by_path(
&mut self,
&self,
vault: &VaultId,
relative_path: &str,
transaction: Option<&mut Transaction<'_>>,
@ -258,7 +255,7 @@ impl Database {
}
pub async fn get_latest_document(
&mut self,
&self,
vault: &VaultId,
document_id: &DocumentId,
transaction: Option<&mut Transaction<'_>>,
@ -291,7 +288,7 @@ impl Database {
}
pub async fn get_document_version(
&mut self,
&self,
vault: &VaultId,
vault_update_id: VaultUpdateId,
transaction: Option<&mut Transaction<'_>>,
@ -322,7 +319,7 @@ impl Database {
}
pub async fn insert_document_version(
&mut self,
&self,
vault: &VaultId,
version: &StoredDocumentVersion,
transaction: Option<&mut Transaction<'_>>,

0
backend/sync_server/src/database/migrations/20241207143519_bootstrap.sql → backend/sync_server/src/app_state/database/migrations/20241207143519_bootstrap.sql
View file

0
backend/sync_server/src/database/models.rs → backend/sync_server/src/app_state/database/models.rs
View file

1
backend/sync_server/src/cli.rs
View file

@ -1 +1,2 @@
pub mod args;
pub mod color_when;

28
backend/sync_server/src/cli/args.rs
View file

@ -1,38 +1,26 @@
use std::ffi::OsString;
use clap::{Parser, ValueEnum};
use clap::Parser;
use clap_verbosity_flag::{InfoLevel, Verbosity};
/// Server for backing the VaultLink plugin
use crate::cli::color_when::ColorWhen;
/// Server for backing the `VaultLink` plugin
#[derive(Parser, Debug)]
#[command(version, about, long_about = None)]
pub struct Args {
#[arg(index = 1)]
pub config_path: Option<OsString>,
#[command(flatten)]
pub verbose: Verbosity<InfoLevel>,
#[arg(
long,
require_equals = true,
value_name = "WHEN",
num_args = 0..=1,
default_value_t = ColorWhen::Auto,
default_missing_value = "always",
value_enum
)]
pub color: ColorWhen,
}
#[derive(ValueEnum, Copy, Clone, Debug, PartialEq, Eq)]
pub enum ColorWhen {
Always,
Auto,
Never,
}
impl std::fmt::Display for ColorWhen {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.to_possible_value()
.expect("no values are skipped")
.get_name()
.fmt(f)
}
}

31
backend/sync_server/src/cli/color_when.rs Normal file
View file

@ -0,0 +1,31 @@
use std::io::IsTerminal;
use clap::ValueEnum;
#[derive(ValueEnum, Copy, Clone, Debug, PartialEq, Eq)]
pub enum ColorWhen {
Always,
Auto,
Never,
}
impl ColorWhen {
pub fn use_colors(self) -> bool {
match self {
ColorWhen::Always => true,
ColorWhen::Auto => {
std::env::var_os("NO_COLOR").is_none() && std::io::stderr().is_terminal()
}
ColorWhen::Never => false,
}
}
}
impl std::fmt::Display for ColorWhen {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.to_possible_value()
.expect("no values are skipped")
.get_name()
.fmt(f)
}
}

14
backend/sync_server/src/config/server_config.rs
View file

@ -1,7 +1,10 @@
use log::debug;
use serde::{Deserialize, Serialize};
use crate::consts::{DEFAULT_HOST, DEFAULT_MAX_BODY_SIZE_MB, DEFAULT_PORT};
use crate::consts::{
DEFAULT_HOST, DEFAULT_MAX_BODY_SIZE_MB, DEFAULT_MAX_CLIENTS_PER_VAULT, DEFAULT_PORT,
};
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct ServerConfig {
#[serde(default = "default_host")]
@ -12,6 +15,9 @@ pub struct ServerConfig {
#[serde(default = "default_max_body_size_mb")]
pub max_body_size_mb: usize,
#[serde(default = "default_max_clients_per_vault")]
pub max_clients_per_vault: usize,
}
fn default_host() -> String {
@ -29,12 +35,18 @@ fn default_max_body_size_mb() -> usize {
DEFAULT_MAX_BODY_SIZE_MB
}
fn default_max_clients_per_vault() -> usize {
debug!("Using default max clients per vault: {DEFAULT_MAX_CLIENTS_PER_VAULT}");
DEFAULT_MAX_CLIENTS_PER_VAULT
}
impl Default for ServerConfig {
fn default() -> Self {
Self {
host: default_host(),
port: default_port(),
max_body_size_mb: default_max_body_size_mb(),
max_clients_per_vault: default_max_clients_per_vault(),
}
}
}

1
backend/sync_server/src/consts.rs
View file

@ -4,3 +4,4 @@ pub const DEFAULT_HOST: &str = "127.0.0.1";
pub const DEFAULT_PORT: u16 = 3000;
pub const DEFAULT_MAX_CONNECTIONS: u32 = 12;
pub const DEFAULT_MAX_BODY_SIZE_MB: usize = 4096;
pub const DEFAULT_MAX_CLIENTS_PER_VAULT: usize = 256;

67
backend/sync_server/src/main.rs
View file

@ -1,38 +1,79 @@
mod app_state;
mod cli;
mod config;
mod consts;
mod database;
mod errors;
mod server;
mod utils;
use std::process::ExitCode;
use anyhow::{Context as _, Result};
use clap::Parser;
use cli::args::Args;
use errors::{SyncServerError, init_error};
use log::info;
use server::create_server;
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
use tracing_subscriber::{EnvFilter, fmt::format, util::SubscriberInitExt};
#[tokio::main]
async fn main() -> Result<(), SyncServerError> {
async fn main() -> ExitCode {
let args = Args::parse();
tracing_subscriber::registry()
.with(
tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| {
format!(
"{}=debug,tower_http=debug,axum::rejection=trace",
env!("CARGO_CRATE_NAME")
)
.into()
}),
let mut result = set_up_logging(&args);
if result.is_ok() {
result = start_server(args).await;
}
match result {
Ok(()) => ExitCode::SUCCESS,
Err(e) => {
eprintln!("Failed to set up logging: {e}");
ExitCode::FAILURE
}
}
}
fn set_up_logging(args: &Args) -> Result<(), SyncServerError> {
let level_filter = match args.verbose.log_level_filter() {
// We don't want to allow disabling all logging
log::LevelFilter::Off | log::LevelFilter::Error => tracing::Level::ERROR,
log::LevelFilter::Warn => tracing::Level::WARN,
log::LevelFilter::Info => tracing::Level::INFO,
log::LevelFilter::Debug => tracing::Level::DEBUG,
log::LevelFilter::Trace => tracing::Level::TRACE,
};
let env_filter = EnvFilter::builder()
.with_default_directive(level_filter.into())
.from_env()
.context("Failed to create logging env filter")
.map_err(init_error)?;
let use_colors = args.color.use_colors();
let is_debug_mode = args.verbose.log_level_filter() >= log::LevelFilter::Debug;
tracing_subscriber::fmt()
.with_ansi(use_colors)
.with_env_filter(env_filter)
.event_format(
format()
.without_time()
.with_target(is_debug_mode)
.with_line_number(is_debug_mode)
.compact(),
)
.with(tracing_subscriber::fmt::layer())
.finish()
.try_init()
.context("Failed to initialise tracing")
.map_err(init_error)?;
Ok(())
}
async fn start_server(args: Args) -> Result<(), SyncServerError> {
info!(
"Starting VaultLink server version {}",
env!("CARGO_PKG_VERSION")

28
backend/sync_server/src/server.rs
View file

@ -1,3 +1,16 @@
mod auth;
mod create_document;
mod delete_document;
mod fetch_document_version;
mod fetch_document_version_content;
mod fetch_latest_document_version;
mod fetch_latest_documents;
mod ping;
mod requests;
mod responses;
mod update_document;
mod websocket;
use std::{ffi::OsString, sync::Arc};
use aide::{
@ -10,7 +23,6 @@ use aide::{
transform::TransformOpenApi,
};
use anyhow::{Context as _, Result, anyhow};
use app_state::AppState;
use axum::{
Extension, Json,
extract::{DefaultBodyLimit, Request},
@ -32,21 +44,10 @@ use tower_http::{
use tracing::{Level, info_span};
use crate::{
app_state::AppState,
config::server_config::ServerConfig,
errors::{SerializedError, not_found_error},
};
mod app_state;
mod auth;
mod create_document;
mod delete_document;
mod fetch_document_version;
mod fetch_document_version_content;
mod fetch_latest_document_version;
mod fetch_latest_documents;
mod ping;
mod requests;
mod responses;
mod update_document;
pub async fn create_server(config_path: Option<OsString>) -> Result<()> {
aide::r#gen::on_error(|err| error!("{err}"));
@ -65,6 +66,7 @@ pub async fn create_server(config_path: Option<OsString>) -> Result<()> {
"/vaults/:vault_id/documents",
get(fetch_latest_documents::fetch_latest_documents),
)
.route("/vaults/:vault_id/ws", get(websocket::websocket_handler))
.api_route(
"/vaults/:vault_id/documents",
post(create_document::create_document_multipart),

3
backend/sync_server/src/server/auth.rs
View file

@ -1,9 +1,10 @@
use super::app_state::AppState;
use crate::{
app_state::AppState,
config::user_config::User,
errors::{SyncServerError, unauthorized_error},
};
// TODO: turn this into a middleware
pub fn auth(app_state: &AppState, token: &str) -> Result<User, SyncServerError> {
app_state
.config

15
backend/sync_server/src/server/create_document.rs
View file

@ -11,12 +11,16 @@ use serde::Deserialize;
use sync_lib::base64_to_bytes;
use super::{
app_state::AppState,
auth::auth,
requests::{CreateDocumentVersion, CreateDocumentVersionMultipart},
};
use crate::{
database::models::{DocumentId, DocumentVersionWithoutContent, StoredDocumentVersion, VaultId},
app_state::{
AppState,
database::models::{
DocumentId, DocumentVersionWithoutContent, StoredDocumentVersion, VaultId,
},
},
errors::{SyncServerError, client_error, server_error},
utils::sanitize_path,
};
@ -77,7 +81,7 @@ pub async fn create_document_json(
async fn internal_create_document(
auth_header: Authorization<Bearer>,
mut state: AppState,
state: AppState,
vault_id: VaultId,
document_id: Option<DocumentId>,
relative_path: String,
@ -139,5 +143,10 @@ async fn internal_create_document(
.context("Failed to commit successful transaction")
.map_err(server_error)?;
state
.broadcasts
.send(vault_id, new_version.clone().into())
.await;
Ok(Json(new_version.into()))
}

16
backend/sync_server/src/server/delete_document.rs
View file

@ -8,9 +8,14 @@ use axum_jsonschema::Json;
use schemars::JsonSchema;
use serde::Deserialize;
use super::{app_state::AppState, auth::auth, requests::DeleteDocumentVersion};
use super::{auth::auth, requests::DeleteDocumentVersion};
use crate::{
database::models::{DocumentId, DocumentVersionWithoutContent, StoredDocumentVersion, VaultId},
app_state::{
AppState,
database::models::{
DocumentId, DocumentVersionWithoutContent, StoredDocumentVersion, VaultId,
},
},
errors::{SyncServerError, server_error},
utils::sanitize_path,
};
@ -29,7 +34,7 @@ pub async fn delete_document(
vault_id,
document_id,
}): Path<DeleteDocumentPathParams>,
State(mut state): State<AppState>,
State(state): State<AppState>,
Json(request): Json<DeleteDocumentVersion>,
) -> Result<Json<DocumentVersionWithoutContent>, SyncServerError> {
auth(&state, auth_header.token())?;
@ -67,5 +72,10 @@ pub async fn delete_document(
.context("Failed to commit successful transaction")
.map_err(server_error)?;
state
.broadcasts
.send(vault_id, new_version.clone().into())
.await;
Ok(Json(new_version.into()))
}

9
backend/sync_server/src/server/fetch_document_version.rs
View file

@ -8,9 +8,12 @@ use axum_jsonschema::Json;
use schemars::JsonSchema;
use serde::Deserialize;
use super::{app_state::AppState, auth::auth};
use super::auth::auth;
use crate::{
database::models::{DocumentId, DocumentVersion, VaultId, VaultUpdateId},
app_state::{
AppState,
database::models::{DocumentId, DocumentVersion, VaultId, VaultUpdateId},
},
errors::{SyncServerError, not_found_error, server_error},
};
@ -30,7 +33,7 @@ pub async fn fetch_document_version(
document_id,
vault_update_id,
}): Path<FetchDocumentVersionPathParams>,
State(mut state): State<AppState>,
State(state): State<AppState>,
) -> Result<Json<DocumentVersion>, SyncServerError> {
auth(&state, auth_header.token())?;

9
backend/sync_server/src/server/fetch_document_version_content.rs
View file

@ -10,9 +10,12 @@ use axum_extra::{
use schemars::JsonSchema;
use serde::Deserialize;
use super::{app_state::AppState, auth::auth};
use super::auth::auth;
use crate::{
database::models::{DocumentId, VaultId, VaultUpdateId},
app_state::{
AppState,
database::models::{DocumentId, VaultId, VaultUpdateId},
},
errors::{SyncServerError, not_found_error, server_error},
};
@ -32,7 +35,7 @@ pub async fn fetch_document_version_content(
document_id,
vault_update_id,
}): Path<FetchDocumentVersionContentPathParams>,
State(mut state): State<AppState>,
State(state): State<AppState>,
) -> Result<Bytes, SyncServerError> {
auth(&state, auth_header.token())?;

9
backend/sync_server/src/server/fetch_latest_document_version.rs
View file

@ -8,9 +8,12 @@ use axum_jsonschema::Json;
use schemars::JsonSchema;
use serde::Deserialize;
use super::{app_state::AppState, auth::auth};
use super::auth::auth;
use crate::{
database::models::{DocumentId, DocumentVersion, VaultId},
app_state::{
AppState,
database::models::{DocumentId, DocumentVersion, VaultId},
},
errors::{SyncServerError, not_found_error, server_error},
};
@ -28,7 +31,7 @@ pub async fn fetch_latest_document_version(
vault_id,
document_id,
}): Path<FetchLatestDocumentVersionPathParams>,
State(mut state): State<AppState>,
State(state): State<AppState>,
) -> Result<Json<DocumentVersion>, SyncServerError> {
auth(&state, auth_header.token())?;

9
backend/sync_server/src/server/fetch_latest_documents.rs
View file

@ -7,9 +7,12 @@ use axum_jsonschema::Json;
use schemars::JsonSchema;
use serde::Deserialize;
use super::{app_state::AppState, auth::auth, responses::FetchLatestDocumentsResponse};
use super::{auth::auth, responses::FetchLatestDocumentsResponse};
use crate::{
database::models::{VaultId, VaultUpdateId},
app_state::{
AppState,
database::models::{VaultId, VaultUpdateId},
},
errors::{SyncServerError, server_error},
};
@ -30,7 +33,7 @@ pub async fn fetch_latest_documents(
TypedHeader(auth_header): TypedHeader<Authorization<Bearer>>,
Path(FetchLatestDocumentsPathParams { vault_id }): Path<FetchLatestDocumentsPathParams>,
Query(QueryParams { since_update_id }): Query<QueryParams>,
State(mut state): State<AppState>,
State(state): State<AppState>,
) -> Result<Json<FetchLatestDocumentsResponse>, SyncServerError> {
auth(&state, auth_header.token())?;

4
backend/sync_server/src/server/ping.rs
View file

@ -4,8 +4,8 @@ use axum_extra::{
headers::{Authorization, authorization::Bearer},
};
use super::{app_state::AppState, auth::auth, responses::PingResponse};
use crate::errors::SyncServerError;
use super::{auth::auth, responses::PingResponse};
use crate::{app_state::AppState, errors::SyncServerError};
#[axum::debug_handler]
pub async fn ping(

2
backend/sync_server/src/server/requests.rs
View file

@ -4,7 +4,7 @@ use axum_typed_multipart::TryFromMultipart;
use schemars::JsonSchema;
use serde::{self, Deserialize};
use crate::database::models::{DocumentId, VaultUpdateId};
use crate::app_state::database::models::{DocumentId, VaultUpdateId};
#[derive(Debug, Deserialize, JsonSchema)]
#[serde(rename_all = "camelCase")]

4
backend/sync_server/src/server/responses.rs
View file

@ -1,7 +1,9 @@
use schemars::JsonSchema;
use serde::{self, Serialize};
use crate::database::models::{DocumentVersion, DocumentVersionWithoutContent, VaultUpdateId};
use crate::app_state::database::models::{
DocumentVersion, DocumentVersionWithoutContent, VaultUpdateId,
};
/// Response to a ping request.
#[derive(Debug, Clone, Serialize, JsonSchema)]

13
backend/sync_server/src/server/update_document.rs
View file

@ -12,13 +12,15 @@ use serde::Deserialize;
use sync_lib::{base64_to_bytes, is_file_type_mergable, merge};
use super::{
app_state::AppState,
auth::auth,
requests::{UpdateDocumentVersion, UpdateDocumentVersionMultipart},
responses::DocumentUpdateResponse,
};
use crate::{
database::models::{DocumentId, StoredDocumentVersion, VaultId, VaultUpdateId},
app_state::{
AppState,
database::models::{DocumentId, StoredDocumentVersion, VaultId, VaultUpdateId},
},
errors::{SyncServerError, client_error, not_found_error, server_error},
utils::{deduped_file_paths, sanitize_path},
};
@ -83,7 +85,7 @@ pub async fn update_document_json(
#[allow(clippy::too_many_arguments, clippy::too_many_lines)]
async fn internal_update_document(
auth_header: Authorization<Bearer>,
mut state: AppState,
state: AppState,
vault_id: VaultId,
document_id: DocumentId,
parent_version_id: VaultUpdateId,
@ -216,6 +218,11 @@ async fn internal_update_document(
.context("Failed to commit successful transaction")
.map_err(server_error)?;
state
.broadcasts
.send(vault_id, new_version.clone().into())
.await;
Ok(Json(if is_different_from_request_content {
DocumentUpdateResponse::MergingUpdate(new_version.into())
} else {

147
backend/sync_server/src/server/websocket.rs Normal file
View file

@ -0,0 +1,147 @@
use anyhow::Context;
use axum::{
extract::{
Path, Query, State,
ws::{Message, WebSocket, WebSocketUpgrade},
},
response::Response,
};
use futures::{
sink::SinkExt,
stream::{SplitSink, StreamExt},
};
use log::{error, info, warn};
use schemars::JsonSchema;
use serde::Deserialize;
use super::auth::auth;
use crate::{
app_state::{
AppState,
database::models::{DocumentVersionWithoutContent, VaultId, VaultUpdateId},
},
errors::{SyncServerError, server_error, unauthorized_error},
};
// This is required for aide to infer the path parameter types and names
#[derive(Deserialize, JsonSchema)]
pub struct WebsocketPathParams {
vault_id: VaultId,
}
// This is required for aide to infer the path parameter types and names
#[derive(Deserialize, JsonSchema)]
pub struct QueryParams {
since_update_id: Option<VaultUpdateId>,
}
pub async fn websocket_handler(
ws: WebSocketUpgrade,
Path(WebsocketPathParams { vault_id }): Path<WebsocketPathParams>,
Query(QueryParams { since_update_id }): Query<QueryParams>,
State(state): State<AppState>,
) -> Result<Response, SyncServerError> {
Ok(ws.on_upgrade(move |socket| websocket_wrapped(state, socket, vault_id, since_update_id)))
}
async fn websocket_wrapped(
state: AppState,
stream: WebSocket,
vault_id: VaultId,
since_update_id: Option<VaultUpdateId>,
) {
info!("Websocket connection opened on vault '{}'", vault_id);
let result = websocket(state, stream, vault_id.clone(), since_update_id).await;
if let Err(err) = result {
error!(
"Websocket connection error on vault '{}': {}",
vault_id, err
);
}
warn!("Websocket connection closed on vault '{}'", vault_id);
}
async fn websocket(
state: AppState,
stream: WebSocket,
vault_id: VaultId,
since_update_id: Option<VaultUpdateId>,
) -> Result<(), SyncServerError> {
let (mut sender, mut receiver) = stream.split();
if let Some(Ok(Message::Text(token))) = receiver.next().await {
auth(&state, &token)?;
} else {
return Err(unauthorized_error(anyhow::anyhow!(
"Failed to authenticate"
)));
}
let mut rx = state.broadcasts.get_receiver(vault_id.clone()).await;
let documents = if let Some(since_update_id) = since_update_id {
state
.database
.get_latest_documents_since(&vault_id, since_update_id, None)
.await
.map_err(server_error)
} else {
state
.database
.get_latest_documents(&vault_id, None)
.await
.map_err(server_error)
}?;
for document in documents {
send_document_over_websocket(document, &mut sender).await?;
}
let mut send_task = tokio::spawn(async move {
while let Ok(update) = rx.recv().await {
send_document_over_websocket(update, &mut sender).await?;
}
Ok::<(), SyncServerError>(())
});
let mut recv_task =
tokio::spawn(
async move { while let Some(Ok(Message::Text(_text))) = receiver.next().await {} },
);
tokio::select! {
_ = &mut send_task => recv_task.abort(),
_ = &mut recv_task => send_task.abort(),
};
send_task
.await
.context("Websocket send task failed")
.map_err(server_error)??;
recv_task
.await
.context("Websocket receive task failed")
.map_err(server_error)?;
Ok(())
}
async fn send_document_over_websocket(
document: DocumentVersionWithoutContent,
sender: &mut SplitSink<WebSocket, Message>,
) -> Result<(), SyncServerError> {
let serialized_update = serde_json::to_string(&document)
.context("Failed to serialize update")
.map_err(server_error)?;
sender
.send(Message::Text(serialized_update))
.await
.context("Failed to send message over websocket")
.map_err(server_error)
}

58
frontend/obsidian-plugin/src/obisidan-event-handler.ts
View file

@ -1,58 +0,0 @@
import type { SyncClient } from "sync-client";
import type { TAbstractFile } from "obsidian";
import { TFile } from "obsidian";
export class ObsidianFileEventHandler {
public constructor(private readonly client: SyncClient) {}
public async onCreate(file: TAbstractFile): Promise<void> {
if (file instanceof TFile) {
this.client.logger.info(`File created: ${file.path}`);
await this.client.syncLocallyCreatedFile(file.path);
} else {
this.client.logger.debug(`Folder created: ${file.path}, ignored`);
}
}
public async onDelete(file: TAbstractFile): Promise<void> {
if (file instanceof TFile) {
this.client.logger.info(`File deleted: ${file.path}`);
await this.client.syncLocallyDeletedFile(file.path);
} else {
this.client.logger.debug(`Folder deleted: ${file.path}, ignored`);
}
}
public async onRename(file: TAbstractFile, oldPath: string): Promise<void> {
if (file instanceof TFile) {
this.client.logger.info(`File renamed: ${oldPath} -> ${file.path}`);
await this.client.syncLocallyUpdatedFile({
oldPath,
relativePath: file.path
});
} else {
this.client.logger.debug(
`Folder renamed: ${oldPath} -> ${file.path}, ignored`
);
}
}
public async onModify(file: TAbstractFile): Promise<void> {
if (file instanceof TFile) {
if (file.basename.startsWith("console-log.iPhone")) {
return;
}
this.client.logger.info(`File modified: ${file.path}`);
await this.client.syncLocallyUpdatedFile({
relativePath: file.path
});
} else {
this.client.logger.debug(`Folder modified: ${file.path}, ignored`);
}
}
}

40
frontend/obsidian-plugin/src/obsidian-file-system.ts
View file

@ -1,23 +1,38 @@
import type { Stat, Vault } from "obsidian";
import { normalizePath } from "obsidian";
import type { Stat, Vault, Workspace } from "obsidian";
import { MarkdownView, normalizePath } from "obsidian";
import type { FileSystemOperations, RelativePath } from "sync-client";
export class ObsidianFileSystemOperations implements FileSystemOperations {
public constructor(private readonly vault: Vault) {}
public constructor(
private readonly vault: Vault,
private readonly workspace: Workspace
) {}
public async listAllFiles(): Promise<RelativePath[]> {
return this.vault.getFiles().map((file) => file.path);
}
public async read(path: RelativePath): Promise<Uint8Array> {
return new Uint8Array(
await this.vault.adapter.readBinary(normalizePath(path))
);
path = normalizePath(path);
const view = this.workspace.getActiveViewOfType(MarkdownView);
if (view?.file?.path === path) {
return new TextEncoder().encode(view.editor.getValue());
}
return new Uint8Array(await this.vault.adapter.readBinary(path));
}
public async write(path: RelativePath, content: Uint8Array): Promise<void> {
path = normalizePath(path);
const view = this.workspace.getActiveViewOfType(MarkdownView);
if (view?.file?.path === path) {
view.editor.setValue(new TextDecoder().decode(content));
return;
}
return this.vault.adapter.writeBinary(
normalizePath(path),
path,
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
content.buffer as ArrayBuffer
);
@ -27,7 +42,16 @@ export class ObsidianFileSystemOperations implements FileSystemOperations {
path: RelativePath,
updater: (currentContent: string) => string
): Promise<string> {
return this.vault.adapter.process(normalizePath(path), updater);
path = normalizePath(path);
const view = this.workspace.getActiveViewOfType(MarkdownView);
if (view?.file?.path === path) {
const result = updater(view.editor.getValue());
view.editor.setValue(result);
return result;
}
return this.vault.adapter.process(path, updater);
}
public async getFileSize(path: RelativePath): Promise<number> {

185
frontend/obsidian-plugin/src/styles.scss
View file

@ -1,185 +0,0 @@
@mixin number-card {
padding: var(--size-2-1) var(--size-4-1);
border-radius: var(--radius-s);
background-color: var(--color-base-30);
font-size: var(--font-ui-small);
&.good {
background-color: rgba(var(--color-green-rgb), 0.35);
}
&.bad {
background-color: rgba(var(--color-red-rgb), 0.35);
}
}
.status-description {
margin: var(--p-spacing) 0;
.number {
@include number-card;
font-family: var(--font-monospace);
font-weight: var(--bold-weight);
}
.error {
color: rgb(var(--color-red-rgb));
}
.warning {
color: rgb(var(--color-yellow-rgb));
}
}
.vault-link-settings {
h2 {
display: flex;
align-items: center;
font-size: var(--h2-size);
.version {
@include number-card;
margin: var(--size-2-2) 0 0 var(--size-4-2);
background-color: var(--color-base-30);
color: var(--color-base-70);
font-size: var(--font-ui-smaller);
}
}
.button-container {
display: flex;
gap: var(--size-4-2);
}
h3 {
font-size: var(--font-ui-large);
margin-top: var(--heading-spacing);
}
button,
input[type="range"],
.checkbox-container,
.slider::-webkit-slider-thumb {
cursor: pointer;
}
input[type="text"],
textarea {
width: 250px;
}
textarea {
resize: none;
height: 75px;
}
}
.sync-status {
display: flex;
gap: var(--size-4-2);
* {
display: block;
}
.initialize-button {
padding: 0 var(--size-4-2);
background: rgba(var(--color-red-rgb), 0.4);
cursor: pointer;
}
}
.logs-view {
display: flex;
flex-direction: column;
.logs-container {
max-width: 100%;
overflow-y: auto;
.log-message {
font: var(--font-monospace);
margin-bottom: var(--size-2-1);
overflow-wrap: break-word;
white-space: pre-wrap;
user-select: all;
.timestamp {
@include number-card;
font-family: var(--font-monospace);
font-weight: var(--bold-weight);
margin-right: var(--size-4-1);
}
&.DEBUG {
color: var(--color-base-50);
}
&.INFO {
color: var(--color-green-rgb);
}
&.WARNING {
color: var(--color-yellow-rgb);
}
&.ERROR {
color: var(--color-red-rgb);
}
}
}
}
.history-card {
padding: var(--size-4-4);
margin: var(--size-4-2);
background-color: var(--color-base-00);
border-radius: var(--radius-l);
container-type: inline-size;
&.clickable {
cursor: pointer;
}
&.success {
background-color: rgba(var(--color-green-rgb), 0.2);
}
&.error {
background-color: rgba(var(--color-red-rgb), 0.2);
}
.history-card-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: var(--size-4-2);
gap: var(--size-4-2);
@container (max-width: 300px) {
flex-direction: column;
align-items: flex-start;
}
.history-card-title {
font: var(--font-monospace);
display: flex;
align-items: center;
gap: var(--size-4-2);
word-break: break-all;
margin: 0;
}
.history-card-timestamp {
font-size: var(--font-ui-small);
font-style: italic;
color: var(--italic-color);
}
}
.history-card-message {
font-size: var(--font-ui-medium);
color: var(--color-base-70);
margin: 0;
}
}

103
frontend/obsidian-plugin/src/vault-link-plugin.ts
View file

@ -1,20 +1,25 @@
import type { WorkspaceLeaf } from "obsidian";
import { Platform, Plugin } from "obsidian";
import "./styles.scss";
import type {
Editor,
MarkdownFileInfo,
MarkdownView,
TAbstractFile,
WorkspaceLeaf
} from "obsidian";
import { Platform, Plugin, TFile } from "obsidian";
import "../manifest.json";
import { SyncSettingsTab } from "./views/settings-tab";
import { HistoryView } from "./views/history-view";
import { ObsidianFileEventHandler } from "./obisidan-event-handler";
import { StatusBar } from "./views/status-bar";
import { LogsView } from "./views/logs-view";
import { StatusDescription } from "./views/status-description";
import { HistoryView } from "./views/history/history-view";
import { StatusBar } from "./views/status-bar/status-bar";
import { LogsView } from "./views/logs/logs-view";
import { StatusDescription } from "./views/status-description/status-description";
import type { LogLine } from "sync-client";
import { SyncClient, LogLevel } from "sync-client";
import { ObsidianFileSystemOperations } from "./obsidian-file-system";
import { SyncSettingsTab } from "./views/settings/settings-tab";
export default class VaultLinkPlugin extends Plugin {
private settingsTab: SyncSettingsTab | undefined;
private client!: SyncClient;
private static registerConsoleForLogging(client: SyncClient): void {
client.logger.addOnMessageListener((logLine: LogLine) => {
const formatted = `${logLine.timestamp.toISOString()} ${logLine.level} ${logLine.message}`;
@ -38,7 +43,10 @@ export default class VaultLinkPlugin extends Plugin {
public async onload(): Promise<void> {
this.client = await SyncClient.create({
fs: new ObsidianFileSystemOperations(this.app.vault),
fs: new ObsidianFileSystemOperations(
this.app.vault,
this.app.workspace
),
persistence: {
load: this.loadData.bind(this),
save: this.saveData.bind(this)
@ -80,35 +88,9 @@ export default class VaultLinkPlugin extends Plugin {
async (_: MouseEvent) => this.activateView(LogsView.TYPE)
);
const eventHandler = new ObsidianFileEventHandler(this.client);
this.app.workspace.onLayoutReady(async () => {
this.client.logger.info("Initialising sync handlers");
[
this.app.vault.on(
"create",
eventHandler.onCreate.bind(eventHandler)
),
this.app.vault.on(
"modify",
eventHandler.onModify.bind(eventHandler)
),
this.app.vault.on(
"delete",
eventHandler.onDelete.bind(eventHandler)
),
this.app.vault.on(
"rename",
eventHandler.onRename.bind(eventHandler)
)
].forEach((event) => {
this.registerEvent(event);
});
this.registerEditorEvents();
void this.client.start();
this.client.logger.info("Sync handlers initialised");
});
}
@ -145,4 +127,51 @@ export default class VaultLinkPlugin extends Plugin {
await workspace.revealLeaf(leaf);
}
}
private registerEditorEvents(): void {
[
this.app.workspace.on(
"editor-change",
async (
_editor: Editor,
info: MarkdownView | MarkdownFileInfo
) => {
const { file } = info;
if (file) {
await this.client.syncLocallyUpdatedFile({
relativePath: file.path
});
}
}
),
this.app.vault.on("create", async (file: TAbstractFile) => {
if (file instanceof TFile) {
await this.client.syncLocallyCreatedFile(file.path);
}
}),
this.app.vault.on("modify", async (file: TAbstractFile) => {
if (file instanceof TFile) {
await this.client.syncLocallyUpdatedFile({
relativePath: file.path
});
}
}),
this.app.vault.on("delete", async (file: TAbstractFile) => {
await this.client.syncLocallyDeletedFile(file.path);
}),
this.app.vault.on(
"rename",
async (file: TAbstractFile, oldPath: string) => {
if (file instanceof TFile) {
await this.client.syncLocallyUpdatedFile({
oldPath,
relativePath: file.path
});
}
}
)
].forEach((event) => {
this.registerEvent(event);
});
}
}

53
frontend/obsidian-plugin/src/views/history/history-view.scss Normal file
View file

@ -0,0 +1,53 @@
.history-card {
padding: var(--size-4-4);
margin: var(--size-4-2);
background-color: var(--color-base-00);
border-radius: var(--radius-l);
container-type: inline-size;
&.clickable {
cursor: pointer;
}
&.success {
background-color: rgba(var(--color-green-rgb), 0.2);
}
&.error {
background-color: rgba(var(--color-red-rgb), 0.2);
}
.history-card-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: var(--size-4-2);
gap: var(--size-4-2);
@container (max-width: 300px) {
flex-direction: column;
align-items: flex-start;
}
.history-card-title {
font: var(--font-monospace);
display: flex;
align-items: center;
gap: var(--size-4-2);
word-break: break-all;
margin: 0;
}
.history-card-timestamp {
font-size: var(--font-ui-small);
font-style: italic;
color: var(--italic-color);
}
}
.history-card-message {
font-size: var(--font-ui-medium);
color: var(--color-base-70);
margin: 0;
}
}

3
frontend/obsidian-plugin/src/views/history-view.ts → frontend/obsidian-plugin/src/views/history/history-view.ts
View file

@ -1,6 +1,7 @@
import "./history-view.scss";
import type { IconName, WorkspaceLeaf } from "obsidian";
import { ItemView, setIcon } from "obsidian";
import { intlFormatDistance } from "date-fns";
import type { HistoryEntry, SyncClient } from "sync-client";
import { SyncType } from "sync-client";

60
frontend/obsidian-plugin/src/views/logs/logs-view.scss Normal file
View file

@ -0,0 +1,60 @@
.logs-view {
display: flex;
flex-direction: column;
.verbosity-selector {
display: flex;
align-items: center;
justify-content: space-between;
font-weight: normal;
gap: var(--size-4-2);
margin: var(--size-4-4) var(--size-4-2);
h4 {
margin: 0;
}
select {
cursor: pointer;
}
}
.logs-container {
max-width: 100%;
overflow-y: auto;
.log-message {
font: var(--font-monospace);
margin-bottom: var(--size-2-1);
overflow-wrap: break-word;
white-space: pre-wrap;
user-select: all;
.timestamp {
padding: var(--size-2-1) var(--size-4-1);
border-radius: var(--radius-s);
background-color: var(--color-base-30);
font-size: var(--font-ui-small);
font-family: var(--font-monospace);
font-weight: var(--bold-weight);
margin-right: var(--size-4-1);
}
&.DEBUG {
color: var(--color-base-50);
}
&.INFO {
color: var(--color-base-100);
}
&.WARNING {
color: rgb(var(--color-yellow-rgb));
}
&.ERROR {
color: rgb(var(--color-red-rgb));
}
}
}
}

55
frontend/obsidian-plugin/src/views/logs-view.ts → frontend/obsidian-plugin/src/views/logs/logs-view.ts
View file

@ -1,3 +1,5 @@
import "./logs-view.scss";
import type { WorkspaceLeaf } from "obsidian";
import { ItemView } from "obsidian";
import type { LogLine } from "sync-client";
@ -7,8 +9,11 @@ export class LogsView extends ItemView {
public static readonly TYPE = "logs-view";
public static readonly ICON = "logs";
private static readonly MAX_OFFSET_FROM_BOTTOM_WITH_AUTO_SCROLL_PX = 300;
private logsContainer: HTMLElement | undefined;
private readonly logLineToElement = new Map<LogLine, HTMLElement>();
private minLogLevel: LogLevel = LogLevel.INFO;
public constructor(
private readonly client: SyncClient,
@ -56,10 +61,43 @@ export class LogsView extends ItemView {
public async onOpen(): Promise<void> {
const container = this.containerEl.children[1];
container.addClass("logs-view");
container.createEl("h4", { text: "VaultLink logs" });
this.logsContainer = container.createDiv({ cls: "logs-container" });
this.updateView();
const logLevels = [
{ label: "Debug", value: LogLevel.DEBUG },
{ label: "Info", value: LogLevel.INFO },
{ label: "Warn", value: LogLevel.WARNING },
{ label: "Error", value: LogLevel.ERROR }
];
container.createDiv(
{
cls: "verbosity-selector"
},
(verbositySection) => {
verbositySection.createEl("h4", {
text: "VaultLink logs"
});
verbositySection.createEl("select", {}, (dropdown) => {
logLevels.forEach(({ label, value }) =>
dropdown.createEl("option", { text: label, value })
);
dropdown.value = this.minLogLevel;
dropdown.addEventListener("change", () => {
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
this.minLogLevel = dropdown.value as LogLevel;
this.logsContainer?.empty();
this.logLineToElement.clear();
this.updateView();
});
});
}
);
this.logsContainer = container.createDiv({ cls: "logs-container" });
}
private updateView(): void {
@ -68,13 +106,20 @@ export class LogsView extends ItemView {
return;
}
const logs = this.client.logger.getMessages(LogLevel.DEBUG);
const logs = this.client.logger.getMessages(this.minLogLevel);
if (this.logLineToElement.size === 0 && logs.length > 0) {
// Clear the "No logs available yet" message
container.empty();
}
const shouldScroll =
container.scrollTop == 0 ||
container.scrollHeight -
container.clientHeight -
container.scrollTop <
LogsView.MAX_OFFSET_FROM_BOTTOM_WITH_AUTO_SCROLL_PX;
logs.forEach((message) => {
if (this.logLineToElement.has(message)) {
return;
@ -98,6 +143,8 @@ export class LogsView extends ItemView {
container.createEl("p", {
text: "No logs available yet."
});
} else if (shouldScroll) {
container.scrollTop = container.scrollHeight;
}
}
}

57
frontend/obsidian-plugin/src/views/settings/settings-tab.scss Normal file
View file

@ -0,0 +1,57 @@
@mixin number-card {
padding: var(--size-2-1) var(--size-4-1);
border-radius: var(--radius-s);
background-color: var(--color-base-30);
font-size: var(--font-ui-small);
&.good {
background-color: rgba(var(--color-green-rgb), 0.35);
}
&.bad {
background-color: rgba(var(--color-red-rgb), 0.35);
}
}
.vault-link-settings {
h2 {
display: flex;
align-items: center;
font-size: var(--h2-size);
.version {
@include number-card;
margin: var(--size-2-2) 0 0 var(--size-4-2);
background-color: var(--color-base-30);
color: var(--color-base-70);
font-size: var(--font-ui-smaller);
}
}
.button-container {
display: flex;
gap: var(--size-4-2);
}
h3 {
font-size: var(--font-ui-large);
margin-top: var(--heading-spacing);
}
button,
input[type="range"],
.checkbox-container,
.slider::-webkit-slider-thumb {
cursor: pointer;
}
input[type="text"],
textarea {
width: 250px;
}
textarea {
resize: none;
height: 75px;
}
}

35
frontend/obsidian-plugin/src/views/settings-tab.ts → frontend/obsidian-plugin/src/views/settings/settings-tab.ts
View file

@ -1,10 +1,12 @@
import "./settings-tab.scss";
import type { App } from "obsidian";
import { Notice, PluginSettingTab, Setting } from "obsidian";
import type VaultLinkPlugin from "../vault-link-plugin";
import type { StatusDescription } from "./status-description";
import { LogsView } from "./logs-view";
import { HistoryView } from "./history-view";
import type VaultLinkPlugin from "src/vault-link-plugin";
import type { SyncClient, SyncSettings } from "sync-client";
import { HistoryView } from "../history/history-view";
import { LogsView } from "../logs/logs-view";
import type { StatusDescription } from "../status-description/status-description";
export class SyncSettingsTab extends PluginSettingTab {
private editedServerUri: string;
@ -220,7 +222,7 @@ export class SyncSettingsTab extends PluginSettingTab {
.addButton((button) =>
button.setButtonText("Test connection").onClick(async () => {
new Notice(
(await this.syncClient.checkConnection()).message
(await this.syncClient.checkConnection()).serverMessage
);
await this.statusDescription.updateConnectionState();
})
@ -246,29 +248,6 @@ export class SyncSettingsTab extends PluginSettingTab {
)
);
new Setting(containerEl)
.setName("Remote fetching frequency (seconds)")
.setDesc(
"Set how often should the plugin check for changes on the server. Lower values will increase the frequency of the checks making it easier to collaborate with others."
)
.setTooltip("todo, links to docs")
.addSlider((text) =>
text
.setLimits(0.5, 60, 0.5)
.setDynamicTooltip()
.setInstant(false)
.setValue(
this.syncClient.getSettings()
.fetchChangesUpdateIntervalMs / 1000
)
.onChange(async (value) =>
this.syncClient.setSetting(
"fetchChangesUpdateIntervalMs",
value * 1000
)
)
);
new Setting(containerEl)
.setName("Sync concurrency")
.setDesc(

14
frontend/obsidian-plugin/src/views/status-bar/status-bar.scss Normal file
View file

@ -0,0 +1,14 @@
.sync-status {
display: flex;
gap: var(--size-4-2);
* {
display: block;
}
.initialize-button {
padding: 0 var(--size-4-2);
background: rgba(var(--color-red-rgb), 0.4);
cursor: pointer;
}
}

4
frontend/obsidian-plugin/src/views/status-bar.ts → frontend/obsidian-plugin/src/views/status-bar/status-bar.ts
View file

@ -1,5 +1,7 @@
import "./status-bar.scss";
import type { HistoryStats, SyncClient } from "sync-client";
import type VaultLinkPlugin from "../vault-link-plugin";
import type VaultLinkPlugin from "../../vault-link-plugin";
export class StatusBar {
private readonly statusBarItem: HTMLElement;

32
frontend/obsidian-plugin/src/views/status-description/status-description.scss Normal file
View file

@ -0,0 +1,32 @@
@mixin number-card {
padding: var(--size-2-1) var(--size-4-1);
border-radius: var(--radius-s);
background-color: var(--color-base-30);
font-size: var(--font-ui-small);
&.good {
background-color: rgba(var(--color-green-rgb), 0.35);
}
&.bad {
background-color: rgba(var(--color-red-rgb), 0.35);
}
}
.status-description {
margin: var(--p-spacing) 0;
.number {
@include number-card;
font-family: var(--font-monospace);
font-weight: var(--bold-weight);
}
.error {
color: rgb(var(--color-red-rgb));
}
.warning {
color: rgb(var(--color-yellow-rgb));
}
}

26
frontend/obsidian-plugin/src/views/status-description.ts → frontend/obsidian-plugin/src/views/status-description/status-description.ts
View file

@ -1,13 +1,15 @@
import "./status-description.scss";
import type {
HistoryStats,
CheckConnectionResult,
NetworkConnectionStatus,
SyncClient
} from "sync-client";
export class StatusDescription {
private lastHistoryStats: HistoryStats | undefined;
private lastRemaining: number | undefined;
private lastConnectionState: CheckConnectionResult | undefined;
private lastConnectionState: NetworkConnectionStatus | undefined;
private statusChangeListeners: (() => void)[] = [];
@ -26,9 +28,13 @@ export class StatusDescription {
}
);
this.syncClient.addOnSettingsChangeListener(() => {
void this.updateConnectionState();
});
this.syncClient.addWebSocketStatusChangeListener(
() => void this.updateConnectionState()
);
this.syncClient.addOnSettingsChangeListener(
() => void this.updateConnectionState()
);
}
public async updateConnectionState(): Promise<void> {
@ -59,7 +65,15 @@ export class StatusDescription {
if (!this.lastConnectionState.isSuccessful) {
container.createSpan({
text: `VaultLink failed to connect to the remote server with the error "${this.lastConnectionState.message}"`,
text: `VaultLink failed to connect to the remote server with error '${this.lastConnectionState.serverMessage}'`,
cls: "error"
});
return;
}
if (!this.lastConnectionState.isWebSocketConnected) {
container.createSpan({
text: `${this.lastConnectionState.serverMessage} but the WebSocket connection could not be established.`,
cls: "error"
});
return;

11
frontend/obsidian-plugin/tsconfig.json
View file

@ -6,7 +6,12 @@
"strict": true,
"moduleResolution": "bundler",
"allowSyntheticDefaultImports": true,
"lib": ["DOM", "ESNext"]
"lib": [
"DOM",
"ESNext"
]
},
"exclude": ["./dist"]
}
"exclude": [
"./dist"
]
}

1669
frontend/package-lock.json generated
View file

File diff suppressed because it is too large Load diff

5
frontend/sync-client/package.json
View file

@ -23,6 +23,7 @@
"@types/jest": "^29.5.14",
"@types/node": "^22.13.10",
"jest": "^29.7.0",
"sync_lib": "file:../../backend/sync_lib/pkg",
"ts-jest": "^29.2.6",
"ts-loader": "^9.5.2",
"tslib": "2.8.1",
@ -30,6 +31,6 @@
"webpack": "^5.98.0",
"webpack-cli": "^6.0.1",
"webpack-merge": "^6.0.1",
"sync_lib": "file:../../backend/sync_lib/pkg"
"ws": "^8.18.1"
}
}
}

65
frontend/sync-client/src/file-operations/document-locks.ts
View file

@ -1,65 +0,0 @@
import type { Logger } from "../tracing/logger";
import type { RelativePath } from "../persistence/database";
// Manages locks on documents to prevent concurrent modifications
// allowing the client's FileOperations implementation to be simpler.
// Locks are granted in a first-in-first-out order.
export class DocumentLocks {
private readonly locked = new Set<RelativePath>();
private readonly waiters = new Map<RelativePath, (() => void)[]>();
public constructor(private readonly logger: Logger) {}
public tryLockDocument(relativePath: RelativePath): boolean {
if (this.locked.has(relativePath)) {
return false;
}
this.locked.add(relativePath);
return true;
}
public async waitForDocumentLock(
relativePath: RelativePath
): Promise<void> {
if (this.tryLockDocument(relativePath)) {
return Promise.resolve();
}
this.logger.debug(`Waiting for lock on ${relativePath}`);
return new Promise((resolve) => {
let waiting = this.waiters.get(relativePath);
if (!waiting) {
waiting = [];
this.waiters.set(relativePath, waiting);
}
waiting.push(resolve);
});
}
public unlockDocument(relativePath: RelativePath): void {
if (!this.locked.has(relativePath)) {
throw new Error(
`Document ${relativePath} is not locked, cannot unlock`
);
}
// Remove the first element to ensure FIFO unblocking order
const nextWaiting = this.waiters.get(relativePath)?.shift();
if (nextWaiting) {
this.logger.debug(`Granted lock on ${relativePath}`);
nextWaiting();
} else {
this.locked.delete(relativePath);
}
}
public reset(): void {
this.locked.clear();
this.waiters.clear();
}
}

10
frontend/sync-client/src/file-operations/safe-filesystem-operations.ts
View file

@ -1,7 +1,7 @@
import type { RelativePath } from "../persistence/database";
import type { FileSystemOperations } from "./filesystem-operations";
import type { Logger } from "../tracing/logger";
import { DocumentLocks } from "./document-locks";
import { Locks } from "../utils/locks";
import { FileNotFoundError } from "./file-not-found-error";
/**
@ -10,13 +10,13 @@ import { FileNotFoundError } from "./file-not-found-error";
* single request in-flight for any one file through the use of locks.
*/
export class SafeFileSystemOperations implements FileSystemOperations {
private readonly locks: DocumentLocks;
private readonly locks: Locks<RelativePath>;
public constructor(
private readonly fs: FileSystemOperations,
private readonly logger: Logger
) {
this.locks = new DocumentLocks(logger);
this.locks = new Locks(logger);
}
public async listAllFiles(): Promise<RelativePath[]> {
@ -117,7 +117,7 @@ export class SafeFileSystemOperations implements FileSystemOperations {
: [pathOrPaths];
await Promise.all(
paths.map(async (path) => this.locks.waitForDocumentLock(path))
paths.map(async (path) => this.locks.waitForLock(path))
);
try {
@ -125,7 +125,7 @@ export class SafeFileSystemOperations implements FileSystemOperations {
} finally {
await Promise.all(
paths.map((path) => {
this.locks.unlockDocument(path);
this.locks.unlock(path);
})
);
}

2
frontend/sync-client/src/index.ts
View file

@ -5,10 +5,10 @@ export {
type HistoryEntry
} from "./tracing/sync-history";
export { Logger, LogLevel, LogLine } from "./tracing/logger";
export type { CheckConnectionResult } from "./services/sync-service";
export { type SyncSettings } from "./persistence/settings";
export type { RelativePath, StoredDatabase } from "./persistence/database";
export type { FileSystemOperations } from "./file-operations/filesystem-operations";
export type { PersistenceProvider } from "./persistence/persistence";
export type { NetworkConnectionStatus } from "./sync-client";
export { SyncClient } from "./sync-client";

3
frontend/sync-client/src/persistence/settings.ts
View file

@ -1,11 +1,9 @@
import type { Logger } from "../tracing/logger";
import { LogLevel } from "../tracing/logger";
export interface SyncSettings {
remoteUri: string;
token: string;
vaultName: string;
fetchChangesUpdateIntervalMs: number;
syncConcurrency: number;
isSyncEnabled: boolean;
maxFileSizeMB: number;
@ -15,7 +13,6 @@ const DEFAULT_SETTINGS: SyncSettings = {
remoteUri: "",
token: "",
vaultName: "default",
fetchChangesUpdateIntervalMs: 1000,
syncConcurrency: 1,
isSyncEnabled: false,
maxFileSizeMB: 10

52
frontend/sync-client/src/services/sync-service.ts
View file

@ -18,6 +18,7 @@ export interface CheckConnectionResult {
}
export class SyncService {
private static readonly NETWORK_RETRY_INTERVAL_MS = 1000;
private client: Client<paths>;
private pingClient: Client<paths>;
private _fetchImplementation: typeof globalThis.fetch = globalThis.fetch;
@ -284,17 +285,35 @@ export class SyncService {
public async checkConnection(): Promise<CheckConnectionResult> {
try {
const result = await this.ping();
const response = await this.pingClient.GET("/ping", {
params: {
header: {
authorization: `Bearer ${this.settings.getSettings().token}`
}
}
});
this.logger.debug(
`Ping response: ${JSON.stringify(response.data)}`
);
if (!response.data) {
throw new Error(
`Failed to ping server: ${SyncService.formatError(response.error)}`
);
}
const result = response.data;
if (result.isAuthenticated) {
return {
isSuccessful: true,
message: `Successfully connected to server (version: ${result.serverVersion}) and authenticated.`
message: `Successfully connected to server (version: ${result.serverVersion}) and authenticated`
};
}
return {
isSuccessful: false,
message: `Successfully connected to server (version: ${result.serverVersion}) but failed to authenticate.`
message: `Successfully connected to server (version: ${result.serverVersion}) but failed to authenticate`
};
} catch (e) {
return {
@ -304,27 +323,6 @@ export class SyncService {
}
}
// No retries
private async ping(): Promise<components["schemas"]["PingResponse"]> {
const response = await this.pingClient.GET("/ping", {
params: {
header: {
authorization: `Bearer ${this.settings.getSettings().token}`
}
}
});
this.logger.debug(`Ping response: ${JSON.stringify(response.data)}`);
if (!response.data) {
throw new Error(
`Failed to ping server: ${SyncService.formatError(response.error)}`
);
}
return response.data;
}
/**
* Create a client and a ping client for the given remote URI.
*/
@ -355,8 +353,10 @@ export class SyncService {
throw e;
}
this.logger.error(`Failed network call (${e}), retrying`);
await sleep(1000);
this.logger.error(
`Failed network call (${e}), retryingin ${SyncService.NETWORK_RETRY_INTERVAL_MS}ms`
);
await sleep(SyncService.NETWORK_RETRY_INTERVAL_MS);
}
}
}

7
frontend/sync-client/src/services/types.ts
View file

@ -566,6 +566,10 @@ export interface components {
/** Format: int64 */
since_update_id?: number | null;
};
QueryParams2: {
/** Format: int64 */
since_update_id?: number | null;
};
SerializedError: {
causes: string[];
message: string;
@ -587,6 +591,9 @@ export interface components {
parentVersionId: number;
relativePath: string;
};
WebsocketPathParams: {
vault_id: string;
};
};
responses: never;
parameters: never;

57
frontend/sync-client/src/sync-client.ts
View file

@ -8,7 +8,6 @@ import type { RelativePath, StoredDatabase } from "./persistence/database";
import { Database } from "./persistence/database";
import type { SyncSettings } from "./persistence/settings";
import { Settings } from "./persistence/settings";
import type { CheckConnectionResult } from "./services/sync-service";
import { SyncService } from "./services/sync-service";
import { Syncer } from "./sync-operations/syncer";
import type { FileSystemOperations } from "./file-operations/filesystem-operations";
@ -16,9 +15,13 @@ import { FileOperations } from "./file-operations/file-operations";
import { ConnectionStatus } from "./services/connection-status";
import { UnrestrictedSyncer } from "./sync-operations/unrestricted-syncer";
export class SyncClient {
private remoteListenerIntervalId: NodeJS.Timeout | null = null;
export interface NetworkConnectionStatus {
isSuccessful: boolean;
serverMessage: string;
isWebSocketConnected: boolean;
}
export class SyncClient {
// eslint-disable-next-line @typescript-eslint/max-params
private constructor(
private readonly history: SyncHistory,
@ -31,15 +34,6 @@ export class SyncClient {
) {
this.settings.addOnSettingsChangeListener(
(newSettings, oldSettings) => {
if (
newSettings.fetchChangesUpdateIntervalMs !==
oldSettings.fetchChangesUpdateIntervalMs
) {
this.setRemoteEventListener(
newSettings.fetchChangesUpdateIntervalMs
);
}
if (newSettings.vaultName !== oldSettings.vaultName) {
void this.reset();
}
@ -145,8 +139,13 @@ export class SyncClient {
return client;
}
public async checkConnection(): Promise<CheckConnectionResult> {
return this.syncService.checkConnection();
public async checkConnection(): Promise<NetworkConnectionStatus> {
const server = await this.syncService.checkConnection();
return {
isSuccessful: server.isSuccessful,
serverMessage: server.message,
isWebSocketConnected: this.syncer.isWebSocketConnected
};
}
public getHistoryEntries(): readonly HistoryEntry[] {
@ -161,20 +160,15 @@ export class SyncClient {
public async start(): Promise<void> {
await this.syncer.scheduleSyncForOfflineChanges();
this.setRemoteEventListener(
this.settings.getSettings().fetchChangesUpdateIntervalMs
);
}
/// Clear all global state that has been touched by SyncClient.
public stop(): void {
this.unsetRemoteEventListener();
this.syncer.stop();
}
public async waitAndStop(): Promise<void> {
await this.syncer.waitUntilFinished();
this.stop();
await this.syncer.waitUntilFinished();
}
/// Wait for the in-flight operations to finish, reset all tracking,
@ -218,6 +212,10 @@ export class SyncClient {
this.syncer.addRemainingOperationsListener(listener);
}
public addWebSocketStatusChangeListener(listener: () => void): void {
this.syncer.addWebSocketStatusChangeListener(listener);
}
public async syncLocallyCreatedFile(
relativePath: RelativePath
): Promise<void> {
@ -242,21 +240,4 @@ export class SyncClient {
relativePath
});
}
private setRemoteEventListener(intervalMs: number): void {
if (this.remoteListenerIntervalId !== null) {
clearInterval(this.remoteListenerIntervalId);
}
this.remoteListenerIntervalId = setInterval(
() => void this.syncer.applyRemoteChangesLocally(),
intervalMs
);
}
private unsetRemoteEventListener(): void {
if (this.remoteListenerIntervalId !== null) {
clearInterval(this.remoteListenerIntervalId);
}
}
}

220
frontend/sync-client/src/sync-operations/syncer.ts
View file

@ -1,30 +1,40 @@
import type { Database, RelativePath } from "../persistence/database";
import type {
Database,
DocumentId,
RelativePath
} from "../persistence/database";
import type { SyncService } from "../services/sync-service";
import type { Logger } from "../tracing/logger";
import PQueue from "p-queue";
import { hash } from "../utils/hash";
import { v4 as uuidv4 } from "uuid";
import type { components } from "../services/types";
import type { Settings } from "../persistence/settings";
import type { Settings, SyncSettings } from "../persistence/settings";
import type { FileOperations } from "../file-operations/file-operations";
import { findMatchingFile } from "../utils/find-matching-file";
import type { UnrestrictedSyncer } from "./unrestricted-syncer";
import { createPromise } from "../utils/create-promise";
import { SyncResetError } from "../services/sync-reset-error";
import { Locks } from "../utils/locks";
export class Syncer {
private readonly remoteDocumentsLock: Locks<DocumentId>;
private readonly remainingOperationsListeners: ((
remainingOperations: number
) => void)[] = [];
private readonly webSocketStatusChangeListeners: (() => void)[] = [];
private readonly syncQueue: PQueue;
private runningScheduleSyncForOfflineChanges: Promise<void> | undefined;
private runningApplyRemoteChangesLocally: Promise<void> | undefined;
private refreshApplyRemoteChangesWebSocketInterval:
| NodeJS.Timeout
| undefined;
private applyRemoteChangesWebSocket: WebSocket | undefined;
public constructor(
private readonly logger: Logger,
private readonly database: Database,
settings: Settings,
private readonly settings: Settings,
private readonly syncService: SyncService,
private readonly operations: FileOperations,
private readonly internalSyncer: UnrestrictedSyncer
@ -33,11 +43,23 @@ export class Syncer {
concurrency: settings.getSettings().syncConcurrency
});
this.updateWebSocket(settings.getSettings());
this.remoteDocumentsLock = new Locks<DocumentId>(this.logger);
settings.addOnSettingsChangeListener((newSettings, oldSettings) => {
if (newSettings.syncConcurrency === oldSettings.syncConcurrency) {
return;
if (
newSettings.remoteUri !== oldSettings.remoteUri ||
newSettings.vaultName !== oldSettings.vaultName ||
newSettings.token !== oldSettings.token ||
newSettings.isSyncEnabled !== oldSettings.isSyncEnabled
) {
this.updateWebSocket(newSettings);
}
if (newSettings.syncConcurrency !== oldSettings.syncConcurrency) {
this.syncQueue.concurrency = newSettings.syncConcurrency;
}
this.syncQueue.concurrency = newSettings.syncConcurrency;
});
this.syncQueue.on("active", () => {
@ -45,6 +67,12 @@ export class Syncer {
listener(this.syncQueue.size);
});
});
this.setWebSocketRefreshInterval();
}
public get isWebSocketConnected(): boolean {
return this.applyRemoteChangesWebSocket?.readyState === WebSocket.OPEN;
}
public addRemainingOperationsListener(
@ -53,6 +81,10 @@ export class Syncer {
this.remainingOperationsListeners.push(listener);
}
public addWebSocketStatusChangeListener(listener: () => void): void {
this.webSocketStatusChangeListeners.push(listener);
}
public async syncLocallyCreatedFile(
relativePath: RelativePath
): Promise<void> {
@ -206,109 +238,139 @@ export class Syncer {
}
}
public async applyRemoteChangesLocally(): Promise<void> {
if (this.runningApplyRemoteChangesLocally !== undefined) {
this.logger.debug(
"Applying remote changes locally is already in progress"
);
return this.runningApplyRemoteChangesLocally;
}
try {
this.runningApplyRemoteChangesLocally =
this.internalApplyRemoteChangesLocally();
await this.runningApplyRemoteChangesLocally;
this.logger.info("All remote changes have been applied locally");
} catch (e) {
if (e instanceof SyncResetError) {
this.logger.info(
"Failed to apply remote changes locally due to a reset"
);
return;
}
this.logger.error(`Failed to apply remote changes locally: ${e}`);
throw e;
} finally {
this.runningApplyRemoteChangesLocally = undefined;
}
public async waitUntilFinished(): Promise<void> {
await this.runningScheduleSyncForOfflineChanges;
return this.syncQueue.onEmpty();
}
public async reset(): Promise<void> {
await this.waitUntilFinished();
this.internalSyncer.reset();
this.setWebSocketRefreshInterval();
this.updateWebSocket(this.settings.getSettings());
}
public async waitUntilFinished(): Promise<void> {
await Promise.allSettled([
this.runningScheduleSyncForOfflineChanges,
this.runningApplyRemoteChangesLocally
]);
return this.syncQueue.onEmpty();
public stop(): void {
clearInterval(this.refreshApplyRemoteChangesWebSocketInterval);
this.applyRemoteChangesWebSocket?.close();
}
private async internalApplyRemoteChangesLocally(): Promise<void> {
const remote = await this.syncQueue.add(async () =>
this.syncService.getAll(this.database.getLastSeenUpdateId())
);
private updateWebSocket(settings: SyncSettings): void {
this.applyRemoteChangesWebSocket?.close();
if (!remote) {
throw new Error("Failed to fetch remote changes");
}
if (remote.latestDocuments.length === 0) {
this.logger.debug("No remote changes to apply");
if (!settings.isSyncEnabled) {
this.applyRemoteChangesWebSocket = undefined;
return;
}
this.logger.info("Applying remote changes locally");
const wsUri = new URL(settings.remoteUri);
wsUri.protocol = wsUri.protocol === "https" ? "wss" : "ws";
wsUri.pathname = `/vaults/${settings.vaultName}/ws`;
await Promise.all(
remote.latestDocuments.map(this.syncRemotelyUpdatedFile.bind(this))
);
const lastSeenUpdateId = this.database.getLastSeenUpdateId();
if (
lastSeenUpdateId === undefined ||
lastSeenUpdateId < remote.lastUpdateId
typeof globalThis !== "undefined" &&
typeof globalThis.WebSocket === "undefined"
) {
this.database.setLastSeenUpdateId(remote.lastUpdateId);
// polyfill for WebSocket in Node.js
// eslint-disable-next-line
globalThis.WebSocket = require("ws");
}
this.applyRemoteChangesWebSocket = new WebSocket(wsUri);
this.applyRemoteChangesWebSocket.onmessage = (event): void =>
void this.syncRemotelyUpdatedFile(event.data).catch(
(e: unknown) => {
this.logger.error(
`Failed to sync remotely updated file: ${e}`
);
}
);
// The JS WebSocket API doesn't support setting headers, so we have to send the token as a message
this.applyRemoteChangesWebSocket.onopen = (): void => {
this.applyRemoteChangesWebSocket?.send(settings.token);
this.webSocketStatusChangeListeners.forEach((listener) => {
listener();
});
};
this.applyRemoteChangesWebSocket.onclose = (event): void => {
this.logger.warn(
`WebSocket closed with code ${event.code}: ${event.reason}`
);
this.webSocketStatusChangeListeners.forEach((listener) => {
listener();
});
};
}
private async syncRemotelyUpdatedFile(
remoteVersion: components["schemas"]["DocumentVersionWithoutContent"]
): Promise<void> {
private setWebSocketRefreshInterval(): void {
this.refreshApplyRemoteChangesWebSocketInterval = setInterval(() => {
if (
this.applyRemoteChangesWebSocket?.readyState === WebSocket.OPEN
) {
return;
}
this.updateWebSocket(this.settings.getSettings());
}, 5000);
}
private async syncRemotelyUpdatedFile(message: string): Promise<void> {
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
const remoteVersion = JSON.parse(
message
) as components["schemas"]["DocumentVersionWithoutContent"];
let document = this.database.getDocumentByDocumentId(
remoteVersion.documentId
);
const [promise, resolve, reject] = createPromise();
let hasLockToRelease = false;
if (document === undefined) {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
remoteVersion
)
// Let's avoid the same documents getting created in parallel multiple times
await this.remoteDocumentsLock.waitForLock(
remoteVersion.documentId
);
} else {
document = await this.database.getResolvedDocumentByRelativePath(
document.relativePath,
promise
hasLockToRelease = true;
document = this.database.getDocumentByDocumentId(
remoteVersion.documentId
);
}
try {
try {
if (document === undefined) {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
remoteVersion,
document
remoteVersion
)
);
} else {
const [promise, resolve, reject] = createPromise();
resolve();
} catch (e) {
reject(e);
} finally {
this.database.removeDocumentPromise(promise);
document =
await this.database.getResolvedDocumentByRelativePath(
document.relativePath,
promise
);
try {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
remoteVersion,
document
)
);
resolve();
} catch (e) {
reject(e);
} finally {
this.database.removeDocumentPromise(promise);
}
}
} finally {
if (hasLockToRelease) {
this.remoteDocumentsLock.unlock(remoteVersion.documentId);
}
}
}

12
frontend/sync-client/src/sync-operations/unrestricted-syncer.ts
View file

@ -13,14 +13,11 @@ import type { components } from "../services/types";
import { deserialize } from "../utils/deserialize";
import type { Settings } from "../persistence/settings";
import type { FileOperations } from "../file-operations/file-operations";
import { DocumentLocks } from "../file-operations/document-locks";
import { createPromise } from "../utils/create-promise";
import { FileNotFoundError } from "../file-operations/file-not-found-error";
import { SyncResetError } from "../services/sync-reset-error";
export class UnrestrictedSyncer {
private readonly locks: DocumentLocks;
public constructor(
private readonly logger: Logger,
private readonly database: Database,
@ -28,10 +25,7 @@ export class UnrestrictedSyncer {
private readonly syncService: SyncService,
private readonly operations: FileOperations,
private readonly history: SyncHistory
) {
this.locks = new DocumentLocks(logger);
}
) {}
public async unrestrictedSyncLocallyCreatedFile(
document: DocumentRecord
): Promise<void> {
@ -416,10 +410,6 @@ export class UnrestrictedSyncer {
}
}
public reset(): void {
this.locks.reset();
}
private tryIncrementVaultUpdateId(responseVaultUpdateId: number): void {
if (this.database.getLastSeenUpdateId() === responseVaultUpdateId - 1) {
this.database.setLastSeenUpdateId(responseVaultUpdateId);

2
frontend/sync-client/src/tracing/logger.ts
View file

@ -21,7 +21,7 @@ export class LogLine {
}
export class Logger {
private static readonly MAX_MESSAGES = 2000;
private static readonly MAX_MESSAGES = 100000;
private readonly messages: LogLine[] = [];
private readonly onMessageListeners: ((message: LogLine) => void)[] = [];

62
frontend/sync-client/src/file-operations/document-locks.test.ts → frontend/sync-client/src/utils/locks.test.ts
View file

@ -1,92 +1,88 @@
import { Logger } from "../tracing/logger";
import type { RelativePath } from "../persistence/database";
import { DocumentLocks } from "./document-locks";
import { Locks } from "./locks";
describe("Document lock", () => {
const testPath: RelativePath = "test/document/path";
const logger = new Logger();
let locks = new DocumentLocks(logger);
// eslint-disable-next-line @typescript-eslint/init-declarations
let locks: Locks<RelativePath>;
beforeEach(() => {
locks = new DocumentLocks(logger);
locks = new Locks<RelativePath>(logger);
});
test("should lock a document successfully", () => {
const result = locks.tryLockDocument(testPath);
const result = locks.tryLock(testPath);
expect(result).toBe(true);
});
test("should not lock a document that is already locked", () => {
locks.tryLockDocument(testPath);
const result = locks.tryLockDocument(testPath);
locks.tryLock(testPath);
const result = locks.tryLock(testPath);
expect(result).toBe(false);
});
test("should unlock a locked document", () => {
locks.tryLockDocument(testPath);
locks.unlockDocument(testPath);
const result = locks.tryLockDocument(testPath);
locks.tryLock(testPath);
locks.unlock(testPath);
const result = locks.tryLock(testPath);
expect(result).toBe(true);
locks.unlockDocument(testPath);
locks.unlock(testPath);
});
test("should throw an error when unlocking a document that is not locked", () => {
expect(() => {
locks.unlockDocument(testPath);
locks.unlock(testPath);
}).toThrow(`Document ${testPath} is not locked, cannot unlock`);
});
test("should wait for a document lock and resolve when unlocked", async () => {
locks.tryLockDocument(testPath);
locks.tryLock(testPath);
let resolved = false;
const waitPromise = locks.waitForDocumentLock(testPath).then(() => {
const waitPromise = locks.waitForLock(testPath).then(() => {
resolved = true;
});
locks.unlockDocument(testPath);
locks.unlock(testPath);
await waitPromise;
expect(resolved).toBe(true);
});
test("should resolve multiple waiters in FIFO order", async () => {
locks.tryLockDocument(testPath);
locks.tryLock(testPath);
let firstResolved = false;
let secondResolved = false;
let thirdResolved = false;
const firstWaitPromise = locks
.waitForDocumentLock(testPath)
.then(() => {
firstResolved = true;
});
const firstWaitPromise = locks.waitForLock(testPath).then(() => {
firstResolved = true;
});
const secondWaitPromise = locks
.waitForDocumentLock(testPath)
.then(() => {
secondResolved = true;
});
const secondWaitPromise = locks.waitForLock(testPath).then(() => {
secondResolved = true;
});
const thirdWaitPromise = locks
.waitForDocumentLock(testPath)
.then(() => {
thirdResolved = true;
});
const thirdWaitPromise = locks.waitForLock(testPath).then(() => {
thirdResolved = true;
});
locks.unlockDocument(testPath);
locks.unlock(testPath);
await firstWaitPromise;
expect(firstResolved).toBe(true);
expect(secondResolved).toBe(false);
expect(thirdResolved).toBe(false);
locks.unlockDocument(testPath);
locks.unlock(testPath);
await secondWaitPromise;
expect(secondResolved).toBe(true);
expect(thirdResolved).toBe(false);
locks.unlockDocument(testPath);
locks.unlock(testPath);
await thirdWaitPromise;
expect(thirdResolved).toBe(true);
});

60
frontend/sync-client/src/utils/locks.ts Normal file
View file

@ -0,0 +1,60 @@
import type { Logger } from "../tracing/logger";
// Manages locks on T to prevent concurrent modifications
// allowing the client's FileOperations implementation to be simpler.
// Locks are granted in a first-in-first-out order.
export class Locks<T> {
private readonly locked = new Set<T>();
private readonly waiters = new Map<T, (() => void)[]>();
public constructor(private readonly logger: Logger) {}
public tryLock(key: T): boolean {
if (this.locked.has(key)) {
return false;
}
this.locked.add(key);
return true;
}
public async waitForLock(key: T): Promise<void> {
if (this.tryLock(key)) {
return Promise.resolve();
}
this.logger.debug(`Waiting for lock on ${key}`);
return new Promise((resolve) => {
let waiting = this.waiters.get(key);
if (!waiting) {
waiting = [];
this.waiters.set(key, waiting);
}
waiting.push(resolve);
});
}
public unlock(key: T): void {
if (!this.locked.has(key)) {
throw new Error(`Document ${key} is not locked, cannot unlock`);
}
// Remove the first element to ensure FIFO unblocking order
const nextWaiting = this.waiters.get(key)?.shift();
if (nextWaiting) {
this.logger.debug(`Granted lock on ${key}`);
nextWaiting();
} else {
this.locked.delete(key);
}
}
public reset(): void {
this.locked.clear();
this.waiters.clear();
}
}

8
frontend/sync-client/tsconfig.json
View file

@ -6,10 +6,12 @@
"allowSyntheticDefaultImports": true,
"moduleResolution": "bundler",
"lib": [
"DOM" // to get "fetch"
"DOM" // to get `fetch` & `WebSocket`
],
"declaration": true,
"declarationDir": "./dist/types"
},
"exclude": ["./dist"]
}
"exclude": [
"./dist"
]
}

11
frontend/sync-client/webpack.config.js
View file

@ -20,7 +20,7 @@ const common = {
minimize: false
},
resolve: {
extensions: [".ts"],
extensions: [".ts", ".js"],
alias: {
root: __dirname,
src: path.resolve(__dirname, "src")
@ -42,6 +42,11 @@ module.exports = [
type: "umd"
},
globalObject: "this"
},
resolve: {
fallback: {
ws: false // Exclude `ws` from the browser bundle
}
}
}),
merge(common, {
@ -50,6 +55,10 @@ module.exports = [
path: path.resolve(__dirname, "dist"),
filename: "sync-client.node.js",
libraryTarget: "commonjs2"
},
externals: {
bufferutil: "bufferutil",
"utf-8-validate": "utf-8-validate" // required for ws: https://github.com/websockets/ws/issues/2245#issuecomment-2250318733
}
})
];

5
frontend/test-client/package.json
View file

@ -18,6 +18,7 @@
"typescript": "5.8.2",
"uuid": "^11.1.0",
"webpack": "^5.98.0",
"webpack-cli": "^6.0.1"
"webpack-cli": "^6.0.1",
"bufferutil": "^4.0.9"
}
}
}

13
frontend/test-client/src/agent/mock-agent.ts
View file

@ -88,8 +88,7 @@ export class MockAgent extends MockClient {
public async act(): Promise<void> {
const options: (() => Promise<unknown>)[] = [
this.createFileAction.bind(this),
this.changeFetchChangesUpdateIntervalMsAction.bind(this)
this.createFileAction.bind(this)
];
if (this.client.getSettings().isSyncEnabled) {
@ -253,16 +252,6 @@ export class MockAgent extends MockClient {
return this.create(file, new TextEncoder().encode(` ${content} `));
}
private async changeFetchChangesUpdateIntervalMsAction(): Promise<void> {
this.client.logger.info(
`Decided to change fetchChangesUpdateIntervalMs`
);
return this.client.setSetting(
"fetchChangesUpdateIntervalMs",
Math.random() * 2000 + 100
);
}
private async disableSyncAction(): Promise<void> {
this.client.logger.info(`Decided to disable sync`);
await this.client.setSetting("isSyncEnabled", false);

18
frontend/test-client/src/agent/mock-client.ts
View file

@ -1,4 +1,4 @@
import type { StoredDatabase } from "sync-client/dist/types/persistence/database";
import type { StoredDatabase } from "sync-client";
import { assert } from "../utils/assert";
import {
type RelativePath,
@ -23,9 +23,11 @@ export class MockClient implements FileSystemOperations {
};
public constructor(
private readonly initialSettings: Partial<SyncSettings>,
initialSettings: Partial<SyncSettings>,
protected readonly useSlowFileEvents: boolean
) {}
) {
this.data.settings = initialSettings;
}
public async init(
fetchImplementation: typeof globalThis.fetch
@ -39,16 +41,6 @@ export class MockClient implements FileSystemOperations {
fetch: fetchImplementation
});
await Promise.all(
Object.keys(this.initialSettings).map(async (key) => {
const settingKey = key as keyof SyncSettings; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
return this.client.setSetting(
settingKey,
this.initialSettings[settingKey]! // eslint-disable-line @typescript-eslint/no-non-null-assertion
);
})
);
await this.client.start();
}

11
frontend/test-client/tsconfig.json
View file

@ -5,8 +5,13 @@
"target": "ES2022",
"module": "CommonJS",
"esModuleInterop": true,
"lib": ["DOM", "ESNext"],
"lib": [
"DOM",
"ESNext"
],
"moduleResolution": "node"
},
"exclude": ["./dist"]
}
"exclude": [
"./dist"
]
}