Add WebSocket support (#12)

.github/workflows/check.yml

@@ -28,7 +28,7 @@ jobs:
           cargo install sqlx-cli wasm-pack
           cd backend
           sqlx database create --database-url sqlite://db.sqlite3
-          sqlx migrate run --source sync_server/src/database/migrations --database-url sqlite://db.sqlite3
+          sqlx migrate run --source sync_server/src/app_state/database/migrations --database-url sqlite://db.sqlite3
 
       - name: Build wasm
         run: |

.github/workflows/e2e.yml

@@ -28,7 +28,7 @@ jobs:
           cargo install sqlx-cli wasm-pack
           cd backend
           sqlx database create --database-url sqlite://db.sqlite3
-          sqlx migrate run --source sync_server/src/database/migrations --database-url sqlite://db.sqlite3
+          sqlx migrate run --source sync_server/src/app_state/database/migrations --database-url sqlite://db.sqlite3
 
       - name: Build wasm
         run: |
@@ -38,7 +38,7 @@ jobs:
       - name: E2E tests
         run: |
           cd backend
-          RUST_BACKTRACE=1 cargo run -p sync_server config-e2e.yml &
+          cargo run -p sync_server config-e2e.yml --color never &
           cd ..
 
           scripts/update-api-types.sh

.gitignore

@@ -14,4 +14,4 @@ backend/databases
 
 *.log
 
-plugin/coverage
+*.sqlx

backend/Cargo.lock

@@ -459,6 +459,16 @@ dependencies = [
  "clap_derive",
 ]
 
+[[package]]
+name = "clap-verbosity-flag"
+version = "3.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2678fade3b77aa3a8ff3aae87e9c008d3fb00473a41c71fbf74e91c8c7b37e84"
+dependencies = [
+ "clap",
+ "log",
+]
+
 [[package]]
 name = "clap_builder"
 version = "4.5.32"
@@ -2069,9 +2079,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.133"
+version = "1.0.140"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377"
+checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373"
 dependencies = [
  "itoa",
  "memchr",
@@ -2524,6 +2534,8 @@ dependencies = [
  "axum_typed_multipart",
  "chrono",
  "clap",
+ "clap-verbosity-flag",
+ "futures",
  "log",
  "rand",
  "reconcile",
@@ -2531,6 +2543,7 @@ dependencies = [
  "sanitize-filename",
  "schemars",
  "serde",
+ "serde_json",
  "serde_yaml",
  "sqlx",
  "sync_lib",

backend/Cargo.toml

@@ -22,6 +22,7 @@ thiserror = { version = "1.0.66", default-features = false  }
 codegen-units = 1
 lto = true
 opt-level = 3
+strip="debuginfo"  # Keep some info for better panics
 
 [workspace.lints.rust]
 unsafe_code = "forbid"

backend/Dockerfile

@@ -8,7 +8,7 @@ RUN cargo install sqlx-cli
 COPY . .
 
 RUN sqlx database create --database-url sqlite://db.sqlite3
-RUN sqlx migrate run --source sync_server/src/database/migrations --database-url sqlite://db.sqlite3
+RUN sqlx migrate run --source sync_server/src/app_state/database/migrations --database-url sqlite://db.sqlite3
 
 RUN cargo build --package sync_server --release --target x86_64-unknown-linux-musl
 

backend/config-e2e.yml

@@ -1,10 +1,13 @@
 database:
   databases_directory_path: databases
   max_connections: 12
+
 server:
   host: 0.0.0.0
   port: 3000
   max_body_size_mb: 512
+  max_clients_per_vault: 256
+
 users:
   user_tokens:
   - name: admin

backend/sync_server/Cargo.toml

@@ -34,6 +34,9 @@ sanitize-filename = "0.6.0"
 axum-jsonschema = { version = "0.8.0", features = ["aide"] }
 regex = "1.11.1"
 clap = { version = "4.5.32", features = ["derive"] }
+futures = "0.3.31"
+serde_json = "1.0.140"
+clap-verbosity-flag = "3.0.2"
 
 [lints]
 workspace = true

backend/sync_server/README.md

@@ -1,4 +1,4 @@
 
 cargo install sqlx-cli
 rm db.sqlite3; sqlx database create --database-url sqlite://db.sqlite3
-sqlx migrate run --source sync_server/src/database/migrations --database-url sqlite://db.sqlite3
+sqlx migrate run --source sync_server/src/app_state/database/migrations --database-url sqlite://db.sqlite3

backend/sync_server/src/app_state.rs

@@ -1,13 +1,19 @@
+pub mod broadcasts;
+pub mod database;
+
 use std::ffi::OsString;
 
 use anyhow::Result;
+use broadcasts::Broadcasts;
+use database::Database;
 
-use crate::{config::Config, consts::DEFAULT_CONFIG_PATH, database::Database};
+use crate::{config::Config, consts::DEFAULT_CONFIG_PATH};
 
 #[derive(Clone, Debug)]
 pub struct AppState {
     pub config: Config,
     pub database: Database,
+    pub broadcasts: Broadcasts,
 }
 
 impl AppState {
@@ -17,7 +23,12 @@ impl AppState {
 
         let config = Config::read_or_create(&path).await?;
         let database = Database::try_new(&config.database).await?;
+        let broadcasts = Broadcasts::new(&config.server);
 
-        Ok(Self { config, database })
+        Ok(Self {
+            config,
+            database,
+            broadcasts,
+        })
     }
 }

backend/sync_server/src/app_state/broadcasts.rs

@@ -0,0 +1,57 @@
+use std::{collections::HashMap, sync::Arc};
+
+use anyhow::Context;
+use tokio::sync::{Mutex, broadcast};
+
+use super::database::models::{DocumentVersionWithoutContent, VaultId};
+use crate::{config::server_config::ServerConfig, errors::server_error};
+
+#[derive(Debug, Clone)]
+pub struct Broadcasts {
+    max_clients_per_vault: usize,
+    tx: Arc<Mutex<HashMap<VaultId, broadcast::Sender<DocumentVersionWithoutContent>>>>,
+}
+
+impl Broadcasts {
+    pub fn new(server_config: &ServerConfig) -> Self {
+        Self {
+            max_clients_per_vault: server_config.max_clients_per_vault,
+            tx: Arc::new(Mutex::new(HashMap::new())),
+        }
+    }
+
+    pub async fn get_receiver(
+        &self,
+        vault: VaultId,
+    ) -> broadcast::Receiver<DocumentVersionWithoutContent> {
+        let tx = self.get_or_create(vault).await;
+
+        tx.subscribe()
+    }
+
+    /// Sent a document update to all clients subscribed to the vault.
+    /// We ignore & log failures.
+    pub async fn send(&self, vault: VaultId, document: DocumentVersionWithoutContent) {
+        let tx = self.get_or_create(vault).await;
+
+        let result = tx
+            .send(document)
+            .context("Cannot broadcast update message to websocket listeners")
+            .map_err(server_error);
+
+        if result.is_err() {
+            log::debug!("Failed to send message: {result:?}");
+        }
+    }
+
+    async fn get_or_create(
+        &self,
+        vault: VaultId,
+    ) -> broadcast::Sender<DocumentVersionWithoutContent> {
+        let mut tx = self.tx.lock().await;
+
+        tx.entry(vault)
+            .or_insert_with(|| broadcast::channel(self.max_clients_per_vault).0.clone())
+            .clone()
+    }
+}

backend/sync_server/src/app_state/database.rs

@@ -85,13 +85,13 @@ impl Database {
     }
 
     async fn run_migrations(pool: &Pool<Sqlite>) -> Result<()> {
-        sqlx::migrate!("src/database/migrations")
+        sqlx::migrate!("src/app_state/database/migrations")
             .run(pool)
             .await
             .context("Cannot check for pending migrations")
     }
 
-    async fn get_connection_pool(&mut self, vault: &VaultId) -> Result<Pool<Sqlite>> {
+    async fn get_connection_pool(&self, vault: &VaultId) -> Result<Pool<Sqlite>> {
         let mut pools = self.connection_pools.lock().await;
         if !pools.contains_key(vault) {
             let pool = Self::create_vault_database(&self.config, vault).await?;
@@ -108,7 +108,7 @@ impl Database {
     /// Attempting to write from this transaction might result in a
     /// database locked error. Use this transaction for read-only operations.
     pub async fn create_readonly_transaction(
-        &mut self,
+        &self,
         vault: &VaultId,
     ) -> Result<Transaction<'static>> {
         self.get_connection_pool(vault)
@@ -118,10 +118,7 @@ impl Database {
             .context("Cannot create transaction")
     }
 
-    pub async fn create_write_transaction(
+    pub async fn create_write_transaction(&self, vault: &VaultId) -> Result<Transaction<'static>> {
-        &mut self,
-        vault: &VaultId,
-    ) -> Result<Transaction<'static>> {
         let mut transaction = self.create_readonly_transaction(vault).await?;
 
         // sqlx doesn't support immediate transactions for sqlite: https://github.com/launchbadge/sqlx/issues/481
@@ -134,7 +131,7 @@ impl Database {
 
     /// Return the latest state of all documents in the vault
     pub async fn get_latest_documents(
-        &mut self,
+        &self,
         vault: &VaultId,
         transaction: Option<&mut Transaction<'_>>,
     ) -> Result<Vec<DocumentVersionWithoutContent>> {
@@ -165,7 +162,7 @@ impl Database {
     /// Return the latest state of all documents (including deleted) in the
     /// vault which have changed since the given update id
     pub async fn get_latest_documents_since(
-        &mut self,
+        &self,
         vault: &VaultId,
         vault_update_id: VaultUpdateId,
         transaction: Option<&mut Transaction<'_>>,
@@ -199,7 +196,7 @@ impl Database {
     }
 
     pub async fn get_max_update_id_in_vault(
-        &mut self,
+        &self,
         vault: &VaultId,
         transaction: Option<&mut Transaction<'_>>,
     ) -> Result<i64> {
@@ -222,7 +219,7 @@ impl Database {
     }
 
     pub async fn get_latest_document_by_path(
-        &mut self,
+        &self,
         vault: &VaultId,
         relative_path: &str,
         transaction: Option<&mut Transaction<'_>>,
@@ -258,7 +255,7 @@ impl Database {
     }
 
     pub async fn get_latest_document(
-        &mut self,
+        &self,
         vault: &VaultId,
         document_id: &DocumentId,
         transaction: Option<&mut Transaction<'_>>,
@@ -291,7 +288,7 @@ impl Database {
     }
 
     pub async fn get_document_version(
-        &mut self,
+        &self,
         vault: &VaultId,
         vault_update_id: VaultUpdateId,
         transaction: Option<&mut Transaction<'_>>,
@@ -322,7 +319,7 @@ impl Database {
     }
 
     pub async fn insert_document_version(
-        &mut self,
+        &self,
         vault: &VaultId,
         version: &StoredDocumentVersion,
         transaction: Option<&mut Transaction<'_>>,

backend/sync_server/src/cli.rs

@@ -1 +1,2 @@
 pub mod args;
+pub mod color_when;

backend/sync_server/src/cli/args.rs

@@ -1,38 +1,26 @@
 use std::ffi::OsString;
 
-use clap::{Parser, ValueEnum};
+use clap::Parser;
+use clap_verbosity_flag::{InfoLevel, Verbosity};
 
-/// Server for backing the VaultLink plugin
+use crate::cli::color_when::ColorWhen;
+
+/// Server for backing the `VaultLink` plugin
 #[derive(Parser, Debug)]
 #[command(version, about, long_about = None)]
 pub struct Args {
     #[arg(index = 1)]
     pub config_path: Option<OsString>,
 
+    #[command(flatten)]
+    pub verbose: Verbosity<InfoLevel>,
+
     #[arg(
             long,
-            require_equals = true,
             value_name = "WHEN",
-            num_args = 0..=1,
             default_value_t = ColorWhen::Auto,
             default_missing_value = "always",
             value_enum
         )]
     pub color: ColorWhen,
 }
-
-#[derive(ValueEnum, Copy, Clone, Debug, PartialEq, Eq)]
-pub enum ColorWhen {
-    Always,
-    Auto,
-    Never,
-}
-
-impl std::fmt::Display for ColorWhen {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        self.to_possible_value()
-            .expect("no values are skipped")
-            .get_name()
-            .fmt(f)
-    }
-}

backend/sync_server/src/cli/color_when.rs

@@ -0,0 +1,31 @@
+use std::io::IsTerminal;
+
+use clap::ValueEnum;
+
+#[derive(ValueEnum, Copy, Clone, Debug, PartialEq, Eq)]
+pub enum ColorWhen {
+    Always,
+    Auto,
+    Never,
+}
+
+impl ColorWhen {
+    pub fn use_colors(self) -> bool {
+        match self {
+            ColorWhen::Always => true,
+            ColorWhen::Auto => {
+                std::env::var_os("NO_COLOR").is_none() && std::io::stderr().is_terminal()
+            }
+            ColorWhen::Never => false,
+        }
+    }
+}
+
+impl std::fmt::Display for ColorWhen {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        self.to_possible_value()
+            .expect("no values are skipped")
+            .get_name()
+            .fmt(f)
+    }
+}

backend/sync_server/src/config/server_config.rs

@@ -1,7 +1,10 @@
 use log::debug;
 use serde::{Deserialize, Serialize};
 
-use crate::consts::{DEFAULT_HOST, DEFAULT_MAX_BODY_SIZE_MB, DEFAULT_PORT};
+use crate::consts::{
+    DEFAULT_HOST, DEFAULT_MAX_BODY_SIZE_MB, DEFAULT_MAX_CLIENTS_PER_VAULT, DEFAULT_PORT,
+};
+
 #[derive(Debug, Deserialize, Serialize, Clone)]
 pub struct ServerConfig {
     #[serde(default = "default_host")]
@@ -12,6 +15,9 @@ pub struct ServerConfig {
 
     #[serde(default = "default_max_body_size_mb")]
     pub max_body_size_mb: usize,
+
+    #[serde(default = "default_max_clients_per_vault")]
+    pub max_clients_per_vault: usize,
 }
 
 fn default_host() -> String {
@@ -29,12 +35,18 @@ fn default_max_body_size_mb() -> usize {
     DEFAULT_MAX_BODY_SIZE_MB
 }
 
+fn default_max_clients_per_vault() -> usize {
+    debug!("Using default max clients per vault: {DEFAULT_MAX_CLIENTS_PER_VAULT}");
+    DEFAULT_MAX_CLIENTS_PER_VAULT
+}
+
 impl Default for ServerConfig {
     fn default() -> Self {
         Self {
             host: default_host(),
             port: default_port(),
             max_body_size_mb: default_max_body_size_mb(),
+            max_clients_per_vault: default_max_clients_per_vault(),
         }
     }
 }

backend/sync_server/src/consts.rs

@@ -4,3 +4,4 @@ pub const DEFAULT_HOST: &str = "127.0.0.1";
 pub const DEFAULT_PORT: u16 = 3000;
 pub const DEFAULT_MAX_CONNECTIONS: u32 = 12;
 pub const DEFAULT_MAX_BODY_SIZE_MB: usize = 4096;
+pub const DEFAULT_MAX_CLIENTS_PER_VAULT: usize = 256;

backend/sync_server/src/main.rs

@@ -1,38 +1,79 @@
+mod app_state;
 mod cli;
 mod config;
 mod consts;
-mod database;
 mod errors;
 mod server;
 mod utils;
 
+use std::process::ExitCode;
+
 use anyhow::{Context as _, Result};
 use clap::Parser;
 use cli::args::Args;
 use errors::{SyncServerError, init_error};
 use log::info;
 use server::create_server;
-use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
+use tracing_subscriber::{EnvFilter, fmt::format, util::SubscriberInitExt};
 
 #[tokio::main]
-async fn main() -> Result<(), SyncServerError> {
+async fn main() -> ExitCode {
     let args = Args::parse();
 
-    tracing_subscriber::registry()
+    let mut result = set_up_logging(&args);
-        .with(
+
-            tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| {
+    if result.is_ok() {
-                format!(
+        result = start_server(args).await;
-                    "{}=debug,tower_http=debug,axum::rejection=trace",
+    }
-                    env!("CARGO_CRATE_NAME")
+
-                )
+    match result {
-                .into()
+        Ok(()) => ExitCode::SUCCESS,
-            }),
+        Err(e) => {
+            eprintln!("Failed to set up logging: {e}");
+            ExitCode::FAILURE
+        }
+    }
+}
+
+fn set_up_logging(args: &Args) -> Result<(), SyncServerError> {
+    let level_filter = match args.verbose.log_level_filter() {
+        // We don't want to allow disabling all logging
+        log::LevelFilter::Off | log::LevelFilter::Error => tracing::Level::ERROR,
+        log::LevelFilter::Warn => tracing::Level::WARN,
+        log::LevelFilter::Info => tracing::Level::INFO,
+        log::LevelFilter::Debug => tracing::Level::DEBUG,
+        log::LevelFilter::Trace => tracing::Level::TRACE,
+    };
+
+    let env_filter = EnvFilter::builder()
+        .with_default_directive(level_filter.into())
+        .from_env()
+        .context("Failed to create logging env filter")
+        .map_err(init_error)?;
+
+    let use_colors = args.color.use_colors();
+
+    let is_debug_mode = args.verbose.log_level_filter() >= log::LevelFilter::Debug;
+
+    tracing_subscriber::fmt()
+        .with_ansi(use_colors)
+        .with_env_filter(env_filter)
+        .event_format(
+            format()
+                .without_time()
+                .with_target(is_debug_mode)
+                .with_line_number(is_debug_mode)
+                .compact(),
         )
-        .with(tracing_subscriber::fmt::layer())
+        .finish()
         .try_init()
         .context("Failed to initialise tracing")
         .map_err(init_error)?;
 
+    Ok(())
+}
+
+async fn start_server(args: Args) -> Result<(), SyncServerError> {
     info!(
         "Starting VaultLink server version {}",
         env!("CARGO_PKG_VERSION")

backend/sync_server/src/server.rs

@@ -1,3 +1,16 @@
+mod auth;
+mod create_document;
+mod delete_document;
+mod fetch_document_version;
+mod fetch_document_version_content;
+mod fetch_latest_document_version;
+mod fetch_latest_documents;
+mod ping;
+mod requests;
+mod responses;
+mod update_document;
+mod websocket;
+
 use std::{ffi::OsString, sync::Arc};
 
 use aide::{
@@ -10,7 +23,6 @@ use aide::{
     transform::TransformOpenApi,
 };
 use anyhow::{Context as _, Result, anyhow};
-use app_state::AppState;
 use axum::{
     Extension, Json,
     extract::{DefaultBodyLimit, Request},
@@ -32,21 +44,10 @@ use tower_http::{
 use tracing::{Level, info_span};
 
 use crate::{
+    app_state::AppState,
     config::server_config::ServerConfig,
     errors::{SerializedError, not_found_error},
 };
-mod app_state;
-mod auth;
-mod create_document;
-mod delete_document;
-mod fetch_document_version;
-mod fetch_document_version_content;
-mod fetch_latest_document_version;
-mod fetch_latest_documents;
-mod ping;
-mod requests;
-mod responses;
-mod update_document;
 
 pub async fn create_server(config_path: Option<OsString>) -> Result<()> {
     aide::r#gen::on_error(|err| error!("{err}"));
@@ -65,6 +66,7 @@ pub async fn create_server(config_path: Option<OsString>) -> Result<()> {
             "/vaults/:vault_id/documents",
             get(fetch_latest_documents::fetch_latest_documents),
         )
+        .route("/vaults/:vault_id/ws", get(websocket::websocket_handler))
         .api_route(
             "/vaults/:vault_id/documents",
             post(create_document::create_document_multipart),

backend/sync_server/src/server/auth.rs

@@ -1,9 +1,10 @@
-use super::app_state::AppState;
 use crate::{
+    app_state::AppState,
     config::user_config::User,
     errors::{SyncServerError, unauthorized_error},
 };
 
+// TODO: turn this into a middleware
 pub fn auth(app_state: &AppState, token: &str) -> Result<User, SyncServerError> {
     app_state
         .config

backend/sync_server/src/server/create_document.rs

@@ -11,12 +11,16 @@ use serde::Deserialize;
 use sync_lib::base64_to_bytes;
 
 use super::{
-    app_state::AppState,
     auth::auth,
     requests::{CreateDocumentVersion, CreateDocumentVersionMultipart},
 };
 use crate::{
-    database::models::{DocumentId, DocumentVersionWithoutContent, StoredDocumentVersion, VaultId},
+    app_state::{
+        AppState,
+        database::models::{
+            DocumentId, DocumentVersionWithoutContent, StoredDocumentVersion, VaultId,
+        },
+    },
     errors::{SyncServerError, client_error, server_error},
     utils::sanitize_path,
 };
@@ -77,7 +81,7 @@ pub async fn create_document_json(
 
 async fn internal_create_document(
     auth_header: Authorization<Bearer>,
-    mut state: AppState,
+    state: AppState,
     vault_id: VaultId,
     document_id: Option<DocumentId>,
     relative_path: String,
@@ -139,5 +143,10 @@ async fn internal_create_document(
         .context("Failed to commit successful transaction")
         .map_err(server_error)?;
 
+    state
+        .broadcasts
+        .send(vault_id, new_version.clone().into())
+        .await;
+
     Ok(Json(new_version.into()))
 }

backend/sync_server/src/server/delete_document.rs

@@ -8,9 +8,14 @@ use axum_jsonschema::Json;
 use schemars::JsonSchema;
 use serde::Deserialize;
 
-use super::{app_state::AppState, auth::auth, requests::DeleteDocumentVersion};
+use super::{auth::auth, requests::DeleteDocumentVersion};
 use crate::{
-    database::models::{DocumentId, DocumentVersionWithoutContent, StoredDocumentVersion, VaultId},
+    app_state::{
+        AppState,
+        database::models::{
+            DocumentId, DocumentVersionWithoutContent, StoredDocumentVersion, VaultId,
+        },
+    },
     errors::{SyncServerError, server_error},
     utils::sanitize_path,
 };
@@ -29,7 +34,7 @@ pub async fn delete_document(
         vault_id,
         document_id,
     }): Path<DeleteDocumentPathParams>,
-    State(mut state): State<AppState>,
+    State(state): State<AppState>,
     Json(request): Json<DeleteDocumentVersion>,
 ) -> Result<Json<DocumentVersionWithoutContent>, SyncServerError> {
     auth(&state, auth_header.token())?;
@@ -67,5 +72,10 @@ pub async fn delete_document(
         .context("Failed to commit successful transaction")
         .map_err(server_error)?;
 
+    state
+        .broadcasts
+        .send(vault_id, new_version.clone().into())
+        .await;
+
     Ok(Json(new_version.into()))
 }

backend/sync_server/src/server/fetch_document_version.rs

@@ -8,9 +8,12 @@ use axum_jsonschema::Json;
 use schemars::JsonSchema;
 use serde::Deserialize;
 
-use super::{app_state::AppState, auth::auth};
+use super::auth::auth;
 use crate::{
-    database::models::{DocumentId, DocumentVersion, VaultId, VaultUpdateId},
+    app_state::{
+        AppState,
+        database::models::{DocumentId, DocumentVersion, VaultId, VaultUpdateId},
+    },
     errors::{SyncServerError, not_found_error, server_error},
 };
 
@@ -30,7 +33,7 @@ pub async fn fetch_document_version(
         document_id,
         vault_update_id,
     }): Path<FetchDocumentVersionPathParams>,
-    State(mut state): State<AppState>,
+    State(state): State<AppState>,
 ) -> Result<Json<DocumentVersion>, SyncServerError> {
     auth(&state, auth_header.token())?;
 

backend/sync_server/src/server/fetch_document_version_content.rs

@@ -10,9 +10,12 @@ use axum_extra::{
 use schemars::JsonSchema;
 use serde::Deserialize;
 
-use super::{app_state::AppState, auth::auth};
+use super::auth::auth;
 use crate::{
-    database::models::{DocumentId, VaultId, VaultUpdateId},
+    app_state::{
+        AppState,
+        database::models::{DocumentId, VaultId, VaultUpdateId},
+    },
     errors::{SyncServerError, not_found_error, server_error},
 };
 
@@ -32,7 +35,7 @@ pub async fn fetch_document_version_content(
         document_id,
         vault_update_id,
     }): Path<FetchDocumentVersionContentPathParams>,
-    State(mut state): State<AppState>,
+    State(state): State<AppState>,
 ) -> Result<Bytes, SyncServerError> {
     auth(&state, auth_header.token())?;
 

backend/sync_server/src/server/fetch_latest_document_version.rs

@@ -8,9 +8,12 @@ use axum_jsonschema::Json;
 use schemars::JsonSchema;
 use serde::Deserialize;
 
-use super::{app_state::AppState, auth::auth};
+use super::auth::auth;
 use crate::{
-    database::models::{DocumentId, DocumentVersion, VaultId},
+    app_state::{
+        AppState,
+        database::models::{DocumentId, DocumentVersion, VaultId},
+    },
     errors::{SyncServerError, not_found_error, server_error},
 };
 
@@ -28,7 +31,7 @@ pub async fn fetch_latest_document_version(
         vault_id,
         document_id,
     }): Path<FetchLatestDocumentVersionPathParams>,
-    State(mut state): State<AppState>,
+    State(state): State<AppState>,
 ) -> Result<Json<DocumentVersion>, SyncServerError> {
     auth(&state, auth_header.token())?;
 

backend/sync_server/src/server/fetch_latest_documents.rs

@@ -7,9 +7,12 @@ use axum_jsonschema::Json;
 use schemars::JsonSchema;
 use serde::Deserialize;
 
-use super::{app_state::AppState, auth::auth, responses::FetchLatestDocumentsResponse};
+use super::{auth::auth, responses::FetchLatestDocumentsResponse};
 use crate::{
-    database::models::{VaultId, VaultUpdateId},
+    app_state::{
+        AppState,
+        database::models::{VaultId, VaultUpdateId},
+    },
     errors::{SyncServerError, server_error},
 };
 
@@ -30,7 +33,7 @@ pub async fn fetch_latest_documents(
     TypedHeader(auth_header): TypedHeader<Authorization<Bearer>>,
     Path(FetchLatestDocumentsPathParams { vault_id }): Path<FetchLatestDocumentsPathParams>,
     Query(QueryParams { since_update_id }): Query<QueryParams>,
-    State(mut state): State<AppState>,
+    State(state): State<AppState>,
 ) -> Result<Json<FetchLatestDocumentsResponse>, SyncServerError> {
     auth(&state, auth_header.token())?;
 

backend/sync_server/src/server/ping.rs

@@ -4,8 +4,8 @@ use axum_extra::{
     headers::{Authorization, authorization::Bearer},
 };
 
-use super::{app_state::AppState, auth::auth, responses::PingResponse};
+use super::{auth::auth, responses::PingResponse};
-use crate::errors::SyncServerError;
+use crate::{app_state::AppState, errors::SyncServerError};
 
 #[axum::debug_handler]
 pub async fn ping(

backend/sync_server/src/server/requests.rs

@@ -4,7 +4,7 @@ use axum_typed_multipart::TryFromMultipart;
 use schemars::JsonSchema;
 use serde::{self, Deserialize};
 
-use crate::database::models::{DocumentId, VaultUpdateId};
+use crate::app_state::database::models::{DocumentId, VaultUpdateId};
 
 #[derive(Debug, Deserialize, JsonSchema)]
 #[serde(rename_all = "camelCase")]

backend/sync_server/src/server/responses.rs

@@ -1,7 +1,9 @@
 use schemars::JsonSchema;
 use serde::{self, Serialize};
 
-use crate::database::models::{DocumentVersion, DocumentVersionWithoutContent, VaultUpdateId};
+use crate::app_state::database::models::{
+    DocumentVersion, DocumentVersionWithoutContent, VaultUpdateId,
+};
 
 /// Response to a ping request.
 #[derive(Debug, Clone, Serialize, JsonSchema)]

backend/sync_server/src/server/update_document.rs

@@ -12,13 +12,15 @@ use serde::Deserialize;
 use sync_lib::{base64_to_bytes, is_file_type_mergable, merge};
 
 use super::{
-    app_state::AppState,
     auth::auth,
     requests::{UpdateDocumentVersion, UpdateDocumentVersionMultipart},
     responses::DocumentUpdateResponse,
 };
 use crate::{
-    database::models::{DocumentId, StoredDocumentVersion, VaultId, VaultUpdateId},
+    app_state::{
+        AppState,
+        database::models::{DocumentId, StoredDocumentVersion, VaultId, VaultUpdateId},
+    },
     errors::{SyncServerError, client_error, not_found_error, server_error},
     utils::{deduped_file_paths, sanitize_path},
 };
@@ -83,7 +85,7 @@ pub async fn update_document_json(
 #[allow(clippy::too_many_arguments, clippy::too_many_lines)]
 async fn internal_update_document(
     auth_header: Authorization<Bearer>,
-    mut state: AppState,
+    state: AppState,
     vault_id: VaultId,
     document_id: DocumentId,
     parent_version_id: VaultUpdateId,
@@ -216,6 +218,11 @@ async fn internal_update_document(
         .context("Failed to commit successful transaction")
         .map_err(server_error)?;
 
+    state
+        .broadcasts
+        .send(vault_id, new_version.clone().into())
+        .await;
+
     Ok(Json(if is_different_from_request_content {
         DocumentUpdateResponse::MergingUpdate(new_version.into())
     } else {

backend/sync_server/src/server/websocket.rs

@@ -0,0 +1,147 @@
+use anyhow::Context;
+use axum::{
+    extract::{
+        Path, Query, State,
+        ws::{Message, WebSocket, WebSocketUpgrade},
+    },
+    response::Response,
+};
+use futures::{
+    sink::SinkExt,
+    stream::{SplitSink, StreamExt},
+};
+use log::{error, info, warn};
+use schemars::JsonSchema;
+use serde::Deserialize;
+
+use super::auth::auth;
+use crate::{
+    app_state::{
+        AppState,
+        database::models::{DocumentVersionWithoutContent, VaultId, VaultUpdateId},
+    },
+    errors::{SyncServerError, server_error, unauthorized_error},
+};
+
+// This is required for aide to infer the path parameter types and names
+#[derive(Deserialize, JsonSchema)]
+pub struct WebsocketPathParams {
+    vault_id: VaultId,
+}
+
+// This is required for aide to infer the path parameter types and names
+#[derive(Deserialize, JsonSchema)]
+pub struct QueryParams {
+    since_update_id: Option<VaultUpdateId>,
+}
+
+pub async fn websocket_handler(
+    ws: WebSocketUpgrade,
+    Path(WebsocketPathParams { vault_id }): Path<WebsocketPathParams>,
+    Query(QueryParams { since_update_id }): Query<QueryParams>,
+    State(state): State<AppState>,
+) -> Result<Response, SyncServerError> {
+    Ok(ws.on_upgrade(move |socket| websocket_wrapped(state, socket, vault_id, since_update_id)))
+}
+
+async fn websocket_wrapped(
+    state: AppState,
+    stream: WebSocket,
+    vault_id: VaultId,
+    since_update_id: Option<VaultUpdateId>,
+) {
+    info!("Websocket connection opened on vault '{}'", vault_id);
+
+    let result = websocket(state, stream, vault_id.clone(), since_update_id).await;
+
+    if let Err(err) = result {
+        error!(
+            "Websocket connection error on vault '{}': {}",
+            vault_id, err
+        );
+    }
+
+    warn!("Websocket connection closed on vault '{}'", vault_id);
+}
+
+async fn websocket(
+    state: AppState,
+    stream: WebSocket,
+    vault_id: VaultId,
+    since_update_id: Option<VaultUpdateId>,
+) -> Result<(), SyncServerError> {
+    let (mut sender, mut receiver) = stream.split();
+
+    if let Some(Ok(Message::Text(token))) = receiver.next().await {
+        auth(&state, &token)?;
+    } else {
+        return Err(unauthorized_error(anyhow::anyhow!(
+            "Failed to authenticate"
+        )));
+    }
+
+    let mut rx = state.broadcasts.get_receiver(vault_id.clone()).await;
+
+    let documents = if let Some(since_update_id) = since_update_id {
+        state
+            .database
+            .get_latest_documents_since(&vault_id, since_update_id, None)
+            .await
+            .map_err(server_error)
+    } else {
+        state
+            .database
+            .get_latest_documents(&vault_id, None)
+            .await
+            .map_err(server_error)
+    }?;
+
+    for document in documents {
+        send_document_over_websocket(document, &mut sender).await?;
+    }
+
+    let mut send_task = tokio::spawn(async move {
+        while let Ok(update) = rx.recv().await {
+            send_document_over_websocket(update, &mut sender).await?;
+        }
+
+        Ok::<(), SyncServerError>(())
+    });
+
+    let mut recv_task =
+        tokio::spawn(
+            async move { while let Some(Ok(Message::Text(_text))) = receiver.next().await {} },
+        );
+
+    tokio::select! {
+        _ = &mut send_task => recv_task.abort(),
+        _ = &mut recv_task => send_task.abort(),
+    };
+
+    send_task
+        .await
+        .context("Websocket send task failed")
+        .map_err(server_error)??;
+
+    recv_task
+        .await
+        .context("Websocket receive task failed")
+        .map_err(server_error)?;
+
+    Ok(())
+}
+
+async fn send_document_over_websocket(
+    document: DocumentVersionWithoutContent,
+    sender: &mut SplitSink<WebSocket, Message>,
+) -> Result<(), SyncServerError> {
+    let serialized_update = serde_json::to_string(&document)
+        .context("Failed to serialize update")
+        .map_err(server_error)?;
+
+    sender
+        .send(Message::Text(serialized_update))
+        .await
+        .context("Failed to send message over websocket")
+        .map_err(server_error)
+}

frontend/obsidian-plugin/src/obisidan-event-handler.ts

@@ -1,58 +0,0 @@
-import type { SyncClient } from "sync-client";
-import type { TAbstractFile } from "obsidian";
-import { TFile } from "obsidian";
-
-export class ObsidianFileEventHandler {
-	public constructor(private readonly client: SyncClient) {}
-
-	public async onCreate(file: TAbstractFile): Promise<void> {
-		if (file instanceof TFile) {
-			this.client.logger.info(`File created: ${file.path}`);
-
-			await this.client.syncLocallyCreatedFile(file.path);
-		} else {
-			this.client.logger.debug(`Folder created: ${file.path}, ignored`);
-		}
-	}
-
-	public async onDelete(file: TAbstractFile): Promise<void> {
-		if (file instanceof TFile) {
-			this.client.logger.info(`File deleted: ${file.path}`);
-
-			await this.client.syncLocallyDeletedFile(file.path);
-		} else {
-			this.client.logger.debug(`Folder deleted: ${file.path}, ignored`);
-		}
-	}
-
-	public async onRename(file: TAbstractFile, oldPath: string): Promise<void> {
-		if (file instanceof TFile) {
-			this.client.logger.info(`File renamed: ${oldPath} -> ${file.path}`);
-
-			await this.client.syncLocallyUpdatedFile({
-				oldPath,
-				relativePath: file.path
-			});
-		} else {
-			this.client.logger.debug(
-				`Folder renamed: ${oldPath} -> ${file.path}, ignored`
-			);
-		}
-	}
-
-	public async onModify(file: TAbstractFile): Promise<void> {
-		if (file instanceof TFile) {
-			if (file.basename.startsWith("console-log.iPhone")) {
-				return;
-			}
-
-			this.client.logger.info(`File modified: ${file.path}`);
-
-			await this.client.syncLocallyUpdatedFile({
-				relativePath: file.path
-			});
-		} else {
-			this.client.logger.debug(`Folder modified: ${file.path}, ignored`);
-		}
-	}
-}

frontend/obsidian-plugin/src/obsidian-file-system.ts

@@ -1,23 +1,38 @@
-import type { Stat, Vault } from "obsidian";
+import type { Stat, Vault, Workspace } from "obsidian";
-import { normalizePath } from "obsidian";
+import { MarkdownView, normalizePath } from "obsidian";
 import type { FileSystemOperations, RelativePath } from "sync-client";
 
 export class ObsidianFileSystemOperations implements FileSystemOperations {
-	public constructor(private readonly vault: Vault) {}
+	public constructor(
+		private readonly vault: Vault,
+		private readonly workspace: Workspace
+	) {}
 
 	public async listAllFiles(): Promise<RelativePath[]> {
 		return this.vault.getFiles().map((file) => file.path);
 	}
 
 	public async read(path: RelativePath): Promise<Uint8Array> {
-		return new Uint8Array(
+		path = normalizePath(path);
-			await this.vault.adapter.readBinary(normalizePath(path))
+		const view = this.workspace.getActiveViewOfType(MarkdownView);
-		);
+		if (view?.file?.path === path) {
+			return new TextEncoder().encode(view.editor.getValue());
+		}
+
+		return new Uint8Array(await this.vault.adapter.readBinary(path));
 	}
 
 	public async write(path: RelativePath, content: Uint8Array): Promise<void> {
+		path = normalizePath(path);
+
+		const view = this.workspace.getActiveViewOfType(MarkdownView);
+		if (view?.file?.path === path) {
+			view.editor.setValue(new TextDecoder().decode(content));
+			return;
+		}
+
 		return this.vault.adapter.writeBinary(
-			normalizePath(path),
+			path,
 			// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
 			content.buffer as ArrayBuffer
 		);
@@ -27,7 +42,16 @@ export class ObsidianFileSystemOperations implements FileSystemOperations {
 		path: RelativePath,
 		updater: (currentContent: string) => string
 	): Promise<string> {
-		return this.vault.adapter.process(normalizePath(path), updater);
+		path = normalizePath(path);
+
+		const view = this.workspace.getActiveViewOfType(MarkdownView);
+		if (view?.file?.path === path) {
+			const result = updater(view.editor.getValue());
+			view.editor.setValue(result);
+			return result;
+		}
+
+		return this.vault.adapter.process(path, updater);
 	}
 
 	public async getFileSize(path: RelativePath): Promise<number> {

frontend/obsidian-plugin/src/styles.scss

@@ -1,185 +0,0 @@
-@mixin number-card {
-	padding: var(--size-2-1) var(--size-4-1);
-	border-radius: var(--radius-s);
-	background-color: var(--color-base-30);
-	font-size: var(--font-ui-small);
-
-	&.good {
-		background-color: rgba(var(--color-green-rgb), 0.35);
-	}
-
-	&.bad {
-		background-color: rgba(var(--color-red-rgb), 0.35);
-	}
-}
-
-.status-description {
-	margin: var(--p-spacing) 0;
-
-	.number {
-		@include number-card;
-		font-family: var(--font-monospace);
-		font-weight: var(--bold-weight);
-	}
-
-	.error {
-		color: rgb(var(--color-red-rgb));
-	}
-
-	.warning {
-		color: rgb(var(--color-yellow-rgb));
-	}
-}
-
-.vault-link-settings {
-	h2 {
-		display: flex;
-		align-items: center;
-		font-size: var(--h2-size);
-
-		.version {
-			@include number-card;
-			margin: var(--size-2-2) 0 0 var(--size-4-2);
-			background-color: var(--color-base-30);
-			color: var(--color-base-70);
-			font-size: var(--font-ui-smaller);
-		}
-	}
-
-	.button-container {
-		display: flex;
-		gap: var(--size-4-2);
-	}
-
-	h3 {
-		font-size: var(--font-ui-large);
-		margin-top: var(--heading-spacing);
-	}
-
-	button,
-	input[type="range"],
-	.checkbox-container,
-	.slider::-webkit-slider-thumb {
-		cursor: pointer;
-	}
-
-	input[type="text"],
-	textarea {
-		width: 250px;
-	}
-
-	textarea {
-		resize: none;
-		height: 75px;
-	}
-}
-
-.sync-status {
-	display: flex;
-	gap: var(--size-4-2);
-
-	* {
-		display: block;
-	}
-
-	.initialize-button {
-		padding: 0 var(--size-4-2);
-		background: rgba(var(--color-red-rgb), 0.4);
-		cursor: pointer;
-	}
-}
-
-.logs-view {
-	display: flex;
-	flex-direction: column;
-
-	.logs-container {
-		max-width: 100%;
-		overflow-y: auto;
-
-		.log-message {
-			font: var(--font-monospace);
-			margin-bottom: var(--size-2-1);
-			overflow-wrap: break-word;
-			white-space: pre-wrap;
-			user-select: all;
-
-			.timestamp {
-				@include number-card;
-				font-family: var(--font-monospace);
-				font-weight: var(--bold-weight);
-				margin-right: var(--size-4-1);
-			}
-
-			&.DEBUG {
-				color: var(--color-base-50);
-			}
-
-			&.INFO {
-				color: var(--color-green-rgb);
-			}
-
-			&.WARNING {
-				color: var(--color-yellow-rgb);
-			}
-
-			&.ERROR {
-				color: var(--color-red-rgb);
-			}
-		}
-	}
-}
-
-.history-card {
-	padding: var(--size-4-4);
-	margin: var(--size-4-2);
-	background-color: var(--color-base-00);
-	border-radius: var(--radius-l);
-	container-type: inline-size;
-
-	&.clickable {
-		cursor: pointer;
-	}
-
-	&.success {
-		background-color: rgba(var(--color-green-rgb), 0.2);
-	}
-
-	&.error {
-		background-color: rgba(var(--color-red-rgb), 0.2);
-	}
-
-	.history-card-header {
-		display: flex;
-		justify-content: space-between;
-		align-items: center;
-		margin-bottom: var(--size-4-2);
-		gap: var(--size-4-2);
-
-		@container (max-width: 300px) {
-			flex-direction: column;
-			align-items: flex-start;
-		}
-
-		.history-card-title {
-			font: var(--font-monospace);
-			display: flex;
-			align-items: center;
-			gap: var(--size-4-2);
-			word-break: break-all;
-			margin: 0;
-		}
-
-		.history-card-timestamp {
-			font-size: var(--font-ui-small);
-			font-style: italic;
-			color: var(--italic-color);
-		}
-	}
-
-	.history-card-message {
-		font-size: var(--font-ui-medium);
-		color: var(--color-base-70);
-		margin: 0;
-	}
-}

frontend/obsidian-plugin/src/vault-link-plugin.ts

@@ -1,20 +1,25 @@
-import type { WorkspaceLeaf } from "obsidian";
+import type {
-import { Platform, Plugin } from "obsidian";
+	Editor,
-import "./styles.scss";
+	MarkdownFileInfo,
+	MarkdownView,
+	TAbstractFile,
+	WorkspaceLeaf
+} from "obsidian";
+import { Platform, Plugin, TFile } from "obsidian";
 import "../manifest.json";
-import { SyncSettingsTab } from "./views/settings-tab";
+import { HistoryView } from "./views/history/history-view";
-import { HistoryView } from "./views/history-view";
+import { StatusBar } from "./views/status-bar/status-bar";
-import { ObsidianFileEventHandler } from "./obisidan-event-handler";
+import { LogsView } from "./views/logs/logs-view";
-import { StatusBar } from "./views/status-bar";
+import { StatusDescription } from "./views/status-description/status-description";
-import { LogsView } from "./views/logs-view";
-import { StatusDescription } from "./views/status-description";
 import type { LogLine } from "sync-client";
 import { SyncClient, LogLevel } from "sync-client";
 import { ObsidianFileSystemOperations } from "./obsidian-file-system";
+import { SyncSettingsTab } from "./views/settings/settings-tab";
 
 export default class VaultLinkPlugin extends Plugin {
 	private settingsTab: SyncSettingsTab | undefined;
 	private client!: SyncClient;
+
 	private static registerConsoleForLogging(client: SyncClient): void {
 		client.logger.addOnMessageListener((logLine: LogLine) => {
 			const formatted = `${logLine.timestamp.toISOString()} ${logLine.level} ${logLine.message}`;
@@ -38,7 +43,10 @@ export default class VaultLinkPlugin extends Plugin {
 
 	public async onload(): Promise<void> {
 		this.client = await SyncClient.create({
-			fs: new ObsidianFileSystemOperations(this.app.vault),
+			fs: new ObsidianFileSystemOperations(
+				this.app.vault,
+				this.app.workspace
+			),
 			persistence: {
 				load: this.loadData.bind(this),
 				save: this.saveData.bind(this)
@@ -80,35 +88,9 @@ export default class VaultLinkPlugin extends Plugin {
 			async (_: MouseEvent) => this.activateView(LogsView.TYPE)
 		);
 
-		const eventHandler = new ObsidianFileEventHandler(this.client);
-
 		this.app.workspace.onLayoutReady(async () => {
-			this.client.logger.info("Initialising sync handlers");
+			this.registerEditorEvents();
-
-			[
-				this.app.vault.on(
-					"create",
-					eventHandler.onCreate.bind(eventHandler)
-				),
-				this.app.vault.on(
-					"modify",
-					eventHandler.onModify.bind(eventHandler)
-				),
-				this.app.vault.on(
-					"delete",
-					eventHandler.onDelete.bind(eventHandler)
-				),
-				this.app.vault.on(
-					"rename",
-					eventHandler.onRename.bind(eventHandler)
-				)
-			].forEach((event) => {
-				this.registerEvent(event);
-			});
-
 			void this.client.start();
-
-			this.client.logger.info("Sync handlers initialised");
 		});
 	}
 
@@ -145,4 +127,51 @@ export default class VaultLinkPlugin extends Plugin {
 			await workspace.revealLeaf(leaf);
 		}
 	}
+
+	private registerEditorEvents(): void {
+		[
+			this.app.workspace.on(
+				"editor-change",
+				async (
+					_editor: Editor,
+					info: MarkdownView | MarkdownFileInfo
+				) => {
+					const { file } = info;
+					if (file) {
+						await this.client.syncLocallyUpdatedFile({
+							relativePath: file.path
+						});
+					}
+				}
+			),
+			this.app.vault.on("create", async (file: TAbstractFile) => {
+				if (file instanceof TFile) {
+					await this.client.syncLocallyCreatedFile(file.path);
+				}
+			}),
+			this.app.vault.on("modify", async (file: TAbstractFile) => {
+				if (file instanceof TFile) {
+					await this.client.syncLocallyUpdatedFile({
+						relativePath: file.path
+					});
+				}
+			}),
+			this.app.vault.on("delete", async (file: TAbstractFile) => {
+				await this.client.syncLocallyDeletedFile(file.path);
+			}),
+			this.app.vault.on(
+				"rename",
+				async (file: TAbstractFile, oldPath: string) => {
+					if (file instanceof TFile) {
+						await this.client.syncLocallyUpdatedFile({
+							oldPath,
+							relativePath: file.path
+						});
+					}
+				}
+			)
+		].forEach((event) => {
+			this.registerEvent(event);
+		});
+	}
 }

frontend/obsidian-plugin/src/views/history/history-view.scss

@@ -0,0 +1,53 @@
+.history-card {
+	padding: var(--size-4-4);
+	margin: var(--size-4-2);
+	background-color: var(--color-base-00);
+	border-radius: var(--radius-l);
+	container-type: inline-size;
+
+	&.clickable {
+		cursor: pointer;
+	}
+
+	&.success {
+		background-color: rgba(var(--color-green-rgb), 0.2);
+	}
+
+	&.error {
+		background-color: rgba(var(--color-red-rgb), 0.2);
+	}
+
+	.history-card-header {
+		display: flex;
+		justify-content: space-between;
+		align-items: center;
+		margin-bottom: var(--size-4-2);
+		gap: var(--size-4-2);
+
+		@container (max-width: 300px) {
+			flex-direction: column;
+			align-items: flex-start;
+		}
+
+		.history-card-title {
+			font: var(--font-monospace);
+			display: flex;
+			align-items: center;
+			gap: var(--size-4-2);
+			word-break: break-all;
+			margin: 0;
+		}
+
+		.history-card-timestamp {
+			font-size: var(--font-ui-small);
+			font-style: italic;
+			color: var(--italic-color);
+		}
+	}
+
+	.history-card-message {
+		font-size: var(--font-ui-medium);
+		color: var(--color-base-70);
+		margin: 0;
+	}
+}

frontend/obsidian-plugin/src/views/history/history-view.ts

@@ -1,6 +1,7 @@
+import "./history-view.scss";
+
 import type { IconName, WorkspaceLeaf } from "obsidian";
 import { ItemView, setIcon } from "obsidian";
-
 import { intlFormatDistance } from "date-fns";
 import type { HistoryEntry, SyncClient } from "sync-client";
 import { SyncType } from "sync-client";

frontend/obsidian-plugin/src/views/logs/logs-view.scss

@@ -0,0 +1,60 @@
+.logs-view {
+	display: flex;
+	flex-direction: column;
+
+	.verbosity-selector {
+		display: flex;
+		align-items: center;
+		justify-content: space-between;
+		font-weight: normal;
+		gap: var(--size-4-2);
+		margin: var(--size-4-4) var(--size-4-2);
+
+		h4 {
+			margin: 0;
+		}
+
+		select {
+			cursor: pointer;
+		}
+	}
+
+	.logs-container {
+		max-width: 100%;
+		overflow-y: auto;
+
+		.log-message {
+			font: var(--font-monospace);
+			margin-bottom: var(--size-2-1);
+			overflow-wrap: break-word;
+			white-space: pre-wrap;
+			user-select: all;
+
+			.timestamp {
+				padding: var(--size-2-1) var(--size-4-1);
+				border-radius: var(--radius-s);
+				background-color: var(--color-base-30);
+				font-size: var(--font-ui-small);
+				font-family: var(--font-monospace);
+				font-weight: var(--bold-weight);
+				margin-right: var(--size-4-1);
+			}
+
+			&.DEBUG {
+				color: var(--color-base-50);
+			}
+
+			&.INFO {
+				color: var(--color-base-100);
+			}
+
+			&.WARNING {
+				color: rgb(var(--color-yellow-rgb));
+			}
+
+			&.ERROR {
+				color: rgb(var(--color-red-rgb));
+			}
+		}
+	}
+}

frontend/obsidian-plugin/src/views/logs/logs-view.ts

@@ -1,3 +1,5 @@
+import "./logs-view.scss";
+
 import type { WorkspaceLeaf } from "obsidian";
 import { ItemView } from "obsidian";
 import type { LogLine } from "sync-client";
@@ -7,8 +9,11 @@ export class LogsView extends ItemView {
 	public static readonly TYPE = "logs-view";
 	public static readonly ICON = "logs";
 
+	private static readonly MAX_OFFSET_FROM_BOTTOM_WITH_AUTO_SCROLL_PX = 300;
+
 	private logsContainer: HTMLElement | undefined;
 	private readonly logLineToElement = new Map<LogLine, HTMLElement>();
+	private minLogLevel: LogLevel = LogLevel.INFO;
 
 	public constructor(
 		private readonly client: SyncClient,
@@ -56,10 +61,43 @@ export class LogsView extends ItemView {
 	public async onOpen(): Promise<void> {
 		const container = this.containerEl.children[1];
 		container.addClass("logs-view");
-		container.createEl("h4", { text: "VaultLink logs" });
-		this.logsContainer = container.createDiv({ cls: "logs-container" });
 
-		this.updateView();
+		const logLevels = [
+			{ label: "Debug", value: LogLevel.DEBUG },
+			{ label: "Info", value: LogLevel.INFO },
+			{ label: "Warn", value: LogLevel.WARNING },
+			{ label: "Error", value: LogLevel.ERROR }
+		];
+
+		container.createDiv(
+			{
+				cls: "verbosity-selector"
+			},
+			(verbositySection) => {
+				verbositySection.createEl("h4", {
+					text: "VaultLink logs"
+				});
+
+				verbositySection.createEl("select", {}, (dropdown) => {
+					logLevels.forEach(({ label, value }) =>
+						dropdown.createEl("option", { text: label, value })
+					);
+
+					dropdown.value = this.minLogLevel;
+
+					dropdown.addEventListener("change", () => {
+						// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
+						this.minLogLevel = dropdown.value as LogLevel;
+
+						this.logsContainer?.empty();
+						this.logLineToElement.clear();
+						this.updateView();
+					});
+				});
+			}
+		);
+
+		this.logsContainer = container.createDiv({ cls: "logs-container" });
 	}
 
 	private updateView(): void {
@@ -68,13 +106,20 @@ export class LogsView extends ItemView {
 			return;
 		}
 
-		const logs = this.client.logger.getMessages(LogLevel.DEBUG);
+		const logs = this.client.logger.getMessages(this.minLogLevel);
 
 		if (this.logLineToElement.size === 0 && logs.length > 0) {
 			// Clear the "No logs available yet" message
 			container.empty();
 		}
 
+		const shouldScroll =
+			container.scrollTop == 0 ||
+			container.scrollHeight -
+				container.clientHeight -
+				container.scrollTop <
+				LogsView.MAX_OFFSET_FROM_BOTTOM_WITH_AUTO_SCROLL_PX;
+
 		logs.forEach((message) => {
 			if (this.logLineToElement.has(message)) {
 				return;
@@ -98,6 +143,8 @@ export class LogsView extends ItemView {
 			container.createEl("p", {
 				text: "No logs available yet."
 			});
+		} else if (shouldScroll) {
+			container.scrollTop = container.scrollHeight;
 		}
 	}
 }

frontend/obsidian-plugin/src/views/settings/settings-tab.scss

@@ -0,0 +1,57 @@
+@mixin number-card {
+	padding: var(--size-2-1) var(--size-4-1);
+	border-radius: var(--radius-s);
+	background-color: var(--color-base-30);
+	font-size: var(--font-ui-small);
+
+	&.good {
+		background-color: rgba(var(--color-green-rgb), 0.35);
+	}
+
+	&.bad {
+		background-color: rgba(var(--color-red-rgb), 0.35);
+	}
+}
+
+.vault-link-settings {
+	h2 {
+		display: flex;
+		align-items: center;
+		font-size: var(--h2-size);
+
+		.version {
+			@include number-card;
+			margin: var(--size-2-2) 0 0 var(--size-4-2);
+			background-color: var(--color-base-30);
+			color: var(--color-base-70);
+			font-size: var(--font-ui-smaller);
+		}
+	}
+
+	.button-container {
+		display: flex;
+		gap: var(--size-4-2);
+	}
+
+	h3 {
+		font-size: var(--font-ui-large);
+		margin-top: var(--heading-spacing);
+	}
+
+	button,
+	input[type="range"],
+	.checkbox-container,
+	.slider::-webkit-slider-thumb {
+		cursor: pointer;
+	}
+
+	input[type="text"],
+	textarea {
+		width: 250px;
+	}
+
+	textarea {
+		resize: none;
+		height: 75px;
+	}
+}

frontend/obsidian-plugin/src/views/settings/settings-tab.ts

@@ -1,10 +1,12 @@
+import "./settings-tab.scss";
+
 import type { App } from "obsidian";
 import { Notice, PluginSettingTab, Setting } from "obsidian";
-import type VaultLinkPlugin from "../vault-link-plugin";
+import type VaultLinkPlugin from "src/vault-link-plugin";
-import type { StatusDescription } from "./status-description";
-import { LogsView } from "./logs-view";
-import { HistoryView } from "./history-view";
 import type { SyncClient, SyncSettings } from "sync-client";
+import { HistoryView } from "../history/history-view";
+import { LogsView } from "../logs/logs-view";
+import type { StatusDescription } from "../status-description/status-description";
 
 export class SyncSettingsTab extends PluginSettingTab {
 	private editedServerUri: string;
@@ -220,7 +222,7 @@ export class SyncSettingsTab extends PluginSettingTab {
 			.addButton((button) =>
 				button.setButtonText("Test connection").onClick(async () => {
 					new Notice(
-						(await this.syncClient.checkConnection()).message
+						(await this.syncClient.checkConnection()).serverMessage
 					);
 					await this.statusDescription.updateConnectionState();
 				})
@@ -246,29 +248,6 @@ export class SyncSettingsTab extends PluginSettingTab {
 					)
 			);
 
-		new Setting(containerEl)
-			.setName("Remote fetching frequency (seconds)")
-			.setDesc(
-				"Set how often should the plugin check for changes on the server. Lower values will increase the frequency of the checks making it easier to collaborate with others."
-			)
-			.setTooltip("todo, links to docs")
-			.addSlider((text) =>
-				text
-					.setLimits(0.5, 60, 0.5)
-					.setDynamicTooltip()
-					.setInstant(false)
-					.setValue(
-						this.syncClient.getSettings()
-							.fetchChangesUpdateIntervalMs / 1000
-					)
-					.onChange(async (value) =>
-						this.syncClient.setSetting(
-							"fetchChangesUpdateIntervalMs",
-							value * 1000
-						)
-					)
-			);
-
 		new Setting(containerEl)
 			.setName("Sync concurrency")
 			.setDesc(

frontend/obsidian-plugin/src/views/status-bar/status-bar.scss

@@ -0,0 +1,14 @@
+.sync-status {
+	display: flex;
+	gap: var(--size-4-2);
+
+	* {
+		display: block;
+	}
+
+	.initialize-button {
+		padding: 0 var(--size-4-2);
+		background: rgba(var(--color-red-rgb), 0.4);
+		cursor: pointer;
+	}
+}

frontend/obsidian-plugin/src/views/status-bar/status-bar.ts

@@ -1,5 +1,7 @@
+import "./status-bar.scss";
+
 import type { HistoryStats, SyncClient } from "sync-client";
-import type VaultLinkPlugin from "../vault-link-plugin";
+import type VaultLinkPlugin from "../../vault-link-plugin";
 
 export class StatusBar {
 	private readonly statusBarItem: HTMLElement;

frontend/obsidian-plugin/src/views/status-description/status-description.scss

@@ -0,0 +1,32 @@
+@mixin number-card {
+	padding: var(--size-2-1) var(--size-4-1);
+	border-radius: var(--radius-s);
+	background-color: var(--color-base-30);
+	font-size: var(--font-ui-small);
+
+	&.good {
+		background-color: rgba(var(--color-green-rgb), 0.35);
+	}
+
+	&.bad {
+		background-color: rgba(var(--color-red-rgb), 0.35);
+	}
+}
+
+.status-description {
+	margin: var(--p-spacing) 0;
+
+	.number {
+		@include number-card;
+		font-family: var(--font-monospace);
+		font-weight: var(--bold-weight);
+	}
+
+	.error {
+		color: rgb(var(--color-red-rgb));
+	}
+
+	.warning {
+		color: rgb(var(--color-yellow-rgb));
+	}
+}

frontend/obsidian-plugin/src/views/status-description/status-description.ts

@@ -1,13 +1,15 @@
+import "./status-description.scss";
+
 import type {
 	HistoryStats,
-	CheckConnectionResult,
+	NetworkConnectionStatus,
 	SyncClient
 } from "sync-client";
 
 export class StatusDescription {
 	private lastHistoryStats: HistoryStats | undefined;
 	private lastRemaining: number | undefined;
-	private lastConnectionState: CheckConnectionResult | undefined;
+	private lastConnectionState: NetworkConnectionStatus | undefined;
 
 	private statusChangeListeners: (() => void)[] = [];
 
@@ -26,9 +28,13 @@ export class StatusDescription {
 			}
 		);
 
-		this.syncClient.addOnSettingsChangeListener(() => {
+		this.syncClient.addWebSocketStatusChangeListener(
-			void this.updateConnectionState();
+			() => void this.updateConnectionState()
-		});
+		);
+
+		this.syncClient.addOnSettingsChangeListener(
+			() => void this.updateConnectionState()
+		);
 	}
 
 	public async updateConnectionState(): Promise<void> {
@@ -59,7 +65,15 @@ export class StatusDescription {
 
 		if (!this.lastConnectionState.isSuccessful) {
 			container.createSpan({
-				text: `VaultLink failed to connect to the remote server with the error "${this.lastConnectionState.message}"`,
+				text: `VaultLink failed to connect to the remote server with error '${this.lastConnectionState.serverMessage}'`,
+				cls: "error"
+			});
+			return;
+		}
+
+		if (!this.lastConnectionState.isWebSocketConnected) {
+			container.createSpan({
+				text: `${this.lastConnectionState.serverMessage} but the WebSocket connection could not be established.`,
 				cls: "error"
 			});
 			return;

frontend/obsidian-plugin/tsconfig.json

@@ -6,7 +6,12 @@
 		"strict": true,
 		"moduleResolution": "bundler",
 		"allowSyntheticDefaultImports": true,
-		"lib": ["DOM", "ESNext"]
+		"lib": [
+			"DOM",
+			"ESNext"
+		]
 	},
-	"exclude": ["./dist"]
+	"exclude": [
-}
+		"./dist"
+	]
+}

frontend/sync-client/package.json

@@ -23,6 +23,7 @@
 		"@types/jest": "^29.5.14",
 		"@types/node": "^22.13.10",
 		"jest": "^29.7.0",
+		"sync_lib": "file:../../backend/sync_lib/pkg",
 		"ts-jest": "^29.2.6",
 		"ts-loader": "^9.5.2",
 		"tslib": "2.8.1",
@@ -30,6 +31,6 @@
 		"webpack": "^5.98.0",
 		"webpack-cli": "^6.0.1",
 		"webpack-merge": "^6.0.1",
-		"sync_lib": "file:../../backend/sync_lib/pkg"
+		"ws": "^8.18.1"
 	}
-}
+}

frontend/sync-client/src/file-operations/document-locks.ts

@@ -1,65 +0,0 @@
-import type { Logger } from "../tracing/logger";
-import type { RelativePath } from "../persistence/database";
-
-// Manages locks on documents to prevent concurrent modifications
-// allowing the client's FileOperations implementation to be simpler.
-// Locks are granted in a first-in-first-out order.
-export class DocumentLocks {
-	private readonly locked = new Set<RelativePath>();
-	private readonly waiters = new Map<RelativePath, (() => void)[]>();
-
-	public constructor(private readonly logger: Logger) {}
-
-	public tryLockDocument(relativePath: RelativePath): boolean {
-		if (this.locked.has(relativePath)) {
-			return false;
-		}
-
-		this.locked.add(relativePath);
-
-		return true;
-	}
-
-	public async waitForDocumentLock(
-		relativePath: RelativePath
-	): Promise<void> {
-		if (this.tryLockDocument(relativePath)) {
-			return Promise.resolve();
-		}
-
-		this.logger.debug(`Waiting for lock on ${relativePath}`);
-
-		return new Promise((resolve) => {
-			let waiting = this.waiters.get(relativePath);
-			if (!waiting) {
-				waiting = [];
-				this.waiters.set(relativePath, waiting);
-			}
-
-			waiting.push(resolve);
-		});
-	}
-
-	public unlockDocument(relativePath: RelativePath): void {
-		if (!this.locked.has(relativePath)) {
-			throw new Error(
-				`Document ${relativePath} is not locked, cannot unlock`
-			);
-		}
-
-		// Remove the first element to ensure FIFO unblocking order
-		const nextWaiting = this.waiters.get(relativePath)?.shift();
-
-		if (nextWaiting) {
-			this.logger.debug(`Granted lock on ${relativePath}`);
-			nextWaiting();
-		} else {
-			this.locked.delete(relativePath);
-		}
-	}
-
-	public reset(): void {
-		this.locked.clear();
-		this.waiters.clear();
-	}
-}

frontend/sync-client/src/file-operations/safe-filesystem-operations.ts

@@ -1,7 +1,7 @@
 import type { RelativePath } from "../persistence/database";
 import type { FileSystemOperations } from "./filesystem-operations";
 import type { Logger } from "../tracing/logger";
-import { DocumentLocks } from "./document-locks";
+import { Locks } from "../utils/locks";
 import { FileNotFoundError } from "./file-not-found-error";
 
 /**
@@ -10,13 +10,13 @@ import { FileNotFoundError } from "./file-not-found-error";
  * single request in-flight for any one file through the use of locks.
  */
 export class SafeFileSystemOperations implements FileSystemOperations {
-	private readonly locks: DocumentLocks;
+	private readonly locks: Locks<RelativePath>;
 
 	public constructor(
 		private readonly fs: FileSystemOperations,
 		private readonly logger: Logger
 	) {
-		this.locks = new DocumentLocks(logger);
+		this.locks = new Locks(logger);
 	}
 
 	public async listAllFiles(): Promise<RelativePath[]> {
@@ -117,7 +117,7 @@ export class SafeFileSystemOperations implements FileSystemOperations {
 				: [pathOrPaths];
 
 			await Promise.all(
-				paths.map(async (path) => this.locks.waitForDocumentLock(path))
+				paths.map(async (path) => this.locks.waitForLock(path))
 			);
 
 			try {
@@ -125,7 +125,7 @@ export class SafeFileSystemOperations implements FileSystemOperations {
 			} finally {
 				await Promise.all(
 					paths.map((path) => {
-						this.locks.unlockDocument(path);
+						this.locks.unlock(path);
 					})
 				);
 			}

frontend/sync-client/src/index.ts

@@ -5,10 +5,10 @@ export {
 	type HistoryEntry
 } from "./tracing/sync-history";
 export { Logger, LogLevel, LogLine } from "./tracing/logger";
-export type { CheckConnectionResult } from "./services/sync-service";
 export { type SyncSettings } from "./persistence/settings";
 export type { RelativePath, StoredDatabase } from "./persistence/database";
 export type { FileSystemOperations } from "./file-operations/filesystem-operations";
 export type { PersistenceProvider } from "./persistence/persistence";
 
+export type { NetworkConnectionStatus } from "./sync-client";
 export { SyncClient } from "./sync-client";

frontend/sync-client/src/persistence/settings.ts

@@ -1,11 +1,9 @@
 import type { Logger } from "../tracing/logger";
-import { LogLevel } from "../tracing/logger";
 
 export interface SyncSettings {
 	remoteUri: string;
 	token: string;
 	vaultName: string;
-	fetchChangesUpdateIntervalMs: number;
 	syncConcurrency: number;
 	isSyncEnabled: boolean;
 	maxFileSizeMB: number;
@@ -15,7 +13,6 @@ const DEFAULT_SETTINGS: SyncSettings = {
 	remoteUri: "",
 	token: "",
 	vaultName: "default",
-	fetchChangesUpdateIntervalMs: 1000,
 	syncConcurrency: 1,
 	isSyncEnabled: false,
 	maxFileSizeMB: 10

frontend/sync-client/src/services/sync-service.ts

@@ -18,6 +18,7 @@ export interface CheckConnectionResult {
 }
 
 export class SyncService {
+	private static readonly NETWORK_RETRY_INTERVAL_MS = 1000;
 	private client: Client<paths>;
 	private pingClient: Client<paths>;
 	private _fetchImplementation: typeof globalThis.fetch = globalThis.fetch;
@@ -284,17 +285,35 @@ export class SyncService {
 
 	public async checkConnection(): Promise<CheckConnectionResult> {
 		try {
-			const result = await this.ping();
+			const response = await this.pingClient.GET("/ping", {
+				params: {
+					header: {
+						authorization: `Bearer ${this.settings.getSettings().token}`
+					}
+				}
+			});
+
+			this.logger.debug(
+				`Ping response: ${JSON.stringify(response.data)}`
+			);
+
+			if (!response.data) {
+				throw new Error(
+					`Failed to ping server: ${SyncService.formatError(response.error)}`
+				);
+			}
+
+			const result = response.data;
 			if (result.isAuthenticated) {
 				return {
 					isSuccessful: true,
-					message: `Successfully connected to server (version: ${result.serverVersion}) and authenticated.`
+					message: `Successfully connected to server (version: ${result.serverVersion}) and authenticated`
 				};
 			}
 
 			return {
 				isSuccessful: false,
-				message: `Successfully connected to server (version: ${result.serverVersion}) but failed to authenticate.`
+				message: `Successfully connected to server (version: ${result.serverVersion}) but failed to authenticate`
 			};
 		} catch (e) {
 			return {
@@ -304,27 +323,6 @@ export class SyncService {
 		}
 	}
 
-	// No retries
-	private async ping(): Promise<components["schemas"]["PingResponse"]> {
-		const response = await this.pingClient.GET("/ping", {
-			params: {
-				header: {
-					authorization: `Bearer ${this.settings.getSettings().token}`
-				}
-			}
-		});
-
-		this.logger.debug(`Ping response: ${JSON.stringify(response.data)}`);
-
-		if (!response.data) {
-			throw new Error(
-				`Failed to ping server: ${SyncService.formatError(response.error)}`
-			);
-		}
-
-		return response.data;
-	}
-
 	/**
 	 * Create a client and a ping client for the given remote URI.
 	 */
@@ -355,8 +353,10 @@ export class SyncService {
 					throw e;
 				}
 
-				this.logger.error(`Failed network call (${e}), retrying`);
+				this.logger.error(
-				await sleep(1000);
+					`Failed network call (${e}), retryingin ${SyncService.NETWORK_RETRY_INTERVAL_MS}ms`
+				);
+				await sleep(SyncService.NETWORK_RETRY_INTERVAL_MS);
 			}
 		}
 	}

frontend/sync-client/src/services/types.ts

@@ -566,6 +566,10 @@ export interface components {
 			/** Format: int64 */
 			since_update_id?: number | null;
 		};
+		QueryParams2: {
+			/** Format: int64 */
+			since_update_id?: number | null;
+		};
 		SerializedError: {
 			causes: string[];
 			message: string;
@@ -587,6 +591,9 @@ export interface components {
 			parentVersionId: number;
 			relativePath: string;
 		};
+		WebsocketPathParams: {
+			vault_id: string;
+		};
 	};
 	responses: never;
 	parameters: never;

frontend/sync-client/src/sync-client.ts

@@ -8,7 +8,6 @@ import type { RelativePath, StoredDatabase } from "./persistence/database";
 import { Database } from "./persistence/database";
 import type { SyncSettings } from "./persistence/settings";
 import { Settings } from "./persistence/settings";
-import type { CheckConnectionResult } from "./services/sync-service";
 import { SyncService } from "./services/sync-service";
 import { Syncer } from "./sync-operations/syncer";
 import type { FileSystemOperations } from "./file-operations/filesystem-operations";
@@ -16,9 +15,13 @@ import { FileOperations } from "./file-operations/file-operations";
 import { ConnectionStatus } from "./services/connection-status";
 import { UnrestrictedSyncer } from "./sync-operations/unrestricted-syncer";
 
-export class SyncClient {
+export interface NetworkConnectionStatus {
-	private remoteListenerIntervalId: NodeJS.Timeout | null = null;
+	isSuccessful: boolean;
+	serverMessage: string;
+	isWebSocketConnected: boolean;
+}
 
+export class SyncClient {
 	// eslint-disable-next-line @typescript-eslint/max-params
 	private constructor(
 		private readonly history: SyncHistory,
@@ -31,15 +34,6 @@ export class SyncClient {
 	) {
 		this.settings.addOnSettingsChangeListener(
 			(newSettings, oldSettings) => {
-				if (
-					newSettings.fetchChangesUpdateIntervalMs !==
-					oldSettings.fetchChangesUpdateIntervalMs
-				) {
-					this.setRemoteEventListener(
-						newSettings.fetchChangesUpdateIntervalMs
-					);
-				}
-
 				if (newSettings.vaultName !== oldSettings.vaultName) {
 					void this.reset();
 				}
@@ -145,8 +139,13 @@ export class SyncClient {
 		return client;
 	}
 
-	public async checkConnection(): Promise<CheckConnectionResult> {
+	public async checkConnection(): Promise<NetworkConnectionStatus> {
-		return this.syncService.checkConnection();
+		const server = await this.syncService.checkConnection();
+		return {
+			isSuccessful: server.isSuccessful,
+			serverMessage: server.message,
+			isWebSocketConnected: this.syncer.isWebSocketConnected
+		};
 	}
 
 	public getHistoryEntries(): readonly HistoryEntry[] {
@@ -161,20 +160,15 @@ export class SyncClient {
 
 	public async start(): Promise<void> {
 		await this.syncer.scheduleSyncForOfflineChanges();
-
-		this.setRemoteEventListener(
-			this.settings.getSettings().fetchChangesUpdateIntervalMs
-		);
 	}
 
-	/// Clear all global state that has been touched by SyncClient.
 	public stop(): void {
-		this.unsetRemoteEventListener();
+		this.syncer.stop();
 	}
 
 	public async waitAndStop(): Promise<void> {
-		await this.syncer.waitUntilFinished();
 		this.stop();
+		await this.syncer.waitUntilFinished();
 	}
 
 	/// Wait for the in-flight operations to finish, reset all tracking,
@@ -218,6 +212,10 @@ export class SyncClient {
 		this.syncer.addRemainingOperationsListener(listener);
 	}
 
+	public addWebSocketStatusChangeListener(listener: () => void): void {
+		this.syncer.addWebSocketStatusChangeListener(listener);
+	}
+
 	public async syncLocallyCreatedFile(
 		relativePath: RelativePath
 	): Promise<void> {
@@ -242,21 +240,4 @@ export class SyncClient {
 			relativePath
 		});
 	}
-
-	private setRemoteEventListener(intervalMs: number): void {
-		if (this.remoteListenerIntervalId !== null) {
-			clearInterval(this.remoteListenerIntervalId);
-		}
-
-		this.remoteListenerIntervalId = setInterval(
-			() => void this.syncer.applyRemoteChangesLocally(),
-			intervalMs
-		);
-	}
-
-	private unsetRemoteEventListener(): void {
-		if (this.remoteListenerIntervalId !== null) {
-			clearInterval(this.remoteListenerIntervalId);
-		}
-	}
 }

frontend/sync-client/src/sync-operations/syncer.ts

@@ -1,30 +1,40 @@
-import type { Database, RelativePath } from "../persistence/database";
+import type {
+	Database,
+	DocumentId,
+	RelativePath
+} from "../persistence/database";
 import type { SyncService } from "../services/sync-service";
 import type { Logger } from "../tracing/logger";
 import PQueue from "p-queue";
 import { hash } from "../utils/hash";
 import { v4 as uuidv4 } from "uuid";
 import type { components } from "../services/types";
-import type { Settings } from "../persistence/settings";
+import type { Settings, SyncSettings } from "../persistence/settings";
 import type { FileOperations } from "../file-operations/file-operations";
 import { findMatchingFile } from "../utils/find-matching-file";
 import type { UnrestrictedSyncer } from "./unrestricted-syncer";
 import { createPromise } from "../utils/create-promise";
 import { SyncResetError } from "../services/sync-reset-error";
+import { Locks } from "../utils/locks";
 
 export class Syncer {
+	private readonly remoteDocumentsLock: Locks<DocumentId>;
 	private readonly remainingOperationsListeners: ((
 		remainingOperations: number
 	) => void)[] = [];
+	private readonly webSocketStatusChangeListeners: (() => void)[] = [];
 	private readonly syncQueue: PQueue;
 
 	private runningScheduleSyncForOfflineChanges: Promise<void> | undefined;
-	private runningApplyRemoteChangesLocally: Promise<void> | undefined;
+	private refreshApplyRemoteChangesWebSocketInterval:
+		| NodeJS.Timeout
+		| undefined;
+	private applyRemoteChangesWebSocket: WebSocket | undefined;
 
 	public constructor(
 		private readonly logger: Logger,
 		private readonly database: Database,
-		settings: Settings,
+		private readonly settings: Settings,
 		private readonly syncService: SyncService,
 		private readonly operations: FileOperations,
 		private readonly internalSyncer: UnrestrictedSyncer
@@ -33,11 +43,23 @@ export class Syncer {
 			concurrency: settings.getSettings().syncConcurrency
 		});
 
+		this.updateWebSocket(settings.getSettings());
+
+		this.remoteDocumentsLock = new Locks<DocumentId>(this.logger);
+
 		settings.addOnSettingsChangeListener((newSettings, oldSettings) => {
-			if (newSettings.syncConcurrency === oldSettings.syncConcurrency) {
+			if (
-				return;
+				newSettings.remoteUri !== oldSettings.remoteUri ||
+				newSettings.vaultName !== oldSettings.vaultName ||
+				newSettings.token !== oldSettings.token ||
+				newSettings.isSyncEnabled !== oldSettings.isSyncEnabled
+			) {
+				this.updateWebSocket(newSettings);
+			}
+
+			if (newSettings.syncConcurrency !== oldSettings.syncConcurrency) {
+				this.syncQueue.concurrency = newSettings.syncConcurrency;
 			}
-			this.syncQueue.concurrency = newSettings.syncConcurrency;
 		});
 
 		this.syncQueue.on("active", () => {
@@ -45,6 +67,12 @@ export class Syncer {
 				listener(this.syncQueue.size);
 			});
 		});
+
+		this.setWebSocketRefreshInterval();
+	}
+
+	public get isWebSocketConnected(): boolean {
+		return this.applyRemoteChangesWebSocket?.readyState === WebSocket.OPEN;
 	}
 
 	public addRemainingOperationsListener(
@@ -53,6 +81,10 @@ export class Syncer {
 		this.remainingOperationsListeners.push(listener);
 	}
 
+	public addWebSocketStatusChangeListener(listener: () => void): void {
+		this.webSocketStatusChangeListeners.push(listener);
+	}
+
 	public async syncLocallyCreatedFile(
 		relativePath: RelativePath
 	): Promise<void> {
@@ -206,109 +238,139 @@ export class Syncer {
 		}
 	}
 
-	public async applyRemoteChangesLocally(): Promise<void> {
+	public async waitUntilFinished(): Promise<void> {
-		if (this.runningApplyRemoteChangesLocally !== undefined) {
+		await this.runningScheduleSyncForOfflineChanges;
-			this.logger.debug(
+		return this.syncQueue.onEmpty();
-				"Applying remote changes locally is already in progress"
-			);
-			return this.runningApplyRemoteChangesLocally;
-		}
-
-		try {
-			this.runningApplyRemoteChangesLocally =
-				this.internalApplyRemoteChangesLocally();
-			await this.runningApplyRemoteChangesLocally;
-			this.logger.info("All remote changes have been applied locally");
-		} catch (e) {
-			if (e instanceof SyncResetError) {
-				this.logger.info(
-					"Failed to apply remote changes locally due to a reset"
-				);
-				return;
-			}
-			this.logger.error(`Failed to apply remote changes locally: ${e}`);
-			throw e;
-		} finally {
-			this.runningApplyRemoteChangesLocally = undefined;
-		}
 	}
 
 	public async reset(): Promise<void> {
 		await this.waitUntilFinished();
-		this.internalSyncer.reset();
+		this.setWebSocketRefreshInterval();
+		this.updateWebSocket(this.settings.getSettings());
 	}
 
-	public async waitUntilFinished(): Promise<void> {
+	public stop(): void {
-		await Promise.allSettled([
+		clearInterval(this.refreshApplyRemoteChangesWebSocketInterval);
-			this.runningScheduleSyncForOfflineChanges,
+		this.applyRemoteChangesWebSocket?.close();
-			this.runningApplyRemoteChangesLocally
-		]);
-		return this.syncQueue.onEmpty();
 	}
 
-	private async internalApplyRemoteChangesLocally(): Promise<void> {
+	private updateWebSocket(settings: SyncSettings): void {
-		const remote = await this.syncQueue.add(async () =>
+		this.applyRemoteChangesWebSocket?.close();
-			this.syncService.getAll(this.database.getLastSeenUpdateId())
-		);
 
-		if (!remote) {
+		if (!settings.isSyncEnabled) {
-			throw new Error("Failed to fetch remote changes");
+			this.applyRemoteChangesWebSocket = undefined;
-		}
-
-		if (remote.latestDocuments.length === 0) {
-			this.logger.debug("No remote changes to apply");
 			return;
 		}
 
-		this.logger.info("Applying remote changes locally");
+		const wsUri = new URL(settings.remoteUri);
+		wsUri.protocol = wsUri.protocol === "https" ? "wss" : "ws";
+		wsUri.pathname = `/vaults/${settings.vaultName}/ws`;
 
-		await Promise.all(
-			remote.latestDocuments.map(this.syncRemotelyUpdatedFile.bind(this))
-		);
-
-		const lastSeenUpdateId = this.database.getLastSeenUpdateId();
 		if (
-			lastSeenUpdateId === undefined ||
+			typeof globalThis !== "undefined" &&
-			lastSeenUpdateId < remote.lastUpdateId
+			typeof globalThis.WebSocket === "undefined"
 		) {
-			this.database.setLastSeenUpdateId(remote.lastUpdateId);
+			// polyfill for WebSocket in Node.js
+			// eslint-disable-next-line
+			globalThis.WebSocket = require("ws");
 		}
+
+		this.applyRemoteChangesWebSocket = new WebSocket(wsUri);
+
+		this.applyRemoteChangesWebSocket.onmessage = (event): void =>
+			void this.syncRemotelyUpdatedFile(event.data).catch(
+				(e: unknown) => {
+					this.logger.error(
+						`Failed to sync remotely updated file: ${e}`
+					);
+				}
+			);
+
+		// The JS WebSocket API doesn't support setting headers, so we have to send the token as a message
+		this.applyRemoteChangesWebSocket.onopen = (): void => {
+			this.applyRemoteChangesWebSocket?.send(settings.token);
+			this.webSocketStatusChangeListeners.forEach((listener) => {
+				listener();
+			});
+		};
+
+		this.applyRemoteChangesWebSocket.onclose = (event): void => {
+			this.logger.warn(
+				`WebSocket closed with code ${event.code}: ${event.reason}`
+			);
+			this.webSocketStatusChangeListeners.forEach((listener) => {
+				listener();
+			});
+		};
 	}
 
-	private async syncRemotelyUpdatedFile(
+	private setWebSocketRefreshInterval(): void {
-		remoteVersion: components["schemas"]["DocumentVersionWithoutContent"]
+		this.refreshApplyRemoteChangesWebSocketInterval = setInterval(() => {
-	): Promise<void> {
+			if (
+				this.applyRemoteChangesWebSocket?.readyState === WebSocket.OPEN
+			) {
+				return;
+			}
+			this.updateWebSocket(this.settings.getSettings());
+		}, 5000);
+	}
+
+	private async syncRemotelyUpdatedFile(message: string): Promise<void> {
+		// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
+		const remoteVersion = JSON.parse(
+			message
+		) as components["schemas"]["DocumentVersionWithoutContent"];
+
 		let document = this.database.getDocumentByDocumentId(
 			remoteVersion.documentId
 		);
 
-		const [promise, resolve, reject] = createPromise();
+		let hasLockToRelease = false;
-
 		if (document === undefined) {
-			await this.syncQueue.add(async () =>
+			// Let's avoid the same documents getting created in parallel multiple times
-				this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
+			await this.remoteDocumentsLock.waitForLock(
-					remoteVersion
+				remoteVersion.documentId
-				)
 			);
-		} else {
+			hasLockToRelease = true;
-			document = await this.database.getResolvedDocumentByRelativePath(
+			document = this.database.getDocumentByDocumentId(
-				document.relativePath,
+				remoteVersion.documentId
-				promise
 			);
+		}
 
-			try {
+		try {
+			if (document === undefined) {
 				await this.syncQueue.add(async () =>
 					this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
-						remoteVersion,
+						remoteVersion
-						document
 					)
 				);
+			} else {
+				const [promise, resolve, reject] = createPromise();
 
-				resolve();
+				document =
-			} catch (e) {
+					await this.database.getResolvedDocumentByRelativePath(
-				reject(e);
+						document.relativePath,
-			} finally {
+						promise
-				this.database.removeDocumentPromise(promise);
+					);
+
+				try {
+					await this.syncQueue.add(async () =>
+						this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
+							remoteVersion,
+							document
+						)
+					);
+
+					resolve();
+				} catch (e) {
+					reject(e);
+				} finally {
+					this.database.removeDocumentPromise(promise);
+				}
+			}
+		} finally {
+			if (hasLockToRelease) {
+				this.remoteDocumentsLock.unlock(remoteVersion.documentId);
 			}
 		}
 	}

frontend/sync-client/src/sync-operations/unrestricted-syncer.ts

@@ -13,14 +13,11 @@ import type { components } from "../services/types";
 import { deserialize } from "../utils/deserialize";
 import type { Settings } from "../persistence/settings";
 import type { FileOperations } from "../file-operations/file-operations";
-import { DocumentLocks } from "../file-operations/document-locks";
 import { createPromise } from "../utils/create-promise";
 import { FileNotFoundError } from "../file-operations/file-not-found-error";
 import { SyncResetError } from "../services/sync-reset-error";
 
 export class UnrestrictedSyncer {
-	private readonly locks: DocumentLocks;
-
 	public constructor(
 		private readonly logger: Logger,
 		private readonly database: Database,
@@ -28,10 +25,7 @@ export class UnrestrictedSyncer {
 		private readonly syncService: SyncService,
 		private readonly operations: FileOperations,
 		private readonly history: SyncHistory
-	) {
+	) {}
-		this.locks = new DocumentLocks(logger);
-	}
-
 	public async unrestrictedSyncLocallyCreatedFile(
 		document: DocumentRecord
 	): Promise<void> {
@@ -416,10 +410,6 @@ export class UnrestrictedSyncer {
 		}
 	}
 
-	public reset(): void {
-		this.locks.reset();
-	}
-
 	private tryIncrementVaultUpdateId(responseVaultUpdateId: number): void {
 		if (this.database.getLastSeenUpdateId() === responseVaultUpdateId - 1) {
 			this.database.setLastSeenUpdateId(responseVaultUpdateId);

frontend/sync-client/src/tracing/logger.ts

@@ -21,7 +21,7 @@ export class LogLine {
 }
 
 export class Logger {
-	private static readonly MAX_MESSAGES = 2000;
+	private static readonly MAX_MESSAGES = 100000;
 	private readonly messages: LogLine[] = [];
 	private readonly onMessageListeners: ((message: LogLine) => void)[] = [];
 

frontend/sync-client/src/utils/locks.test.ts

@@ -1,92 +1,88 @@
 import { Logger } from "../tracing/logger";
 import type { RelativePath } from "../persistence/database";
-import { DocumentLocks } from "./document-locks";
+import { Locks } from "./locks";
 
 describe("Document lock", () => {
 	const testPath: RelativePath = "test/document/path";
 	const logger = new Logger();
-	let locks = new DocumentLocks(logger);
+
+	// eslint-disable-next-line @typescript-eslint/init-declarations
+	let locks: Locks<RelativePath>;
 
 	beforeEach(() => {
-		locks = new DocumentLocks(logger);
+		locks = new Locks<RelativePath>(logger);
 	});
 
 	test("should lock a document successfully", () => {
-		const result = locks.tryLockDocument(testPath);
+		const result = locks.tryLock(testPath);
 		expect(result).toBe(true);
 	});
 
 	test("should not lock a document that is already locked", () => {
-		locks.tryLockDocument(testPath);
+		locks.tryLock(testPath);
-		const result = locks.tryLockDocument(testPath);
+		const result = locks.tryLock(testPath);
 		expect(result).toBe(false);
 	});
 
 	test("should unlock a locked document", () => {
-		locks.tryLockDocument(testPath);
+		locks.tryLock(testPath);
-		locks.unlockDocument(testPath);
+		locks.unlock(testPath);
-		const result = locks.tryLockDocument(testPath);
+		const result = locks.tryLock(testPath);
 		expect(result).toBe(true);
-		locks.unlockDocument(testPath);
+		locks.unlock(testPath);
 	});
 
 	test("should throw an error when unlocking a document that is not locked", () => {
 		expect(() => {
-			locks.unlockDocument(testPath);
+			locks.unlock(testPath);
 		}).toThrow(`Document ${testPath} is not locked, cannot unlock`);
 	});
 
 	test("should wait for a document lock and resolve when unlocked", async () => {
-		locks.tryLockDocument(testPath);
+		locks.tryLock(testPath);
 
 		let resolved = false;
-		const waitPromise = locks.waitForDocumentLock(testPath).then(() => {
+		const waitPromise = locks.waitForLock(testPath).then(() => {
 			resolved = true;
 		});
 
-		locks.unlockDocument(testPath);
+		locks.unlock(testPath);
 		await waitPromise;
 
 		expect(resolved).toBe(true);
 	});
 
 	test("should resolve multiple waiters in FIFO order", async () => {
-		locks.tryLockDocument(testPath);
+		locks.tryLock(testPath);
 
 		let firstResolved = false;
 		let secondResolved = false;
 		let thirdResolved = false;
 
-		const firstWaitPromise = locks
+		const firstWaitPromise = locks.waitForLock(testPath).then(() => {
-			.waitForDocumentLock(testPath)
+			firstResolved = true;
-			.then(() => {
+		});
-				firstResolved = true;
-			});
 
-		const secondWaitPromise = locks
+		const secondWaitPromise = locks.waitForLock(testPath).then(() => {
-			.waitForDocumentLock(testPath)
+			secondResolved = true;
-			.then(() => {
+		});
-				secondResolved = true;
-			});
 
-		const thirdWaitPromise = locks
+		const thirdWaitPromise = locks.waitForLock(testPath).then(() => {
-			.waitForDocumentLock(testPath)
+			thirdResolved = true;
-			.then(() => {
+		});
-				thirdResolved = true;
-			});
 
-		locks.unlockDocument(testPath);
+		locks.unlock(testPath);
 		await firstWaitPromise;
 		expect(firstResolved).toBe(true);
 		expect(secondResolved).toBe(false);
 		expect(thirdResolved).toBe(false);
 
-		locks.unlockDocument(testPath);
+		locks.unlock(testPath);
 		await secondWaitPromise;
 		expect(secondResolved).toBe(true);
 		expect(thirdResolved).toBe(false);
 
-		locks.unlockDocument(testPath);
+		locks.unlock(testPath);
 		await thirdWaitPromise;
 		expect(thirdResolved).toBe(true);
 	});

frontend/sync-client/src/utils/locks.ts

@@ -0,0 +1,60 @@
+import type { Logger } from "../tracing/logger";
+
+// Manages locks on T to prevent concurrent modifications
+// allowing the client's FileOperations implementation to be simpler.
+// Locks are granted in a first-in-first-out order.
+export class Locks<T> {
+	private readonly locked = new Set<T>();
+	private readonly waiters = new Map<T, (() => void)[]>();
+
+	public constructor(private readonly logger: Logger) {}
+
+	public tryLock(key: T): boolean {
+		if (this.locked.has(key)) {
+			return false;
+		}
+
+		this.locked.add(key);
+
+		return true;
+	}
+
+	public async waitForLock(key: T): Promise<void> {
+		if (this.tryLock(key)) {
+			return Promise.resolve();
+		}
+
+		this.logger.debug(`Waiting for lock on ${key}`);
+
+		return new Promise((resolve) => {
+			let waiting = this.waiters.get(key);
+			if (!waiting) {
+				waiting = [];
+				this.waiters.set(key, waiting);
+			}
+
+			waiting.push(resolve);
+		});
+	}
+
+	public unlock(key: T): void {
+		if (!this.locked.has(key)) {
+			throw new Error(`Document ${key} is not locked, cannot unlock`);
+		}
+
+		// Remove the first element to ensure FIFO unblocking order
+		const nextWaiting = this.waiters.get(key)?.shift();
+
+		if (nextWaiting) {
+			this.logger.debug(`Granted lock on ${key}`);
+			nextWaiting();
+		} else {
+			this.locked.delete(key);
+		}
+	}
+
+	public reset(): void {
+		this.locked.clear();
+		this.waiters.clear();
+	}
+}

frontend/sync-client/tsconfig.json

@@ -6,10 +6,12 @@
 		"allowSyntheticDefaultImports": true,
 		"moduleResolution": "bundler",
 		"lib": [
-			"DOM" // to get "fetch"
+			"DOM" // to get `fetch` & `WebSocket`
 		],
 		"declaration": true,
 		"declarationDir": "./dist/types"
 	},
-	"exclude": ["./dist"]
+	"exclude": [
-}
+		"./dist"
+	]
+}

frontend/sync-client/webpack.config.js

@@ -20,7 +20,7 @@ const common = {
 		minimize: false
 	},
 	resolve: {
-		extensions: [".ts"],
+		extensions: [".ts", ".js"],
 		alias: {
 			root: __dirname,
 			src: path.resolve(__dirname, "src")
@@ -42,6 +42,11 @@ module.exports = [
 				type: "umd"
 			},
 			globalObject: "this"
+		},
+		resolve: {
+			fallback: {
+				ws: false // Exclude `ws` from the browser bundle
+			}
 		}
 	}),
 	merge(common, {
@@ -50,6 +55,10 @@ module.exports = [
 			path: path.resolve(__dirname, "dist"),
 			filename: "sync-client.node.js",
 			libraryTarget: "commonjs2"
+		},
+		externals: {
+			bufferutil: "bufferutil",
+			"utf-8-validate": "utf-8-validate" // required for ws: https://github.com/websockets/ws/issues/2245#issuecomment-2250318733
 		}
 	})
 ];

frontend/test-client/package.json

@@ -18,6 +18,7 @@
 		"typescript": "5.8.2",
 		"uuid": "^11.1.0",
 		"webpack": "^5.98.0",
-		"webpack-cli": "^6.0.1"
+		"webpack-cli": "^6.0.1",
+		"bufferutil": "^4.0.9"
 	}
-}
+}

frontend/test-client/src/agent/mock-agent.ts

@@ -88,8 +88,7 @@ export class MockAgent extends MockClient {
 
 	public async act(): Promise<void> {
 		const options: (() => Promise<unknown>)[] = [
-			this.createFileAction.bind(this),
+			this.createFileAction.bind(this)
-			this.changeFetchChangesUpdateIntervalMsAction.bind(this)
 		];
 
 		if (this.client.getSettings().isSyncEnabled) {
@@ -253,16 +252,6 @@ export class MockAgent extends MockClient {
 		return this.create(file, new TextEncoder().encode(` ${content} `));
 	}
 
-	private async changeFetchChangesUpdateIntervalMsAction(): Promise<void> {
-		this.client.logger.info(
-			`Decided to change fetchChangesUpdateIntervalMs`
-		);
-		return this.client.setSetting(
-			"fetchChangesUpdateIntervalMs",
-			Math.random() * 2000 + 100
-		);
-	}
-
 	private async disableSyncAction(): Promise<void> {
 		this.client.logger.info(`Decided to disable sync`);
 		await this.client.setSetting("isSyncEnabled", false);

frontend/test-client/src/agent/mock-client.ts

@@ -1,4 +1,4 @@
-import type { StoredDatabase } from "sync-client/dist/types/persistence/database";
+import type { StoredDatabase } from "sync-client";
 import { assert } from "../utils/assert";
 import {
 	type RelativePath,
@@ -23,9 +23,11 @@ export class MockClient implements FileSystemOperations {
 	};
 
 	public constructor(
-		private readonly initialSettings: Partial<SyncSettings>,
+		initialSettings: Partial<SyncSettings>,
 		protected readonly useSlowFileEvents: boolean
-	) {}
+	) {
+		this.data.settings = initialSettings;
+	}
 
 	public async init(
 		fetchImplementation: typeof globalThis.fetch
@@ -39,16 +41,6 @@ export class MockClient implements FileSystemOperations {
 			fetch: fetchImplementation
 		});
 
-		await Promise.all(
-			Object.keys(this.initialSettings).map(async (key) => {
-				const settingKey = key as keyof SyncSettings; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
-				return this.client.setSetting(
-					settingKey,
-					this.initialSettings[settingKey]! // eslint-disable-line @typescript-eslint/no-non-null-assertion
-				);
-			})
-		);
-
 		await this.client.start();
 	}
 

frontend/test-client/tsconfig.json

@@ -5,8 +5,13 @@
 		"target": "ES2022",
 		"module": "CommonJS",
 		"esModuleInterop": true,
-		"lib": ["DOM", "ESNext"],
+		"lib": [
+			"DOM",
+			"ESNext"
+		],
 		"moduleResolution": "node"
 	},
-	"exclude": ["./dist"]
+	"exclude": [
-}
+		"./dist"
+	]
+}