This commit is contained in:
Andras Schmelczer 2025-07-12 12:59:26 +01:00
parent fc19e650ca
commit 0f441e38d8
No known key found for this signature in database
GPG key ID: FC8F2C3D3D1A718C
24 changed files with 553 additions and 68 deletions

View file

@ -25,29 +25,22 @@ jobs:
- name: Setup rust
run: |
cargo install sqlx-cli wasm-pack cargo-machete
cd backend
cargo install sqlx-cli cargo-machete
cd sync-server
sqlx database create --database-url sqlite://db.sqlite3
sqlx migrate run --source sync_server/src/app_state/database/migrations --database-url sqlite://db.sqlite3
- name: Build wasm
- name: Lint sync-server
run: |
cd backend
wasm-pack build --target web sync_lib
- name: Lint backend
run: |
cd backend
cd sync-server
cargo clippy --all-targets --all-features
cargo fmt --all -- --check
cargo machete
- name: Test backend
- name: Test sync-server
run: |
cd backend
cd sync-server
cargo test --verbose -- --include-ignored
cd sync_lib
wasm-pack test --node
- name: Lint frontend
run: |

View file

@ -25,20 +25,15 @@ jobs:
- name: Setup rust
run: |
cargo install sqlx-cli wasm-pack
cd backend
cargo install sqlx-cli
cd sync-server
sqlx database create --database-url sqlite://db.sqlite3
sqlx migrate run --source sync_server/src/app_state/database/migrations --database-url sqlite://db.sqlite3
- name: Build wasm
run: |
cd backend
wasm-pack build --target web sync_lib
- name: E2E tests
run: |
cd backend
cargo run -p sync_server config-e2e.yml --color never &
cd sync-server
cargo run config-e2e.yml --color never &
cd ..
scripts/update-api-types.sh

View file

@ -66,7 +66,7 @@ jobs:
id: build-and-push
uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
with:
context: backend
context: sync-server
push: ${{ github.ref_type == 'tag' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}

View file

@ -20,12 +20,6 @@ jobs:
node-version: "22.x"
check-latest: true
- name: Build wasm
run: |
cd backend
cargo install wasm-pack
wasm-pack build --target web sync_lib
- name: Build plugin
run: |
cd frontend

12
.gitignore vendored
View file

@ -4,15 +4,17 @@ node_modules
# Exclude macOS Finder (System Explorer) View States
.DS_Store
# Rust build folder
backend/target
# Frontend build folders
frontend/*/dist
backend/db.sqlite3*
backend/databases
backend/sync_server/bindings/*.ts
sync-server/db.sqlite3*
sync-server/databases
# Rust build folders
sync-server/target
sync-server/bindings/*.ts
*.log
*.sqlx

View file

@ -56,4 +56,4 @@ And to clean up the logs & database files, run `scripts/clean-up.sh`
## Projects
- [Sync server](./backend/sync_server/README.md)
- [Sync server](./sync-server/README.md)

View file

@ -0,0 +1,502 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "addr2line"
version = "0.24.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1"
dependencies = [
"compiler_builtins",
"gimli",
"rustc-std-workspace-alloc",
"rustc-std-workspace-core",
]
[[package]]
name = "adler2"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627"
dependencies = [
"compiler_builtins",
"rustc-std-workspace-core",
]
[[package]]
name = "alloc"
version = "0.0.0"
dependencies = [
"compiler_builtins",
"core",
]
[[package]]
name = "alloctests"
version = "0.0.0"
dependencies = [
"rand",
"rand_xorshift",
]
[[package]]
name = "cc"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1aeb932158bd710538c73702db6945cb68a8fb08c519e6e12706b94263b36db8"
dependencies = [
"shlex",
]
[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
dependencies = [
"compiler_builtins",
"rustc-std-workspace-core",
]
[[package]]
name = "compiler_builtins"
version = "0.1.160"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6376049cfa92c0aa8b9ac95fae22184b981c658208d4ed8a1dc553cd83612895"
dependencies = [
"cc",
"rustc-std-workspace-core",
]
[[package]]
name = "core"
version = "0.0.0"
[[package]]
name = "coretests"
version = "0.0.0"
dependencies = [
"rand",
"rand_xorshift",
]
[[package]]
name = "dlmalloc"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8cff88b751e7a276c4ab0e222c3f355190adc6dde9ce39c851db39da34990df7"
dependencies = [
"cfg-if",
"compiler_builtins",
"libc",
"rustc-std-workspace-core",
"windows-sys",
]
[[package]]
name = "fortanix-sgx-abi"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57cafc2274c10fab234f176b25903ce17e690fca7597090d50880e047a0389c5"
dependencies = [
"compiler_builtins",
"rustc-std-workspace-core",
]
[[package]]
name = "getopts"
version = "0.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5"
dependencies = [
"rustc-std-workspace-core",
"rustc-std-workspace-std",
"unicode-width",
]
[[package]]
name = "gimli"
version = "0.31.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f"
dependencies = [
"compiler_builtins",
"rustc-std-workspace-alloc",
"rustc-std-workspace-core",
]
[[package]]
name = "hashbrown"
version = "0.15.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "84b26c544d002229e640969970a2e74021aadf6e2f96372b9c58eff97de08eb3"
dependencies = [
"compiler_builtins",
"rustc-std-workspace-alloc",
"rustc-std-workspace-core",
]
[[package]]
name = "hermit-abi"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f154ce46856750ed433c8649605bf7ed2de3bc35fd9d2a9f30cddd873c80cb08"
dependencies = [
"compiler_builtins",
"rustc-std-workspace-alloc",
"rustc-std-workspace-core",
]
[[package]]
name = "libc"
version = "0.2.172"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa"
dependencies = [
"rustc-std-workspace-core",
]
[[package]]
name = "memchr"
version = "2.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
dependencies = [
"compiler_builtins",
"rustc-std-workspace-core",
]
[[package]]
name = "miniz_oxide"
version = "0.8.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a"
dependencies = [
"adler2",
"compiler_builtins",
"rustc-std-workspace-alloc",
"rustc-std-workspace-core",
]
[[package]]
name = "object"
version = "0.36.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87"
dependencies = [
"compiler_builtins",
"memchr",
"rustc-std-workspace-alloc",
"rustc-std-workspace-core",
]
[[package]]
name = "panic_abort"
version = "0.0.0"
dependencies = [
"alloc",
"compiler_builtins",
"core",
"libc",
]
[[package]]
name = "panic_unwind"
version = "0.0.0"
dependencies = [
"alloc",
"cfg-if",
"compiler_builtins",
"core",
"libc",
"unwind",
]
[[package]]
name = "proc_macro"
version = "0.0.0"
dependencies = [
"core",
"rustc-literal-escaper",
"std",
]
[[package]]
name = "profiler_builtins"
version = "0.0.0"
dependencies = [
"cc",
]
[[package]]
name = "r-efi"
version = "5.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5"
dependencies = [
"compiler_builtins",
"rustc-std-workspace-core",
]
[[package]]
name = "r-efi-alloc"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e43c53ff1a01d423d1cb762fd991de07d32965ff0ca2e4f80444ac7804198203"
dependencies = [
"compiler_builtins",
"r-efi",
"rustc-std-workspace-core",
]
[[package]]
name = "rand"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97"
dependencies = [
"rand_core",
]
[[package]]
name = "rand_core"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38"
[[package]]
name = "rand_xorshift"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a"
dependencies = [
"rand_core",
]
[[package]]
name = "rustc-demangle"
version = "0.1.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
dependencies = [
"compiler_builtins",
"rustc-std-workspace-core",
]
[[package]]
name = "rustc-literal-escaper"
version = "0.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0041b6238913c41fe704213a4a9329e2f685a156d1781998128b4149c230ad04"
dependencies = [
"rustc-std-workspace-std",
]
[[package]]
name = "rustc-std-workspace-alloc"
version = "1.99.0"
dependencies = [
"alloc",
]
[[package]]
name = "rustc-std-workspace-core"
version = "1.99.0"
dependencies = [
"core",
]
[[package]]
name = "rustc-std-workspace-std"
version = "1.99.0"
dependencies = [
"std",
]
[[package]]
name = "shlex"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
[[package]]
name = "std"
version = "0.0.0"
dependencies = [
"addr2line",
"alloc",
"cfg-if",
"compiler_builtins",
"core",
"dlmalloc",
"fortanix-sgx-abi",
"hashbrown",
"hermit-abi",
"libc",
"miniz_oxide",
"object",
"panic_abort",
"panic_unwind",
"r-efi",
"r-efi-alloc",
"rand",
"rand_xorshift",
"rustc-demangle",
"std_detect",
"unwind",
"wasi",
"windows-targets 0.0.0",
]
[[package]]
name = "std_detect"
version = "0.1.5"
dependencies = [
"cfg-if",
"compiler_builtins",
"libc",
"rustc-std-workspace-alloc",
"rustc-std-workspace-core",
]
[[package]]
name = "sysroot"
version = "0.0.0"
dependencies = [
"proc_macro",
"profiler_builtins",
"std",
"test",
]
[[package]]
name = "test"
version = "0.0.0"
dependencies = [
"core",
"getopts",
"libc",
"std",
]
[[package]]
name = "unicode-width"
version = "0.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"
dependencies = [
"compiler_builtins",
"rustc-std-workspace-core",
"rustc-std-workspace-std",
]
[[package]]
name = "unwind"
version = "0.0.0"
dependencies = [
"cfg-if",
"compiler_builtins",
"core",
"libc",
"unwinding",
]
[[package]]
name = "unwinding"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8393f2782b6060a807337ff353780c1ca15206f9ba2424df18cb6e733bd7b345"
dependencies = [
"compiler_builtins",
"gimli",
"rustc-std-workspace-core",
]
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
dependencies = [
"compiler_builtins",
"rustc-std-workspace-alloc",
"rustc-std-workspace-core",
]
[[package]]
name = "windows-sys"
version = "0.59.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
dependencies = [
"windows-targets 0.52.6",
]
[[package]]
name = "windows-targets"
version = "0.0.0"
[[package]]
name = "windows-targets"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
dependencies = [
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_gnullvm",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc",
]
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
[[package]]
name = "windows_aarch64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
[[package]]
name = "windows_i686_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
[[package]]
name = "windows_i686_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
[[package]]
name = "windows_i686_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
[[package]]
name = "windows_x86_64_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
[[package]]
name = "windows_x86_64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"

View file

@ -7,8 +7,7 @@ import { FileOperations } from "./file-operations";
import { Logger } from "../tracing/logger";
import { assertSetContainsExactly } from "../utils/assert-set-contains-exactly";
import type { FileSystemOperations } from "./filesystem-operations";
import fs from "fs";
import { TextWithCursors } from "reconcile-text";
import type { TextWithCursors } from "reconcile-text";
class MockDatabase implements Partial<Database> {
public getLatestDocumentByRelativePath(

View file

@ -4,6 +4,7 @@ import type { Database, RelativePath } from "../persistence/database";
import { SafeFileSystemOperations } from "./safe-filesystem-operations";
import type { TextWithCursors } from "reconcile-text";
import { isBinary, reconcile } from "reconcile-text";
import { isFileTypeMergable } from "../utils/is-file-type-mergable";
export class FileOperations {
private static readonly PARENTHESES_REGEX = / \((\d+)\)$/;
private readonly fs: SafeFileSystemOperations;

View file

@ -3,7 +3,7 @@ import type { FileSystemOperations } from "./filesystem-operations";
import type { Logger } from "../tracing/logger";
import { Locks } from "../utils/locks";
import { FileNotFoundError } from "./file-not-found-error";
import { TextWithCursors } from "reconcile-text";
import type { TextWithCursors } from "reconcile-text";
/**
* Decorates `FileSystemOperations` to replace errors with `FileNotFoundError`

View file

@ -77,11 +77,6 @@ export class SyncClient {
const history = new SyncHistory(logger);
await initWasm(
// eslint-disable-next-line
(wasmBin as any).default // it is loaded as a base64 string by webpack
);
let state = (await persistence.load()) ?? {
settings: undefined,
database: undefined

View file

@ -25,4 +25,4 @@ describe("isFileTypeMergable", () => {
expect(isFileTypeMergable("HELLO.JSON")).toBe(false);
expect(isFileTypeMergable("my/config.yml")).toBe(false);
});
});
});

View file

@ -1,6 +1,6 @@
export function isFileTypeMergable(pathOrFileName: string): boolean {
const parts = pathOrFileName.split(".");
const fileExtension = parts.at(-1) || "";
const fileExtension = parts.at(-1) ?? "";
return ["md", "txt"].includes(fileExtension.toLowerCase());
}

View file

@ -33,7 +33,7 @@ async function runTest({
console.info(`Using vault name: ${vaultName}`);
const initialSettings: Partial<SyncSettings> = {
isSyncEnabled: true,
token: " test-token-change-me ", // same as in backend/config-e2e.yml with spaces
token: " test-token-change-me ", // same as in sync-server/config-e2e.yml with spaces
vaultName: randomCasing(vaultName) + (Math.random() > 0.5 ? " " : ""), // extra spaces shouldn't matter
syncConcurrency: concurrency,
remoteUri: "http://localhost:3000"

View file

@ -22,22 +22,14 @@ else
echo "Your working directory is clean."
fi
echo "Bumping backend versions"
cd backend
echo "Bumping sync-server versions"
cd sync-server
cargo set-version --bump $1
echo "Bumping frontend versions"
cd ../frontend
npm version $1 --workspaces
echo "Updating frontend dependencies to match the new backend versions"
cd ../backend/sync_lib
wasm-pack build --target web --features console_error_panic_hook
cd ../../frontend
npm install
cd ..
cp frontend/obsidian-plugin/manifest.json manifest.json # for BRAT, otherwise it wouldn't update
# Commit and tag

View file

@ -1,4 +1,4 @@
#!/bin/bash
rm -rf backend/databases
rm -rf sync-server/databases
rm -rf logs

View file

@ -2,10 +2,10 @@
set -e
rm -rf backend/sync_server/bindings
rm -rf sync-server/bindings
cd backend
cd sync-server
cargo test export_bindings
cd -
cp -r backend/sync_server/bindings/* frontend/sync-client/src/services/types/
cp -r sync-server/bindings/* frontend/sync-client/src/services/types/

View file

@ -1,6 +1,6 @@
[package]
name = "sync_server"
rust-version = "1.83"
rust-version = "1.87.0"
authors = ["Andras Schmelczer <andras@schmelczer.dev>"]
edition = "2024"
license = "MIT"

View file

@ -124,5 +124,7 @@ impl ClientCursorsWithTimeToLive {
}
}
pub fn is_expired(&self, ttl: Duration) -> bool { self.last_updated.elapsed() > ttl }
pub fn is_expired(&self, ttl: Duration) -> bool {
self.last_updated.elapsed() > ttl
}
}

View file

@ -23,7 +23,9 @@ pub struct StoredDocumentVersion {
}
impl PartialEq<Self> for StoredDocumentVersion {
fn eq(&self, other: &Self) -> bool { self.vault_update_id == other.vault_update_id }
fn eq(&self, other: &Self) -> bool {
self.vault_update_id == other.vault_update_id
}
}
#[derive(TS, Debug, Clone, Serialize)]

View file

@ -179,6 +179,10 @@ async fn shutdown_signal() {
}
}
async fn handle_404() -> impl IntoResponse { not_found_error(anyhow!("Page not found")) }
async fn handle_404() -> impl IntoResponse {
not_found_error(anyhow!("Page not found"))
}
async fn handle_405() -> impl IntoResponse { client_error(anyhow!("Method not allowed")) }
async fn handle_405() -> impl IntoResponse {
client_error(anyhow!("Method not allowed"))
}

View file

@ -6,7 +6,9 @@ pub struct DeviceIdHeader(pub String);
pub static DEVICE_ID_HEADER_NAME: HeaderName = HeaderName::from_static("device-id");
impl Header for DeviceIdHeader {
fn name() -> &'static HeaderName { &DEVICE_ID_HEADER_NAME }
fn name() -> &'static HeaderName {
&DEVICE_ID_HEADER_NAME
}
fn decode<'i, I>(values: &mut I) -> Result<Self, headers::Error>
where

View file

@ -126,7 +126,7 @@ pub async fn update_document(
&& is_binary(&content)
{
reconcile(
&str::from_utf8(&parent_document.content)
str::from_utf8(&parent_document.content)
.expect("parent must be valid UTF-8 because it's not binary"),
&str::from_utf8(&latest_version.content)
.expect("latest_version must be valid UTF-8 because it's not binary")

View file

@ -8,4 +8,6 @@ where
Ok(normalize_string(&s))
}
pub fn normalize_string(s: &str) -> String { s.trim().to_lowercase() }
pub fn normalize_string(s: &str) -> String {
s.trim().to_lowercase()
}