andras/reconcile
1
0
Fork 0
Code Issues Pull requests 6 Projects Releases Packages Wiki Activity Actions

Fix syncing when network latency is present (#4)

Browse source 
* WIP

* Add debug

* Dedupe inserts

* Add deterministic ordering

* Fix whitespaces

* Update insta

* Add integration test script

* Rename

* Add test

* Working for non-deletes

* omg it mostly works for deletes

* Isdeleted fix

* remove created dates

* update api

* Take document id

* No max attempt

* works

* Use string uuids

* .

* working!!!! (hopefully)

* Improve bundling

* Add module

* lint

* .

* lint

* Fix CI

* use toolchain

* clean up

* Add useSlowFileEvents

* Delete fuzz

* Fix CI

* use docker

* fix script

* clean up

* Clean up

* change node version

* Build docker image on every commit

* fix ci

* 1 db per vault

* Add scritps folder

* Bump versions

* Lint

* .

* Fix tests for real

* Style

* .

* try

* Consistent ordering

* Fix tests

* hmm

* .

* Clean up diff

* Fixes

* .

* Fix version bump

* .

* .

* .
This commit is contained in:
Andras Schmelczer 2025-03-16 20:13:49 +00:00 • committed by GitHub
parent bcf48c428d
commit 8b8f1d91d9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
91 changed files with 2252 additions and 1586 deletions
Download patch file Download diff file Expand all files Collapse all files

16
backend/sync_server/src/database/migrations/20241207143519_bootstrap.sql
View file

@ -1,25 +1,21 @@
CREATE TABLE IF NOT EXISTS documents (
vault_id TEXT NOT NULL,
vault_update_id INTEGER NOT NULL,
vault_update_id INTEGER NOT NULL PRIMARY KEY,
document_id TEXT NOT NULL,
relative_path TEXT NOT NULL,
created_date TIMESTAMP NOT NULL,
updated_date TIMESTAMP NOT NULL,
content BLOB NOT NULL,
is_deleted BOOLEAN NOT NULL,
PRIMARY KEY (vault_id, vault_update_id)
is_deleted BOOLEAN NOT NULL
);
CREATE VIEW IF NOT EXISTS latest_document_versions AS
SELECT d.*
FROM documents d
INNER JOIN (
SELECT vault_id, MAX(vault_update_id) AS max_version_id
SELECT MAX(vault_update_id) AS max_version_id
FROM documents
GROUP BY vault_id, document_id
GROUP BY document_id
) max_versions
ON d.vault_id = max_versions.vault_id
AND d.vault_update_id = max_versions.max_version_id;
ON d.vault_update_id = max_versions.max_version_id;
CREATE INDEX IF NOT EXISTS idx_documents_vault_id_relative_path
ON documents (vault_id, relative_path);
ON documents (relative_path);

14
backend/sync_server/src/database/models.rs
View file

@ -9,30 +9,24 @@ pub type DocumentId = uuid::Uuid;
#[derive(Debug, Clone)]
pub struct StoredDocumentVersion {
pub vault_id: VaultId,
pub vault_update_id: VaultUpdateId,
pub document_id: DocumentId,
pub relative_path: String,
pub created_date: DateTime<Utc>,
pub updated_date: DateTime<Utc>,
pub content: Vec<u8>,
pub is_deleted: bool,
}
impl PartialEq<Self> for StoredDocumentVersion {
fn eq(&self, other: &Self) -> bool {
self.vault_id == other.vault_id && self.vault_update_id == other.vault_update_id
}
fn eq(&self, other: &Self) -> bool { self.vault_update_id == other.vault_update_id }
}
#[derive(Debug, Clone, Serialize, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct DocumentVersionWithoutContent {
pub vault_id: VaultId,
pub vault_update_id: VaultUpdateId,
pub document_id: DocumentId,
pub relative_path: String,
pub created_date: DateTime<Utc>,
pub updated_date: DateTime<Utc>,
pub is_deleted: bool,
}
@ -40,11 +34,9 @@ pub struct DocumentVersionWithoutContent {
impl From<StoredDocumentVersion> for DocumentVersionWithoutContent {
fn from(value: StoredDocumentVersion) -> Self {
Self {
vault_id: value.vault_id,
vault_update_id: value.vault_update_id,
document_id: value.document_id,
relative_path: value.relative_path,
created_date: value.created_date,
updated_date: value.updated_date,
is_deleted: value.is_deleted,
}
@ -54,11 +46,9 @@ impl From<StoredDocumentVersion> for DocumentVersionWithoutContent {
#[derive(Debug, Clone, Serialize, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct DocumentVersion {
pub vault_id: VaultId,
pub vault_update_id: VaultUpdateId,
pub document_id: DocumentId,
pub relative_path: String,
pub created_date: DateTime<Utc>,
pub updated_date: DateTime<Utc>,
pub content_base64: String,
pub is_deleted: bool,
@ -67,11 +57,9 @@ pub struct DocumentVersion {
impl From<StoredDocumentVersion> for DocumentVersion {
fn from(value: StoredDocumentVersion) -> Self {
Self {
vault_id: value.vault_id,
vault_update_id: value.vault_update_id,
document_id: value.document_id,
relative_path: value.relative_path,
created_date: value.created_date,
updated_date: value.updated_date,
content_base64: bytes_to_base64(&value.content),
is_deleted: value.is_deleted,