andras/vault-link
1
0
Fork 0
Code Issues Pull requests 18 Projects Releases 55 Packages Wiki Activity Actions 1

Fix syncing when network latency is present (#4)

Browse source 
* WIP

* Add debug

* Dedupe inserts

* Add deterministic ordering

* Fix whitespaces

* Update insta

* Add integration test script

* Rename

* Add test

* Working for non-deletes

* omg it mostly works for deletes

* Isdeleted fix

* remove created dates

* update api

* Take document id

* No max attempt

* works

* Use string uuids

* .

* working!!!! (hopefully)

* Improve bundling

* Add module

* lint

* .

* lint

* Fix CI

* use toolchain

* clean up

* Add useSlowFileEvents

* Delete fuzz

* Fix CI

* use docker

* fix script

* clean up

* Clean up

* change node version

* Build docker image on every commit

* fix ci

* 1 db per vault

* Add scritps folder

* Bump versions

* Lint

* .

* Fix tests for real

* Style

* .

* try

* Consistent ordering

* Fix tests

* hmm

* .

* Clean up diff

* Fixes

* .

* Fix version bump

* .

* .

* .
This commit is contained in:
Andras Schmelczer 2025-03-16 20:13:49 +00:00 • committed by GitHub
parent bcf48c428d
commit 8b8f1d91d9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
91 changed files with 2252 additions and 1586 deletions
Download patch file Download diff file Expand all files Collapse all files

38
backend/sync_server/src/server/create_document.rs
View file

@ -6,7 +6,6 @@ use axum_extra::{
headers::{Authorization, authorization::Bearer},
};
use axum_jsonschema::Json;
use chrono::{DateTime, Utc};
use schemars::JsonSchema;
use serde::Deserialize;
use sync_lib::base64_to_bytes;
@ -17,7 +16,7 @@ use super::{
requests::{CreateDocumentVersion, CreateDocumentVersionMultipart},
};
use crate::{
database::models::{DocumentVersionWithoutContent, StoredDocumentVersion, VaultId},
database::models::{DocumentId, DocumentVersionWithoutContent, StoredDocumentVersion, VaultId},
errors::{SyncServerError, client_error, server_error},
utils::sanitize_path,
};
@ -44,8 +43,8 @@ pub async fn create_document_multipart(
auth_header,
state,
vault_id,
request.document_id,
request.relative_path,
request.created_date,
request.content.contents.to_vec(),
)
.await
@ -69,8 +68,8 @@ pub async fn create_document_json(
auth_header,
state,
vault_id,
request.document_id,
request.relative_path,
request.created_date,
content_bytes,
)
.await
@ -78,20 +77,39 @@ pub async fn create_document_json(
async fn internal_create_document(
auth_header: Authorization<Bearer>,
state: AppState,
mut state: AppState,
vault_id: VaultId,
document_id: Option<DocumentId>,
relative_path: String,
created_date: DateTime<Utc>,
content: Vec<u8>,
) -> Result<Json<DocumentVersionWithoutContent>, SyncServerError> {
auth(&state, auth_header.token())?;
let mut transaction = state
.database
.create_write_transaction()
.create_write_transaction(&vault_id)
.await
.map_err(server_error)?;
let document_id = match document_id {
Some(document_id) => {
let existing_version = state
.database
.get_latest_document(&vault_id, &document_id, Some(&mut transaction))
.await
.map_err(server_error)?;
if existing_version.is_some() {
return Err(client_error(anyhow::anyhow!(
"Document with the same ID already exists"
)));
}
document_id
}
None => uuid::Uuid::new_v4(),
};
let last_update_id = state
.database
.get_max_update_id_in_vault(&vault_id, Some(&mut transaction))
@ -101,19 +119,17 @@ async fn internal_create_document(
let sanitized_relative_path = sanitize_path(&relative_path);
let new_version = StoredDocumentVersion {
vault_id,
vault_update_id: last_update_id + 1,
document_id: uuid::Uuid::new_v4(),
document_id,
relative_path: sanitized_relative_path,
content,
created_date,
updated_date: chrono::Utc::now(),
is_deleted: false,
};
state
.database
.insert_document_version(&new_version, Some(&mut transaction))
.insert_document_version(&vault_id, &new_version, Some(&mut transaction))
.await
.map_err(server_error)?;

14
backend/sync_server/src/server/delete_document.rs
View file

@ -10,7 +10,7 @@ use serde::Deserialize;
use super::{app_state::AppState, auth::auth, requests::DeleteDocumentVersion};
use crate::{
database::models::{DocumentId, StoredDocumentVersion, VaultId},
database::models::{DocumentId, DocumentVersionWithoutContent, StoredDocumentVersion, VaultId},
errors::{SyncServerError, server_error},
utils::sanitize_path,
};
@ -29,14 +29,14 @@ pub async fn delete_document(
vault_id,
document_id,
}): Path<PathParams>,
State(state): State<AppState>,
State(mut state): State<AppState>,
Json(request): Json<DeleteDocumentVersion>,
) -> Result<(), SyncServerError> {
) -> Result<Json<DocumentVersionWithoutContent>, SyncServerError> {
auth(&state, auth_header.token())?;
let mut transaction = state
.database
.create_write_transaction()
.create_write_transaction(&vault_id)
.await
.map_err(server_error)?;
@ -47,19 +47,17 @@ pub async fn delete_document(
.map_err(server_error)?;
let new_version = StoredDocumentVersion {
vault_id,
vault_update_id: last_update_id + 1,
document_id,
relative_path: sanitize_path(&request.relative_path),
content: vec![],
created_date: request.created_date,
updated_date: chrono::Utc::now(),
is_deleted: true,
};
state
.database
.insert_document_version(&new_version, Some(&mut transaction))
.insert_document_version(&vault_id, &new_version, Some(&mut transaction))
.await
.map_err(server_error)?;
@ -69,5 +67,5 @@ pub async fn delete_document(
.context("Failed to commit successful transaction")
.map_err(server_error)?;
Ok(())
Ok(Json(new_version.into()))
}

16
backend/sync_server/src/server/fetch_document_version.rs
View file

@ -30,7 +30,7 @@ pub async fn fetch_document_version(
document_id,
vault_update_id,
}): Path<PathParams>,
State(state): State<AppState>,
State(mut state): State<AppState>,
) -> Result<Json<DocumentVersion>, SyncServerError> {
auth(&state, auth_header.token())?;
@ -39,12 +39,14 @@ pub async fn fetch_document_version(
.get_document_version(&vault_id, vault_update_id, None)
.await
.map_err(server_error)?
.map(Ok)
.unwrap_or_else(|| {
Err(not_found_error(anyhow!(
"Document with vault update id `{vault_update_id}` not found",
)))
})?;
.map_or_else(
|| {
Err(not_found_error(anyhow!(
"Document with vault update id `{vault_update_id}` not found",
)))
},
Ok,
)?;
if result.document_id != document_id {
return Err(not_found_error(anyhow!(

16
backend/sync_server/src/server/fetch_document_version_content.rs
View file

@ -32,7 +32,7 @@ pub async fn fetch_document_version_content(
document_id,
vault_update_id,
}): Path<PathParams>,
State(state): State<AppState>,
State(mut state): State<AppState>,
) -> Result<Bytes, SyncServerError> {
auth(&state, auth_header.token())?;
@ -41,12 +41,14 @@ pub async fn fetch_document_version_content(
.get_document_version(&vault_id, vault_update_id, None)
.await
.map_err(server_error)?
.map(Ok)
.unwrap_or_else(|| {
Err(not_found_error(anyhow!(
"Document with vault update id `{vault_update_id}` not found",
)))
})?;
.map_or_else(
|| {
Err(not_found_error(anyhow!(
"Document with vault update id `{vault_update_id}` not found",
)))
},
Ok,
)?;
if result.document_id != document_id {
return Err(not_found_error(anyhow!(

16
backend/sync_server/src/server/fetch_latest_document_version.rs
View file

@ -28,7 +28,7 @@ pub async fn fetch_latest_document_version(
vault_id,
document_id,
}): Path<PathParams>,
State(state): State<AppState>,
State(mut state): State<AppState>,
) -> Result<Json<DocumentVersion>, SyncServerError> {
auth(&state, auth_header.token())?;
@ -37,12 +37,14 @@ pub async fn fetch_latest_document_version(
.get_latest_document(&vault_id, &document_id, None)
.await
.map_err(server_error)?
.map(Ok)
.unwrap_or_else(|| {
Err(not_found_error(anyhow!(
"Document with id `{document_id}` not found",
)))
})?;
.map_or_else(
|| {
Err(not_found_error(anyhow!(
"Document with id `{document_id}` not found",
)))
},
Ok,
)?;
Ok(Json(latest_version.into()))
}

2
backend/sync_server/src/server/fetch_latest_documents.rs
View file

@ -30,7 +30,7 @@ pub async fn fetch_latest_documents(
TypedHeader(auth_header): TypedHeader<Authorization<Bearer>>,
Path(PathParams { vault_id }): Path<PathParams>,
Query(QueryParams { since_update_id }): Query<QueryParams>,
State(state): State<AppState>,
State(mut state): State<AppState>,
) -> Result<Json<FetchLatestDocumentsResponse>, SyncServerError> {
auth(&state, auth_header.token())?;

14
backend/sync_server/src/server/requests.rs
View file

@ -1,24 +1,27 @@
use aide_axum_typed_multipart::FieldData;
use axum::body::Bytes;
use axum_typed_multipart::TryFromMultipart;
use chrono::{DateTime, Utc};
use schemars::JsonSchema;
use serde::{self, Deserialize};
use crate::database::models::VaultUpdateId;
use crate::database::models::{DocumentId, VaultUpdateId};
#[derive(Debug, Deserialize, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct CreateDocumentVersion {
/// The client can decide the document id (if it wishes to) in order
/// to help with syncing. If the client does not provide a document id,
/// the server will generate one. If the client provides a document id
/// it must not already exist in the database.
pub document_id: Option<DocumentId>,
pub relative_path: String,
pub created_date: DateTime<Utc>,
pub content_base64: String,
}
#[derive(Debug, TryFromMultipart, JsonSchema)]
pub struct CreateDocumentVersionMultipart {
pub document_id: Option<DocumentId>,
pub relative_path: String,
pub created_date: DateTime<Utc>,
#[form_data(limit = "unlimited")]
pub content: FieldData<Bytes>,
}
@ -28,7 +31,6 @@ pub struct CreateDocumentVersionMultipart {
pub struct UpdateDocumentVersion {
pub parent_version_id: VaultUpdateId,
pub relative_path: String,
pub created_date: DateTime<Utc>,
pub content_base64: String,
}
@ -37,7 +39,6 @@ pub struct UpdateDocumentVersion {
pub struct UpdateDocumentVersionMultipart {
pub parent_version_id: VaultUpdateId,
pub relative_path: String,
pub created_date: DateTime<Utc>,
#[form_data(limit = "unlimited")]
pub content: FieldData<Bytes>,
}
@ -46,5 +47,4 @@ pub struct UpdateDocumentVersionMultipart {
#[serde(rename_all = "camelCase")]
pub struct DeleteDocumentVersion {
pub relative_path: String,
pub created_date: DateTime<Utc>,
}

30
backend/sync_server/src/server/update_document.rs
View file

@ -6,7 +6,6 @@ use axum_extra::{
headers::{Authorization, authorization::Bearer},
};
use axum_jsonschema::Json;
use chrono::{DateTime, Utc};
use log::info;
use schemars::JsonSchema;
use serde::Deserialize;
@ -50,7 +49,6 @@ pub async fn update_document_multipart(
document_id,
request.parent_version_id,
request.relative_path,
request.created_date,
request.content.contents.to_vec(),
)
.await
@ -77,21 +75,19 @@ pub async fn update_document_json(
document_id,
request.parent_version_id,
request.relative_path,
request.created_date,
content_bytes,
)
.await
}
#[allow(clippy::too_many_arguments)]
#[allow(clippy::too_many_arguments, clippy::too_many_lines)]
async fn internal_update_document(
auth_header: Authorization<Bearer>,
state: AppState,
mut state: AppState,
vault_id: VaultId,
document_id: DocumentId,
parent_version_id: VaultUpdateId,
relative_path: String,
created_date: DateTime<Utc>,
content: Vec<u8>,
) -> Result<Json<DocumentUpdateResponse>, SyncServerError> {
auth(&state, auth_header.token())?;
@ -114,7 +110,7 @@ async fn internal_update_document(
let mut transaction = state
.database
.create_write_transaction()
.create_write_transaction(&vault_id)
.await
.map_err(server_error)?;
@ -138,6 +134,18 @@ async fn internal_update_document(
Ok,
)?;
if latest_version.is_deleted {
transaction
.rollback()
.await
.context("Failed to roll back transaction")
.map_err(server_error)?;
return Ok(Json(DocumentUpdateResponse::FastForwardUpdate(
latest_version.into(),
)));
}
let sanitized_relative_path = sanitize_path(&relative_path);
// Return the latest version if the content and path are the same as the latest
@ -168,7 +176,7 @@ async fn internal_update_document(
let new_relative_path = if parent_document.relative_path == latest_version.relative_path
&& latest_version.relative_path != sanitized_relative_path
{
let mut new_relative_path = Default::default();
let mut new_relative_path = String::default();
for candidate in deduped_file_paths(&sanitized_relative_path) {
if state
.database
@ -188,19 +196,17 @@ async fn internal_update_document(
};
let new_version = StoredDocumentVersion {
vault_id,
document_id,
vault_update_id: last_update_id + 1,
relative_path: new_relative_path,
content: merged_content,
created_date,
updated_date: chrono::Utc::now(),
is_deleted: latest_version.is_deleted,
is_deleted: false,
};
state
.database
.insert_document_version(&new_version, Some(&mut transaction))
.insert_document_version(&vault_id, &new_version, Some(&mut transaction))
.await
.map_err(server_error)?;