From a2522ca44a52a68511d084cfd0ae071d79f85d8a Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 2 Mar 2025 10:25:38 +0000 Subject: [PATCH 01/58] WIP --- .../sync_server/src/server/delete_document.rs | 6 +- .../sync_server/src/server/update_document.rs | 14 +- .../file-operations/file-operations.test.ts | 14 +- .../src/file-operations/file-operations.ts | 61 +- .../file-operations/filesystem-operations.ts | 2 - .../safe-filesystem-operations.ts | 114 +- .../sync-client/src/persistence/database.ts | 108 +- .../sync-client/src/services/sync-service.ts | 4 +- frontend/sync-client/src/services/types.ts | 1199 ++++++++--------- frontend/sync-client/src/sync-client.ts | 2 +- .../sync-client/src/sync-operations/syncer.ts | 173 ++- .../sync-operations/unrestricted-syncer.ts | 633 ++++----- frontend/test-client/src/agent/mock-agent.ts | 39 +- frontend/test-client/src/cli.ts | 41 +- 14 files changed, 1370 insertions(+), 1040 deletions(-) diff --git a/backend/sync_server/src/server/delete_document.rs b/backend/sync_server/src/server/delete_document.rs index a9d307b5..afef37a7 100644 --- a/backend/sync_server/src/server/delete_document.rs +++ b/backend/sync_server/src/server/delete_document.rs @@ -10,7 +10,7 @@ use serde::Deserialize; use super::{app_state::AppState, auth::auth, requests::DeleteDocumentVersion}; use crate::{ - database::models::{DocumentId, StoredDocumentVersion, VaultId}, + database::models::{DocumentId, DocumentVersionWithoutContent, StoredDocumentVersion, VaultId}, errors::{SyncServerError, server_error}, utils::sanitize_path, }; @@ -31,7 +31,7 @@ pub async fn delete_document( }): Path, State(state): State, Json(request): Json, -) -> Result<(), SyncServerError> { +) -> Result, SyncServerError> { auth(&state, auth_header.token())?; let mut transaction = state @@ -69,5 +69,5 @@ pub async fn delete_document( .context("Failed to commit successful transaction") .map_err(server_error)?; - Ok(()) + Ok(Json(new_version.into())) } diff --git a/backend/sync_server/src/server/update_document.rs b/backend/sync_server/src/server/update_document.rs index 414180bf..17a647ae 100644 --- a/backend/sync_server/src/server/update_document.rs +++ b/backend/sync_server/src/server/update_document.rs @@ -138,6 +138,18 @@ async fn internal_update_document( Ok, )?; + if latest_version.is_deleted { + transaction + .rollback() + .await + .context("Failed to roll back transaction") + .map_err(server_error)?; + + return Ok(Json(DocumentUpdateResponse::FastForwardUpdate( + latest_version.into(), + ))); + } + let sanitized_relative_path = sanitize_path(&relative_path); // Return the latest version if the content and path are the same as the latest @@ -195,7 +207,7 @@ async fn internal_update_document( content: merged_content, created_date, updated_date: chrono::Utc::now(), - is_deleted: latest_version.is_deleted, + is_deleted: false, }; state diff --git a/frontend/sync-client/src/file-operations/file-operations.test.ts b/frontend/sync-client/src/file-operations/file-operations.test.ts index 2e7c57b7..9d2945d5 100644 --- a/frontend/sync-client/src/file-operations/file-operations.test.ts +++ b/frontend/sync-client/src/file-operations/file-operations.test.ts @@ -1,17 +1,27 @@ import type { FileSystemOperations } from "sync-client"; -import type { Database, RelativePath } from "../persistence/database"; +import type { + Database, + DocumentMetadata, + RelativePath +} from "../persistence/database"; import { FileOperations } from "./file-operations"; import { Logger } from "../tracing/logger"; import { assertSetContainsExactly } from "../utils/assert-set-contains-exactly"; describe("File operations", () => { class MockDatabase { - public async updatePath( + public async move( _oldRelativePath: RelativePath, _newRelativePath: RelativePath ): Promise { // this is called but irrelevant for this mock } + + public getResolvedDocument( + _relativePath: RelativePath | undefined + ): DocumentMetadata | undefined { + return undefined; + } } class FakeFileSystemOperations implements FileSystemOperations { diff --git a/frontend/sync-client/src/file-operations/file-operations.ts b/frontend/sync-client/src/file-operations/file-operations.ts index 9977d60b..73818786 100644 --- a/frontend/sync-client/src/file-operations/file-operations.ts +++ b/frontend/sync-client/src/file-operations/file-operations.ts @@ -6,7 +6,10 @@ import type { RelativePath } from "src/persistence/database"; import { isBinary, isFileTypeMergable, mergeText } from "sync_lib"; -import { SafeFileSystemOperations } from "./safe-filesystem-operations"; +import { + FileNotFoundError, + SafeFileSystemOperations +} from "./safe-filesystem-operations"; export class FileOperations { private static readonly PARENTHESES_REGEX = / \((\d+)\)$/; @@ -17,7 +20,7 @@ export class FileOperations { private readonly database: Database, fs: FileSystemOperations ) { - this.fs = new SafeFileSystemOperations(fs); + this.fs = new SafeFileSystemOperations(fs, logger); } public async listAllFiles(): Promise { @@ -58,15 +61,37 @@ export class FileOperations { // All parent directories are created if they don't exist. public async create( path: RelativePath, - newContent: Uint8Array + newContent: Uint8Array, + documentId?: DocumentId ): Promise { + this.logger.debug(`Creating file: ${path}`); if (await this.fs.exists(path)) { const deconflictedPath = await this.deconflictPath(path); this.logger.debug( `Didn't expect ${path} to exist, deconflicting by moving it to '${deconflictedPath}'` ); - await this.database.updatePath(path, deconflictedPath); - await this.fs.rename(path, deconflictedPath); + + const existingMetadata = this.database.getResolvedDocument(path); + this.logger.debug( + `Existing metadata for ${path}: ${JSON.stringify(existingMetadata)}` + ); + if ( + existingMetadata === undefined || + existingMetadata.isDeleted || + existingMetadata.documentId !== documentId || + !documentId + ) { + this.logger.debug( + `We need to save what's at ${path} to ${deconflictedPath}` + ); + await this.move(path, deconflictedPath, documentId); + await this.database.move(path, deconflictedPath); + } else { + // This can happen if the document got moved both locally and remotely + // to the same file path. In this case, we shouldn't deconflict, however, + // we also can't overwrite otherwise we'd lose changes. + throw new FileNotFoundError(path); + } } else { await this.createParentDirectories(path); } @@ -126,9 +151,13 @@ export class FileOperations { return new TextEncoder().encode(resultText); } - public async remove(path: RelativePath): Promise { - this.logger.debug(`Deleting file: ${path}`); - return this.fs.delete(path); + public async delete(path: RelativePath): Promise { + if (!(await this.exists(path))) { + this.logger.debug(`Deleting file: ${path}`); + return this.fs.delete(path); + } else { + this.logger.debug(`No need to delete '${path}', it doesn't exist`); + } } public async move( @@ -145,16 +174,20 @@ export class FileOperations { this.logger.debug( `Conflict when moving '${oldPath}' to '${newPath}', the latter already exists, deconflicting by moving it to '${deconflictedPath}'` ); - - const existingMetadata = this.database.getDocument(newPath); + const existingMetadata = this.database.getResolvedDocument(newPath); if ( existingMetadata === undefined || - existingMetadata.documentId !== documentId + existingMetadata.isDeleted || + existingMetadata.documentId !== documentId || + !documentId ) { - await this.database.updatePath(newPath, deconflictedPath); - await this.fs.rename(newPath, deconflictedPath); + await this.move(newPath, deconflictedPath, documentId); + await this.database.move(oldPath, newPath); } else { - await this.database.deleteDocument(newPath); + // This can happen if the document got moved both locally and remotely + // to the same file path. In this case, we shouldn't deconflict, however, + // we also can't overwrite otherwise we'd lose changes. + throw new FileNotFoundError(newPath); } } else { await this.createParentDirectories(newPath); diff --git a/frontend/sync-client/src/file-operations/filesystem-operations.ts b/frontend/sync-client/src/file-operations/filesystem-operations.ts index 9ea577f7..b58d3c23 100644 --- a/frontend/sync-client/src/file-operations/filesystem-operations.ts +++ b/frontend/sync-client/src/file-operations/filesystem-operations.ts @@ -13,7 +13,5 @@ export interface FileSystemOperations { exists: (path: RelativePath) => Promise; createDirectory: (path: RelativePath) => Promise; delete: (path: RelativePath) => Promise; - - // Must be able to handle renaming to a file that already exists rename: (oldPath: RelativePath, newPath: RelativePath) => Promise; } diff --git a/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts b/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts index e493d12f..f1036073 100644 --- a/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts +++ b/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts @@ -1,5 +1,7 @@ -import type { RelativePath } from "src/persistence/database"; +import type { RelativePath } from "../persistence/database"; import type { FileSystemOperations } from "./filesystem-operations"; +import type { Logger } from "../tracing/logger"; +import { DocumentLocks } from "./document-locks"; export class FileNotFoundError extends Error { public constructor(message: string) { @@ -9,71 +11,145 @@ export class FileNotFoundError extends Error { } // Decorate FileSystemOperations replacing errors with FileNotFoundError -// if the accessed file doesn't exist. +// if the accessed file doesn't exist. It also ensures that there's only +// ever a single request in-flight for any one file. export class SafeFileSystemOperations implements FileSystemOperations { - public constructor(private readonly fs: FileSystemOperations) {} + private readonly locks: DocumentLocks; + + public constructor( + private readonly fs: FileSystemOperations, + private readonly logger: Logger + ) { + this.locks = new DocumentLocks(logger); + } public async listAllFiles(): Promise { + this.logger.debug("Listing all files"); return this.fs.listAllFiles(); } public async read(path: RelativePath): Promise { - return this.safeOperation(path, async () => this.fs.read(path)); + this.logger.debug(`Reading file: ${path}`); + return this.safeOperation( + path, + this.decorateToHoldLock(path, async () => this.fs.read(path)), + "read" + ); } public async write(path: RelativePath, content: Uint8Array): Promise { - return this.fs.write(path, content); + this.logger.debug(`Writing file: ${path}`); + return this.decorateToHoldLock(path, async () => + this.fs.write(path, content) + )(); } public async atomicUpdateText( path: RelativePath, updater: (currentContent: string) => string ): Promise { - return this.safeOperation(path, async () => - this.fs.atomicUpdateText(path, updater) + this.logger.debug(`Atomic update of file: ${path}`); + return this.safeOperation( + path, + this.decorateToHoldLock(path, async () => + this.fs.atomicUpdateText(path, updater) + ), + "atomicUpdateText" ); } public async getFileSize(path: RelativePath): Promise { - return this.safeOperation(path, async () => this.fs.getFileSize(path)); + this.logger.debug(`Getting file size: ${path}`); + return this.safeOperation( + path, + this.decorateToHoldLock(path, async () => + this.fs.getFileSize(path) + ), + "getFileSize" + ); } public async getModificationTime(path: RelativePath): Promise { - return this.safeOperation(path, async () => - this.fs.getModificationTime(path) + this.logger.debug(`Getting modification time: ${path}`); + return this.safeOperation( + path, + this.decorateToHoldLock(path, async () => + this.fs.getModificationTime(path) + ), + "getModificationTime" ); } public async exists(path: RelativePath): Promise { - return this.fs.exists(path); + this.logger.debug(`Checking if file exists: ${path}`); + return this.decorateToHoldLock(path, async () => + this.fs.exists(path) + )(); } public async createDirectory(path: RelativePath): Promise { - return this.fs.createDirectory(path); + this.logger.debug(`Creating directory: ${path}`); + return this.decorateToHoldLock(path, async () => + this.fs.createDirectory(path) + )(); } public async delete(path: RelativePath): Promise { - return this.fs.delete(path); + this.logger.debug(`Deleting file: ${path}`); + return this.decorateToHoldLock(path, async () => + this.fs.delete(path) + )(); } public async rename( oldPath: RelativePath, newPath: RelativePath ): Promise { - return this.safeOperation(oldPath, async () => - this.fs.rename(oldPath, newPath) + this.logger.debug(`Renaming file: ${oldPath} to ${newPath}`); + return this.safeOperation( + oldPath, + this.decorateToHoldLock([oldPath, newPath], async () => + this.fs.rename(oldPath, newPath) + ), + "rename" ); } + private decorateToHoldLock( + pathOrPaths: RelativePath | RelativePath[], + operation: () => Promise + ): () => Promise { + return async () => { + const paths = Array.isArray(pathOrPaths) + ? pathOrPaths + : [pathOrPaths]; + await Promise.all( + paths.map(async (path) => this.locks.waitForDocumentLock(path)) + ); + try { + return await operation(); + } finally { + await Promise.all( + paths.map((path) => { + this.locks.unlockDocument(path); + }) + ); + } + }; + } + private async safeOperation( path: RelativePath, - operation: () => Promise + operation: () => Promise, + operationName: string ): Promise { // Without locking the file, this isn't atomic, however, it's good enough practicaly. // This will only break if the file exists, gets deleted and then immediately // recreated while `operation` is running. if (!(await this.fs.exists(path))) { - throw new FileNotFoundError(path); + throw new FileNotFoundError( + `File not found: ${path} before trying to ${operationName}` + ); } try { return await operation(); @@ -81,7 +157,9 @@ export class SafeFileSystemOperations implements FileSystemOperations { if (await this.fs.exists(path)) { throw error; } else { - throw new FileNotFoundError(path); + throw new FileNotFoundError( + `File not found: ${path} when trying to ${operationName}` + ); } } } diff --git a/frontend/sync-client/src/persistence/database.ts b/frontend/sync-client/src/persistence/database.ts index a013d9ac..705f3aea 100644 --- a/frontend/sync-client/src/persistence/database.ts +++ b/frontend/sync-client/src/persistence/database.ts @@ -6,6 +6,7 @@ export interface DocumentMetadata { parentVersionId: VaultUpdateId; documentId: DocumentId; hash: string; + isDeleted: boolean; } import type { Logger } from "src/tracing/logger"; @@ -16,7 +17,10 @@ export interface StoredDatabase { } export class Database { - private documents = new Map(); + private documents = new Map< + RelativePath, + DocumentMetadata | Promise + >(); private lastSeenUpdateId: VaultUpdateId | undefined; @@ -43,8 +47,15 @@ export class Database { ); } - public getDocuments(): Map { - return this.documents; + public get length(): number { + return this.documents.size; + } + + public get resolvedDocuments(): [RelativePath, DocumentMetadata][] { + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + return Array.from(this.documents.entries()).filter( + ([_, metadata]) => !(metadata instanceof Promise) + ) as [RelativePath, DocumentMetadata][]; } public getLastSeenUpdateId(): VaultUpdateId | undefined { @@ -67,58 +78,106 @@ export class Database { public getDocumentByDocumentId( documentId: DocumentId ): [RelativePath, DocumentMetadata] | undefined { - return [...this.documents.entries()].find( + return this.resolvedDocuments.find( ([_, metadata]) => metadata.documentId === documentId ); } + public getDocumentByIdentity( + document: + | DocumentMetadata + | Promise + | undefined + ): + | [ + RelativePath, + DocumentMetadata | Promise + ] + | undefined { + if (document === undefined) { + return undefined; + } + + return Array.from(this.documents.entries()).find( + ([_, metadata]) => metadata === document + ); + } + public async setDocument({ documentId, relativePath, parentVersionId, - hash + hash, + isDeleted }: { documentId: DocumentId; relativePath: RelativePath; parentVersionId: VaultUpdateId; hash: string; + isDeleted: boolean; }): Promise { this.documents.set(relativePath, { documentId, parentVersionId, - hash + hash, + isDeleted }); await this.save(); } - public async removeDocument(relativePath: RelativePath): Promise { - this.documents.delete(relativePath); - await this.save(); + public async setDocumentPromise({ + relativePath, + promise + }: { + relativePath: RelativePath; + promise: Promise; + }): Promise { + this.documents.set(relativePath, promise); + // No need to save as Promises don't get serialized + // and a crash would only result in the document being + // creatied again. + } + + public getResolvedDocument( + relativePath: RelativePath | undefined + ): DocumentMetadata | undefined { + if (relativePath == undefined) { + return undefined; + } + + const metadata = this.documents.get(relativePath); + if (metadata instanceof Promise) { + return undefined; + } + + return metadata; } public getDocument( - relativePath: RelativePath - ): DocumentMetadata | undefined { + relativePath: RelativePath | undefined + ): Promise | DocumentMetadata | undefined { + if (relativePath == undefined) { + return undefined; + } + return this.documents.get(relativePath); } - public async deleteDocument(relativePath: RelativePath): Promise { - this.documents.delete(relativePath); - await this.save(); - } - - public async updatePath( + public async move( oldRelativePath: RelativePath, newRelativePath: RelativePath ): Promise { const document = this.documents.get(oldRelativePath); if (!document) { - throw new Error( - `Cannot update physical path for document that does not exist: ${oldRelativePath}` - ); + return; } - if (this.documents.has(newRelativePath)) { + const resolvedDocument = this.getResolvedDocument(oldRelativePath); + if ( + this.documents.has(newRelativePath) && + resolvedDocument != undefined && + resolvedDocument.isDeleted + ) { throw new Error( `Cannot update physical path to path that is already in use: ${newRelativePath}` ); @@ -133,16 +192,15 @@ export class Database { private async save(): Promise { this.ensureConsistency(); await this.saveData({ - documents: Object.fromEntries(this.documents.entries()), + documents: Object.fromEntries(this.resolvedDocuments), lastSeenUpdateId: this.lastSeenUpdateId }); } private ensureConsistency(): void { - const allMetadata = Array.from(this.documents.entries()); - const idToPath = new Map>(); + const idToPath = new Map(); - allMetadata.forEach(([name, metadata]) => { + this.resolvedDocuments.forEach(([name, metadata]) => { idToPath.set(metadata.documentId, [ ...(idToPath.get(metadata.documentId) ?? []), name diff --git a/frontend/sync-client/src/services/sync-service.ts b/frontend/sync-client/src/services/sync-service.ts index 56e8a697..6f5b52e7 100644 --- a/frontend/sync-client/src/services/sync-service.ts +++ b/frontend/sync-client/src/services/sync-service.ts @@ -59,7 +59,7 @@ export class SyncService { relativePath: RelativePath; contentBytes: Uint8Array; createdDate: Date; - }): Promise { + }): Promise { const formData = new FormData(); formData.append("relative_path", relativePath); formData.append("created_date", createdDate.toISOString()); @@ -155,7 +155,7 @@ export class SyncService { documentId: DocumentId; relativePath: RelativePath; createdDate: Date; - }): Promise { + }): Promise { const response = await this.client.DELETE( "/vaults/{vault_id}/documents/{document_id}", { diff --git a/frontend/sync-client/src/services/types.ts b/frontend/sync-client/src/services/types.ts index 79d4b5f8..c1c4446e 100644 --- a/frontend/sync-client/src/services/types.ts +++ b/frontend/sync-client/src/services/types.ts @@ -4,609 +4,608 @@ */ export interface paths { - "/ping": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - get: { - parameters: { - query?: never; - header?: { - authorization?: string; - }; - path?: never; - cookie?: never; - }; - requestBody?: never; - responses: { - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["PingResponse"]; - }; - }; - default: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["SerializedError"]; - }; - }; - }; - }; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/vaults/{vault_id}/documents": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - get: { - parameters: { - query?: { - since_update_id?: number | null; - }; - header: { - authorization: string; - }; - path: { - vault_id: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["FetchLatestDocumentsResponse"]; - }; - }; - default: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["SerializedError"]; - }; - }; - }; - }; - put?: never; - post: { - parameters: { - query?: never; - header: { - authorization: string; - }; - path: { - vault_id: string; - }; - cookie?: never; - }; - requestBody: { - content: { - "multipart/form-data": components["schemas"]["CreateDocumentVersionMultipart"]; - }; - }; - responses: { - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["DocumentVersionWithoutContent"]; - }; - }; - default: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["SerializedError"]; - }; - }; - }; - }; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/vaults/{vault_id}/documents/json": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - get?: never; - put?: never; - post: { - parameters: { - query?: never; - header: { - authorization: string; - }; - path: { - vault_id: string; - }; - cookie?: never; - }; - requestBody: { - content: { - "application/json": components["schemas"]["CreateDocumentVersion"]; - }; - }; - responses: { - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["DocumentVersionWithoutContent"]; - }; - }; - default: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["SerializedError"]; - }; - }; - }; - }; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/vaults/{vault_id}/documents/{document_id}": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - get: { - parameters: { - query?: never; - header: { - authorization: string; - }; - path: { - document_id: string; - vault_id: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["DocumentVersion"]; - }; - }; - default: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["SerializedError"]; - }; - }; - }; - }; - put: { - parameters: { - query?: never; - header: { - authorization: string; - }; - path: { - document_id: string; - vault_id: string; - }; - cookie?: never; - }; - requestBody: { - content: { - "multipart/form-data": components["schemas"]["UpdateDocumentVersionMultipart"]; - }; - }; - responses: { - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["DocumentUpdateResponse"]; - }; - }; - default: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["SerializedError"]; - }; - }; - }; - }; - post?: never; - delete: { - parameters: { - query?: never; - header: { - authorization: string; - }; - path: { - document_id: string; - vault_id: string; - }; - cookie?: never; - }; - requestBody: { - content: { - "application/json": components["schemas"]["DeleteDocumentVersion"]; - }; - }; - responses: { - /** @description no content */ - 200: { - headers: { - [name: string]: unknown; - }; - content?: never; - }; - default: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["SerializedError"]; - }; - }; - }; - }; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/vaults/{vault_id}/documents/{document_id}/json": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - get?: never; - put: { - parameters: { - query?: never; - header: { - authorization: string; - }; - path: { - document_id: string; - vault_id: string; - }; - cookie?: never; - }; - requestBody: { - content: { - "application/json": components["schemas"]["UpdateDocumentVersion"]; - }; - }; - responses: { - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["DocumentUpdateResponse"]; - }; - }; - default: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["SerializedError"]; - }; - }; - }; - }; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/vaults/{vault_id}/documents/{document_id}/versions/{version_id}": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - get?: never; - put: { - parameters: { - query?: never; - header: { - authorization: string; - }; - path: { - document_id: string; - vault_id: string; - vault_update_id: number; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["DocumentVersion"]; - }; - }; - default: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["SerializedError"]; - }; - }; - }; - }; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/vaults/{vault_id}/documents/{document_id}/versions/{version_id}/content": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - get?: never; - put: { - parameters: { - query?: never; - header: { - authorization: string; - }; - path: { - document_id: string; - vault_id: string; - vault_update_id: number; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description byte stream */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/octet-stream": unknown; - }; - }; - default: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["SerializedError"]; - }; - }; - }; - }; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; + "/ping": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: { + parameters: { + query?: never; + header?: { + authorization?: string; + }; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["PingResponse"]; + }; + }; + default: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SerializedError"]; + }; + }; + }; + }; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/vaults/{vault_id}/documents": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: { + parameters: { + query?: { + since_update_id?: number | null; + }; + header: { + authorization: string; + }; + path: { + vault_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["FetchLatestDocumentsResponse"]; + }; + }; + default: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SerializedError"]; + }; + }; + }; + }; + put?: never; + post: { + parameters: { + query?: never; + header: { + authorization: string; + }; + path: { + vault_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "multipart/form-data": components["schemas"]["CreateDocumentVersionMultipart"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DocumentVersionWithoutContent"]; + }; + }; + default: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SerializedError"]; + }; + }; + }; + }; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/vaults/{vault_id}/documents/json": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: { + parameters: { + query?: never; + header: { + authorization: string; + }; + path: { + vault_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["CreateDocumentVersion"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DocumentVersionWithoutContent"]; + }; + }; + default: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SerializedError"]; + }; + }; + }; + }; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/vaults/{vault_id}/documents/{document_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: { + parameters: { + query?: never; + header: { + authorization: string; + }; + path: { + document_id: string; + vault_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DocumentVersion"]; + }; + }; + default: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SerializedError"]; + }; + }; + }; + }; + put: { + parameters: { + query?: never; + header: { + authorization: string; + }; + path: { + document_id: string; + vault_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "multipart/form-data": components["schemas"]["UpdateDocumentVersionMultipart"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DocumentUpdateResponse"]; + }; + }; + default: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SerializedError"]; + }; + }; + }; + }; + post?: never; + delete: { + parameters: { + query?: never; + header: { + authorization: string; + }; + path: { + document_id: string; + vault_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["DeleteDocumentVersion"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DocumentVersionWithoutContent"]; + }; + }; + default: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SerializedError"]; + }; + }; + }; + }; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/vaults/{vault_id}/documents/{document_id}/json": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put: { + parameters: { + query?: never; + header: { + authorization: string; + }; + path: { + document_id: string; + vault_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["UpdateDocumentVersion"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DocumentUpdateResponse"]; + }; + }; + default: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SerializedError"]; + }; + }; + }; + }; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/vaults/{vault_id}/documents/{document_id}/versions/{version_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put: { + parameters: { + query?: never; + header: { + authorization: string; + }; + path: { + document_id: string; + vault_id: string; + vault_update_id: number; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DocumentVersion"]; + }; + }; + default: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SerializedError"]; + }; + }; + }; + }; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/vaults/{vault_id}/documents/{document_id}/versions/{version_id}/content": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put: { + parameters: { + query?: never; + header: { + authorization: string; + }; + path: { + document_id: string; + vault_id: string; + vault_update_id: number; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description byte stream */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/octet-stream": unknown; + }; + }; + default: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SerializedError"]; + }; + }; + }; + }; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; } export type webhooks = Record; export interface components { - schemas: { - Array_of_uint8: number[]; - CreateDocumentVersion: { - contentBase64: string; - /** Format: date-time */ - createdDate: string; - relativePath: string; - }; - CreateDocumentVersionMultipart: { - content: components["schemas"]["Array_of_uint8"]; - /** Format: date-time */ - created_date: string; - relative_path: string; - }; - DeleteDocumentVersion: { - /** Format: date-time */ - createdDate: string; - relativePath: string; - }; - /** @description Response to a update document request. */ - DocumentUpdateResponse: - | { - /** Format: date-time */ - createdDate: string; - /** Format: uuid */ - documentId: string; - isDeleted: boolean; - relativePath: string; - /** @enum {string} */ - type: "FastForwardUpdate"; - /** Format: date-time */ - updatedDate: string; - vaultId: string; - /** Format: int64 */ - vaultUpdateId: number; - } - | { - contentBase64: string; - /** Format: date-time */ - createdDate: string; - /** Format: uuid */ - documentId: string; - isDeleted: boolean; - relativePath: string; - /** @enum {string} */ - type: "MergingUpdate"; - /** Format: date-time */ - updatedDate: string; - vaultId: string; - /** Format: int64 */ - vaultUpdateId: number; - }; - DocumentVersion: { - contentBase64: string; - /** Format: date-time */ - createdDate: string; - /** Format: uuid */ - documentId: string; - isDeleted: boolean; - relativePath: string; - /** Format: date-time */ - updatedDate: string; - vaultId: string; - /** Format: int64 */ - vaultUpdateId: number; - }; - DocumentVersionWithoutContent: { - /** Format: date-time */ - createdDate: string; - /** Format: uuid */ - documentId: string; - isDeleted: boolean; - relativePath: string; - /** Format: date-time */ - updatedDate: string; - vaultId: string; - /** Format: int64 */ - vaultUpdateId: number; - }; - /** @description Response to a fetch latest documents request. */ - FetchLatestDocumentsResponse: { - /** - * Format: int64 - * @description The update ID of the latest document in the response. - */ - lastUpdateId: number; - latestDocuments: components["schemas"]["DocumentVersionWithoutContent"][]; - }; - PathParams: { - vault_id: string; - }; - PathParams2: { - vault_id: string; - }; - PathParams3: { - /** Format: uuid */ - document_id: string; - vault_id: string; - }; - PathParams4: { - /** Format: uuid */ - document_id: string; - vault_id: string; - }; - PathParams5: { - /** Format: uuid */ - document_id: string; - vault_id: string; - /** Format: int64 */ - vault_update_id: number; - }; - PathParams6: { - /** Format: uuid */ - document_id: string; - vault_id: string; - /** Format: int64 */ - vault_update_id: number; - }; - PathParams7: { - /** Format: uuid */ - document_id: string; - vault_id: string; - }; - /** @description Response to a ping request. */ - PingResponse: { - /** @description Whether the client is authenticated based on the sent Authorization header. */ - isAuthenticated: boolean; - /** @description Semantic version of the server. */ - serverVersion: string; - }; - QueryParams: { - /** Format: int64 */ - since_update_id?: number | null; - }; - SerializedError: { - causes: string[]; - message: string; - }; - UpdateDocumentVersion: { - contentBase64: string; - /** Format: date-time */ - createdDate: string; - /** Format: int64 */ - parentVersionId: number; - relativePath: string; - }; - UpdateDocumentVersionMultipart: { - content: components["schemas"]["Array_of_uint8"]; - /** Format: date-time */ - createdDate: string; - /** Format: int64 */ - parentVersionId: number; - relativePath: string; - }; - }; - responses: never; - parameters: never; - requestBodies: never; - headers: never; - pathItems: never; + schemas: { + Array_of_uint8: number[]; + CreateDocumentVersion: { + contentBase64: string; + /** Format: date-time */ + createdDate: string; + relativePath: string; + }; + CreateDocumentVersionMultipart: { + content: components["schemas"]["Array_of_uint8"]; + /** Format: date-time */ + created_date: string; + relative_path: string; + }; + DeleteDocumentVersion: { + /** Format: date-time */ + createdDate: string; + relativePath: string; + }; + /** @description Response to an update document request. */ + DocumentUpdateResponse: { + /** Format: date-time */ + createdDate: string; + /** Format: uuid */ + documentId: string; + isDeleted: boolean; + relativePath: string; + /** @enum {string} */ + type: "FastForwardUpdate"; + /** Format: date-time */ + updatedDate: string; + vaultId: string; + /** Format: int64 */ + vaultUpdateId: number; + } | { + contentBase64: string; + /** Format: date-time */ + createdDate: string; + /** Format: uuid */ + documentId: string; + isDeleted: boolean; + relativePath: string; + /** @enum {string} */ + type: "MergingUpdate"; + /** Format: date-time */ + updatedDate: string; + vaultId: string; + /** Format: int64 */ + vaultUpdateId: number; + }; + DocumentVersion: { + contentBase64: string; + /** Format: date-time */ + createdDate: string; + /** Format: uuid */ + documentId: string; + isDeleted: boolean; + relativePath: string; + /** Format: date-time */ + updatedDate: string; + vaultId: string; + /** Format: int64 */ + vaultUpdateId: number; + }; + DocumentVersionWithoutContent: { + /** Format: date-time */ + createdDate: string; + /** Format: uuid */ + documentId: string; + isDeleted: boolean; + relativePath: string; + /** Format: date-time */ + updatedDate: string; + vaultId: string; + /** Format: int64 */ + vaultUpdateId: number; + }; + /** @description Response to a fetch latest documents request. */ + FetchLatestDocumentsResponse: { + /** + * Format: int64 + * @description The update ID of the latest document in the response. + */ + lastUpdateId: number; + latestDocuments: components["schemas"]["DocumentVersionWithoutContent"][]; + }; + PathParams: { + vault_id: string; + }; + PathParams2: { + vault_id: string; + }; + PathParams3: { + /** Format: uuid */ + document_id: string; + vault_id: string; + }; + PathParams4: { + /** Format: uuid */ + document_id: string; + vault_id: string; + }; + PathParams5: { + /** Format: uuid */ + document_id: string; + vault_id: string; + /** Format: int64 */ + vault_update_id: number; + }; + PathParams6: { + /** Format: uuid */ + document_id: string; + vault_id: string; + /** Format: int64 */ + vault_update_id: number; + }; + PathParams7: { + /** Format: uuid */ + document_id: string; + vault_id: string; + }; + /** @description Response to a ping request. */ + PingResponse: { + /** @description Whether the client is authenticated based on the sent Authorization header. */ + isAuthenticated: boolean; + /** @description Semantic version of the server. */ + serverVersion: string; + }; + QueryParams: { + /** Format: int64 */ + since_update_id?: number | null; + }; + SerializedError: { + causes: string[]; + message: string; + }; + UpdateDocumentVersion: { + contentBase64: string; + /** Format: date-time */ + createdDate: string; + /** Format: int64 */ + parentVersionId: number; + relativePath: string; + }; + UpdateDocumentVersionMultipart: { + content: components["schemas"]["Array_of_uint8"]; + /** Format: date-time */ + createdDate: string; + /** Format: int64 */ + parentVersionId: number; + relativePath: string; + }; + }; + responses: never; + parameters: never; + requestBodies: never; + headers: never; + pathItems: never; } export type $defs = Record; export type operations = Record; diff --git a/frontend/sync-client/src/sync-client.ts b/frontend/sync-client/src/sync-client.ts index 302daf35..54ba6b45 100644 --- a/frontend/sync-client/src/sync-client.ts +++ b/frontend/sync-client/src/sync-client.ts @@ -42,7 +42,7 @@ export class SyncClient { } public get documentCount(): number { - return this._database.getDocuments().size; + return this._database.length; } public set fetchImplementation(fetch: typeof globalThis.fetch) { diff --git a/frontend/sync-client/src/sync-operations/syncer.ts b/frontend/sync-client/src/sync-operations/syncer.ts index dcb476dd..193848c4 100644 --- a/frontend/sync-client/src/sync-operations/syncer.ts +++ b/frontend/sync-client/src/sync-operations/syncer.ts @@ -1,4 +1,8 @@ -import type { Database, RelativePath } from "../persistence/database"; +import type { + Database, + DocumentMetadata, + RelativePath +} from "../persistence/database"; import type { SyncService } from "src/services/sync-service"; import type { Logger } from "src/tracing/logger"; @@ -10,6 +14,7 @@ import type { Settings } from "src/persistence/settings"; import type { FileOperations } from "src/file-operations/file-operations"; import { findMatchingFileBasedOnHash } from "src/utils/find-matching-file-based-on-hash"; import { UnrestrictedSyncer } from "./unrestricted-syncer"; +import { FileNotFoundError } from "src/file-operations/safe-filesystem-operations"; export class Syncer { private readonly remainingOperationsListeners: (( @@ -58,6 +63,23 @@ export class Syncer { ); } + private static async forgivingFileNotFoundWrapper( + fn: () => Promise, + logger: Logger + ): Promise { + try { + return await fn(); + } catch (e) { + if (e instanceof FileNotFoundError) { + logger.debug( + `File has been deleted or moved before we had a chance to inspect it, skipping` + ); + } else { + throw e; + } + } + } + public addRemainingOperationsListener( listener: (remainingOperations: number) => void ): void { @@ -68,10 +90,42 @@ export class Syncer { relativePath: RelativePath, updateTime: Date ): Promise { + let resolve: + | undefined + | ((metadata: DocumentMetadata | undefined) => void) = undefined; + + const creationPromise = new Promise( + (r) => (resolve = r) + ); + + await this.database.setDocumentPromise({ + relativePath, + promise: creationPromise + }); + + await this.syncQueue.add(async () => { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + resolve!( + await this.internalSyncer.unrestrictedSyncLocallyCreatedFile( + relativePath, + updateTime + ) + ); + }); + } + + public async syncLocallyDeletedFile( + relativePath: RelativePath + ): Promise { + let metadata = this.database.getDocument(relativePath); + if (metadata !== undefined && !(metadata instanceof Promise)) { + metadata = Promise.resolve(metadata); + } + await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncLocallyCreatedFile( + this.internalSyncer.unrestrictedSyncLocallyDeletedFile( relativePath, - updateTime + metadata ) ); } @@ -81,8 +135,25 @@ export class Syncer { relativePath: RelativePath; updateTime: Date; }): Promise { + if (args.oldPath === args.relativePath) { + throw new Error( + `Old path and new path are the same: ${args.oldPath}` + ); + } + + if (args.oldPath !== undefined) { + await this.database.move(args.oldPath, args.relativePath); + } + + let metadata = this.database.getDocument(args.relativePath); + if (metadata !== undefined && !(metadata instanceof Promise)) { + metadata = Promise.resolve(metadata); + } await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncLocallyUpdatedFile(args) + this.internalSyncer.unrestrictedSyncLocallyUpdatedFile({ + ...args, + metadata + }) ); } @@ -90,14 +161,6 @@ export class Syncer { return this.syncQueue.onEmpty(); } - public async syncLocallyDeletedFile( - relativePath: RelativePath - ): Promise { - await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncLocallyDeletedFile(relativePath) - ); - } - public async scheduleSyncForOfflineChanges(): Promise { if (!this.settings.getSettings().isSyncEnabled) { this.logger.debug( @@ -178,32 +241,50 @@ export class Syncer { // This includes renamed files for now let locallyPossiblyDeletedFiles = [ - ...this.database.getDocuments().entries() + ...this.database.resolvedDocuments ].filter(([path, _]) => !allLocalFiles.includes(path)); - await Promise.all( + const updates = Promise.all( allLocalFiles.map(async (relativePath) => this.syncQueue.add(async () => { - const metadata = this.database.getDocument(relativePath); + const metadata = + this.database.getResolvedDocument(relativePath); if (metadata) { this.logger.debug( `Document ${relativePath} might have been updated locally, scheduling sync to validate and update it` ); + const updateTime = + await Syncer.forgivingFileNotFoundWrapper( + async () => + this.operations.getModificationTime( + relativePath + ), + this.logger + ); + if (updateTime === undefined) { + return; + } + return this.internalSyncer.unrestrictedSyncLocallyUpdatedFile( { relativePath, - updateTime: - await this.operations.getModificationTime( - relativePath - ) + updateTime, + metadata: Promise.resolve(metadata) } ); } // Perhaps the file has been moved. Let's check by looking at the deleted files const contentBytes = - await this.operations.read(relativePath); + await Syncer.forgivingFileNotFoundWrapper( + async () => this.operations.read(relativePath), + this.logger + ); + if (contentBytes === undefined) { + return; + } + const contentHash = hash(contentBytes); // todo: make this smarter so that offline files can be renamed & edited at the same time @@ -221,14 +302,29 @@ export class Syncer { this.logger.debug( `Document '${originalFile[0]}' was not found under its current path in the database but was found under a different path (${relativePath}), scheduling sync to move it` ); + + const updateTime = + await Syncer.forgivingFileNotFoundWrapper( + async () => + this.operations.getModificationTime( + relativePath + ), + this.logger + ); + if (updateTime === undefined) { + return; + } + return this.internalSyncer.unrestrictedSyncLocallyUpdatedFile( { oldPath: originalFile[0], - relativePath: relativePath, - updateTime: - await this.operations.getModificationTime( + relativePath, + updateTime, + metadata: Promise.resolve( + this.database.getResolvedDocument( relativePath - ), + ) + ), optimisations: { contentBytes, contentHash @@ -240,15 +336,26 @@ export class Syncer { this.logger.debug( `Document ${relativePath} not found in database, scheduling sync to create it` ); + const updateTime = + await Syncer.forgivingFileNotFoundWrapper( + async () => + this.operations.getModificationTime( + relativePath + ), + this.logger + ); + if (updateTime === undefined) { + return; + } return this.internalSyncer.unrestrictedSyncLocallyCreatedFile( relativePath, - await this.operations.getModificationTime(relativePath) + updateTime ); }) ) ); - await Promise.all( + const deletes = Promise.all( locallyPossiblyDeletedFiles.map(async ([relativePath, _]) => { this.logger.debug( `Document ${relativePath} has been deleted locally, scheduling sync to delete it` @@ -265,6 +372,8 @@ export class Syncer { return this.syncLocallyDeletedFile(relativePath); }) ); + + await Promise.all([updates, deletes]); } private async internalApplyRemoteChangesLocally(): Promise { @@ -280,9 +389,15 @@ export class Syncer { this.logger.info("Applying remote changes locally"); await Promise.all( - remote.latestDocuments.map(async (remoteDocument) => - this.syncRemotelyUpdatedFile(remoteDocument) - ) + remote.latestDocuments + .filter( + (remoteDocument) => + remoteDocument.vaultUpdateId > + (this.database.getDocumentByDocumentId( + remoteDocument.documentId + )?.[1].parentVersionId ?? -1) + ) + .map(this.syncRemotelyUpdatedFile.bind(this)) ); const lastSeenUpdateId = this.database.getLastSeenUpdateId(); diff --git a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts index 86ed3089..c7b6f044 100644 --- a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts +++ b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts @@ -1,19 +1,23 @@ -import type { Database, RelativePath } from "../persistence/database"; +import type { + Database, + DocumentMetadata, + RelativePath +} from "../persistence/database"; import type { SyncService } from "src/services/sync-service"; import type { Logger } from "src/tracing/logger"; import type { SyncHistory } from "src/tracing/sync-history"; import { SyncSource, SyncStatus, SyncType } from "src/tracing/sync-history"; -import { hash } from "src/utils/hash"; +import { EMPTY_HASH, hash } from "src/utils/hash"; import type { components } from "src/services/types"; import { deserialize } from "src/utils/deserialize"; import type { Settings } from "src/persistence/settings"; import type { FileOperations } from "src/file-operations/file-operations"; import { FileNotFoundError } from "src/file-operations/safe-filesystem-operations"; -import { DocumentLocks } from "./document-locks"; +import { DocumentLocks } from "../file-operations/document-locks"; export class UnrestrictedSyncer { - private readonly locks = new DocumentLocks(); + private readonly locks: DocumentLocks; public constructor( private readonly logger: Logger, @@ -22,7 +26,9 @@ export class UnrestrictedSyncer { private readonly syncService: SyncService, private readonly operations: FileOperations, private readonly history: SyncHistory - ) {} + ) { + this.locks = new DocumentLocks(logger); + } public async unrestrictedSyncLocallyCreatedFile( relativePath: RelativePath, @@ -31,96 +37,132 @@ export class UnrestrictedSyncer { contentBytes?: Uint8Array; contentHash?: string; } - ): Promise { - await this.executeWhileHoldingFileLock( + ): Promise { + return this.executeSync( [relativePath], SyncType.CREATE, SyncSource.PUSH, async () => { + const localMetadata = this.database.getDocument(relativePath); + if ( - (await this.operations.getFileSize(relativePath)) / // this can throw FileNotFoundError - 1024 / - 1024 > - this.settings.getSettings().maxFileSizeMB + !(localMetadata instanceof Promise) && + localMetadata && + !localMetadata.isDeleted ) { - this.history.addHistoryEntry({ - status: SyncStatus.ERROR, - relativePath, - message: `File size exceeds the maximum file size limit of ${ - this.settings.getSettings().maxFileSizeMB - }MB`, - type: SyncType.CREATE - }); + this.logger.debug( + `Document metadata already exists for ${relativePath}, it must have been downloaded from the server` + ); + return; } const contentBytes = optimisations?.contentBytes ?? (await this.operations.read(relativePath)); // this can throw FileNotFoundError - let contentHash = + const contentHash = optimisations?.contentHash ?? hash(contentBytes); - const localMetadata = this.database.getDocument(relativePath); - if (localMetadata) { - this.logger.debug( - `Document metadata already exists for ${relativePath}, it must have been downloaded from the server` - ); - - if (localMetadata.hash === contentHash) { - this.history.addHistoryEntry({ - status: SyncStatus.NO_OP, - relativePath, - message: `File hash matches with last synced version, no need to sync`, - type: SyncType.UPDATE - }); - return; - } - } - const response = await this.syncService.create({ relativePath, contentBytes, createdDate: updateTime }); + const currentMetadata = + this.database.getDocumentByIdentity(localMetadata); + if (!currentMetadata) { + throw new Error( + `Document metadata for ${relativePath} not found after creation` + ); + } + this.history.addHistoryEntry({ status: SyncStatus.SUCCESS, source: SyncSource.PUSH, - relativePath, + relativePath: currentMetadata[0], message: `Successfully uploaded locally created file`, type: SyncType.CREATE }); - // The response can't have a different relative path than the one we sent - // because the relative path is the key when finding existing documents - // when a create request is sent. - - if (response.type === "MergingUpdate") { - const responseBytes = deserialize(response.contentBase64); - contentHash = hash(responseBytes); - - await this.operations.write( - relativePath, - contentBytes, - responseBytes - ); - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - source: SyncSource.PULL, - relativePath, - message: `The file we created locally has already existed remotely, so we have merged them`, - type: SyncType.UPDATE - }); - } + const newMetadata = { + documentId: response.documentId, + parentVersionId: response.vaultUpdateId, + hash: contentHash, + isDeleted: false + }; await this.database.setDocument({ - documentId: response.documentId, - relativePath: response.relativePath, - parentVersionId: response.vaultUpdateId, - hash: contentHash + relativePath: currentMetadata[0], + ...newMetadata }); await this.tryIncrementVaultUpdateId(response.vaultUpdateId); + + return newMetadata; + } + ); + } + + public async unrestrictedSyncLocallyDeletedFile( + relativePath: RelativePath, + metadata: Promise | undefined + ): Promise { + await this.executeSync( + [relativePath], + SyncType.DELETE, + SyncSource.PUSH, + async () => { + const localMetadata = + metadata !== undefined + ? await metadata + : this.database.getResolvedDocument(relativePath); + + if (!localMetadata || localMetadata.isDeleted) { + this.logger.info( + `Locally deleted file hasn't been uploaded yet, so there's no need to delete it on the remote server` + ); + + return; + } + + const response = await this.syncService.delete({ + documentId: localMetadata.documentId, + relativePath, + createdDate: new Date() // We got the event now, so it must have been deleted just now + }); + + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + source: SyncSource.PUSH, + relativePath, + message: `Successfully deleted locally deleted file on the remote server`, + type: SyncType.DELETE + }); + + const currentMetadata = this.database.getDocumentByDocumentId( + localMetadata.documentId + ); + + if (!currentMetadata || currentMetadata[1].isDeleted) { + this.logger.info( + `No metadata found for deleted file, '${relativePath}' must have been deleted by another operation` + ); + + return; + } + + await this.operations.delete(currentMetadata[0]); + + // We have to have a record of the delete in case there's an in-flight update for the same + // document which finishes after the delete has succeeded and would introduce a phantom metadata record. + await this.database.setDocument({ + relativePath: currentMetadata[0], + documentId: response.documentId, + parentVersionId: response.vaultUpdateId, + hash: EMPTY_HASH, + isDeleted: true + }); } ); } @@ -128,53 +170,34 @@ export class UnrestrictedSyncer { public async unrestrictedSyncLocallyUpdatedFile({ oldPath, relativePath, + metadata, updateTime, optimisations }: { oldPath?: RelativePath; relativePath: RelativePath; + metadata: Promise | undefined; updateTime: Date; optimisations?: { contentBytes?: Uint8Array; contentHash?: string; }; }): Promise { - await this.executeWhileHoldingFileLock( + await this.executeSync( [oldPath, relativePath].filter((path) => path !== undefined), SyncType.UPDATE, SyncSource.PUSH, async () => { - // Check the new path first in case the metadata has been already moved - let localMetadata = this.database.getDocument(relativePath); - let metadataPath = relativePath; + const localMetadata = + metadata !== undefined + ? await metadata + : this.database.getResolvedDocument(relativePath); - if (localMetadata === undefined && oldPath !== undefined) { - localMetadata = this.database.getDocument(oldPath); - metadataPath = oldPath; - } - - if (!localMetadata) { + if (!localMetadata || localMetadata.isDeleted) { // It's fine, a subsequent sync operation must have dealt with this return; } - if ( - (await this.operations.getFileSize(relativePath)) / // this can throw FileNotFoundError - 1024 / - 1024 > - this.settings.getSettings().maxFileSizeMB - ) { - this.history.addHistoryEntry({ - status: SyncStatus.ERROR, - relativePath, - message: `File size exceeds the maximum file size limit of ${ - this.settings.getSettings().maxFileSizeMB - }MB`, - type: SyncType.CREATE - }); - return; - } - const contentBytes = optimisations?.contentBytes ?? (await this.operations.read(relativePath)); // this can throw FileNotFoundError @@ -186,23 +209,48 @@ export class UnrestrictedSyncer { localMetadata.hash === contentHash && oldPath === undefined ) { - this.history.addHistoryEntry({ - status: SyncStatus.NO_OP, - relativePath, - message: `File hash matches with last synced version, no need to sync`, - type: SyncType.UPDATE - }); + this.logger.debug( + `File hash of ${relativePath} matches with last synced version and the path hasn't changed; no need to sync` + ); + return; + } + + // Re-fetch based on the documentId instead of the relativePath because + // the relativePath might have changed since this operation was scheduled + let latestMetadata = this.database.getDocumentByDocumentId( + localMetadata.documentId + ); + if (!latestMetadata || latestMetadata[1].isDeleted) { + // It's fine, a subsequent sync operation must have dealt with this return; } const response = await this.syncService.put({ - documentId: localMetadata.documentId, - parentVersionId: localMetadata.parentVersionId, - relativePath, + documentId: latestMetadata[1].documentId, + parentVersionId: latestMetadata[1].parentVersionId, + relativePath: latestMetadata[0], contentBytes, createdDate: updateTime }); + latestMetadata = this.database.getDocumentByDocumentId( + response.documentId + ); + + if (!latestMetadata || latestMetadata[1].isDeleted) { + // The document has been deleted since this operation was scheduled + return; + } + + if ( + latestMetadata[1].parentVersionId >= response.vaultUpdateId + ) { + this.logger.debug( + `Document ${relativePath} is already more up to date than the fetched version` + ); + return; + } + this.history.addHistoryEntry({ status: SyncStatus.SUCCESS, source: SyncSource.PUSH, @@ -212,11 +260,7 @@ export class UnrestrictedSyncer { }); if (response.isDeleted) { - await this.operations.remove(oldPath ?? relativePath); - await this.database.removeDocument(oldPath ?? relativePath); - await this.tryIncrementVaultUpdateId( - response.vaultUpdateId - ); + await this.operations.delete(relativePath); this.history.addHistoryEntry({ status: SyncStatus.SUCCESS, @@ -227,110 +271,69 @@ export class UnrestrictedSyncer { type: SyncType.DELETE }); - return; - } - - if ( - response.relativePath != relativePath && - response.relativePath != oldPath - ) { - await this.locks.waitForDocumentLock(response.relativePath); - } - - try { - if (response.relativePath != relativePath) { - // TODO: this can fail, that's bad - await this.operations.move( - // this can throw FileNotFoundError - relativePath, - response.relativePath, - response.documentId - ); - } - - if (response.type === "MergingUpdate") { - const responseBytes = deserialize( - response.contentBase64 - ); - contentHash = hash(responseBytes); - - await this.operations.write( - response.relativePath, - contentBytes, - responseBytes - ); - - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - source: SyncSource.PULL, - relativePath, - message: `The file we updated had been updated remotely, so we downloaded the merged version`, - type: SyncType.UPDATE - }); - } - - if (metadataPath !== response.relativePath) { - await this.database.updatePath( - metadataPath, - response.relativePath - ); - } await this.database.setDocument({ - documentId: localMetadata.documentId, - relativePath: response.relativePath, + documentId: response.documentId, + relativePath: latestMetadata[0], parentVersionId: response.vaultUpdateId, - hash: contentHash + hash: EMPTY_HASH, + isDeleted: true }); await this.tryIncrementVaultUpdateId( response.vaultUpdateId ); - } finally { - if ( - response.relativePath != relativePath && - response.relativePath != oldPath - ) { - this.locks.unlockDocument(response.relativePath); - } - } - } - ); - } - public async unrestrictedSyncLocallyDeletedFile( - relativePath: RelativePath - ): Promise { - await this.executeWhileHoldingFileLock( - [relativePath], - SyncType.DELETE, - SyncSource.PUSH, - async () => { - const localMetadata = this.database.getDocument(relativePath); - if (!localMetadata) { - this.history.addHistoryEntry({ - status: SyncStatus.NO_OP, - relativePath, - message: `Locally deleted file hasn't been uploaded yet, so there's no need to delete it on the remote server`, - type: SyncType.DELETE - }); return; } - await this.syncService.delete({ - documentId: localMetadata.documentId, - relativePath, - createdDate: new Date() // We got the event now, so it must have been deleted just now + if ( + latestMetadata[1].parentVersionId >= response.vaultUpdateId + ) { + this.logger.debug( + `Document ${relativePath} is already more up to date than the fetched version` + ); + return; + } + + if (response.relativePath != relativePath) { + await this.operations.move( + latestMetadata[0], + response.relativePath, + response.documentId + ); // this can throw FileNotFoundError + } + + if (response.type === "MergingUpdate") { + const responseBytes = deserialize(response.contentBase64); + contentHash = hash(responseBytes); + + await this.operations.write( + response.relativePath, + contentBytes, + responseBytes + ); + + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + source: SyncSource.PULL, + relativePath, + message: `The file we updated had been updated remotely, so we downloaded the merged version`, + type: SyncType.UPDATE + }); + } + + await this.database.setDocument({ + documentId: response.documentId, + relativePath: + response.relativePath != relativePath + ? response.relativePath + : latestMetadata[0], + parentVersionId: response.vaultUpdateId, + hash: contentHash, + isDeleted: response.isDeleted }); - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - source: SyncSource.PUSH, - relativePath, - message: `Successfully deleted locally deleted file on the remote server`, - type: SyncType.DELETE - }); - - await this.database.removeDocument(relativePath); + await this.tryIncrementVaultUpdateId(response.vaultUpdateId); } ); } @@ -338,56 +341,68 @@ export class UnrestrictedSyncer { public async unrestrictedSyncRemotelyUpdatedFile( remoteVersion: components["schemas"]["DocumentVersionWithoutContent"] ): Promise { - await this.executeWhileHoldingFileLock( + await this.executeSync( [remoteVersion.relativePath], SyncType.UPDATE, SyncSource.PULL, async () => { - let localMetadata = this.database.getDocumentByDocumentId( + const content = ( + await this.syncService.get({ + documentId: remoteVersion.documentId + }) + ).contentBase64; + const contentBytes = deserialize(content); + const contentHash = hash(contentBytes); + + const localMetadata = this.database.getDocumentByDocumentId( remoteVersion.documentId ); - if ( - localMetadata && - localMetadata[0] !== remoteVersion.relativePath + localMetadata?.[1].documentId === + remoteVersion.documentId && + localMetadata[1].parentVersionId > + remoteVersion.vaultUpdateId ) { - await this.locks.waitForDocumentLock(localMetadata[0]); + this.logger.info( + `Document ${remoteVersion.relativePath} is already up to date` + ); + return; } - // Waiting for the new lock might take a while so we need to fetch the database - // entry again in case it's changed. - localMetadata = this.database.getDocumentByDocumentId( - remoteVersion.documentId - ); - if (!localMetadata) { + const localBytes = await this.operations.read( + remoteVersion.relativePath + ); // this can throw FileNotFoundError + const localHash = hash(localBytes); + + if (localHash !== localMetadata?.[1].hash) { + this.logger.info( + `Document ${remoteVersion.relativePath} has pending local changes, so we shouldn't update it here` + ); + return; + } + + if (!localMetadata || localMetadata[1].isDeleted) { if (remoteVersion.isDeleted) { - this.history.addHistoryEntry({ - status: SyncStatus.NO_OP, - source: SyncSource.PULL, - relativePath: remoteVersion.relativePath, - message: `Remotely deleted file hasn't been synced yet, so there's no need to delete it locally`, - type: SyncType.DELETE - }); + this.logger.info( + `Remotely deleted file hasn't been synced yet, so there's no need to delete it locally` + ); return; } - const content = ( - await this.syncService.get({ - documentId: remoteVersion.documentId - }) - ).contentBase64; - const contentBytes = deserialize(content); - await this.operations.create( remoteVersion.relativePath, - contentBytes + contentBytes, + remoteVersion.documentId ); + await this.database.setDocument({ documentId: remoteVersion.documentId, relativePath: remoteVersion.relativePath, parentVersionId: remoteVersion.vaultUpdateId, - hash: hash(contentBytes) + hash: hash(contentBytes), + isDeleted: remoteVersion.isDeleted }); + this.history.addHistoryEntry({ status: SyncStatus.SUCCESS, source: SyncSource.PULL, @@ -399,7 +414,6 @@ export class UnrestrictedSyncer { } const [relativePath, metadata] = localMetadata; - if (remoteVersion.vaultUpdateId <= metadata.parentVersionId) { this.logger.debug( `Document ${relativePath} is already up to date` @@ -407,89 +421,70 @@ export class UnrestrictedSyncer { return; } - try { - if (remoteVersion.isDeleted) { - await this.operations.remove(relativePath); - await this.database.removeDocument(relativePath); + if (remoteVersion.isDeleted) { + await this.operations.delete(relativePath); - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - source: SyncSource.PULL, - relativePath: remoteVersion.relativePath, - message: `Successfully deleted remotely deleted file locally`, - type: SyncType.DELETE - }); - } else { - // TODO: this can fail, that's bad - const currentContent = - await this.operations.read(relativePath); // this can throw FileNotFoundError - const currentHash = hash(currentContent); + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + source: SyncSource.PULL, + relativePath: remoteVersion.relativePath, + message: `Successfully deleted remotely deleted file locally`, + type: SyncType.DELETE + }); - if (currentHash !== metadata.hash) { - this.logger.info( - `Document ${relativePath} has been updated both remotely and locally, letting the local file update event handle it` - ); - return; - } + await this.database.setDocument({ + documentId: remoteVersion.documentId, + relativePath: relativePath, + parentVersionId: remoteVersion.vaultUpdateId, + hash: EMPTY_HASH, + isDeleted: true + }); - const content = ( - await this.syncService.get({ - documentId: remoteVersion.documentId - }) - ).contentBase64; - const contentBytes = deserialize(content); - const contentHash = hash(contentBytes); - - if (relativePath !== remoteVersion.relativePath) { - // TODO: this can fail, that's bad - await this.operations.move( - // this can throw FileNotFoundError - relativePath, - remoteVersion.relativePath, - remoteVersion.documentId - ); - - await this.database.updatePath( - relativePath, - remoteVersion.relativePath - ); - } - - await this.operations.write( - remoteVersion.relativePath, - currentContent, - contentBytes - ); - await this.database.setDocument({ - documentId: remoteVersion.documentId, - relativePath: remoteVersion.relativePath, - parentVersionId: remoteVersion.vaultUpdateId, - hash: contentHash - }); - - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - source: SyncSource.PULL, - relativePath: remoteVersion.relativePath, - message: `Successfully updated remotely updated file locally`, - type: SyncType.UPDATE - }); - } - } finally { - if (relativePath !== remoteVersion.relativePath) { - this.locks.unlockDocument(relativePath); - } + return; } + + if (relativePath !== remoteVersion.relativePath) { + // TODO: this can fail, that's bad + await this.operations.move( + // this can throw FileNotFoundError + relativePath, + remoteVersion.relativePath, + remoteVersion.documentId + ); + } + + // todo: why + await this.operations.create( + remoteVersion.relativePath, + contentBytes, + remoteVersion.documentId + ); + + await this.database.setDocument({ + documentId: remoteVersion.documentId, + relativePath: remoteVersion.relativePath, + parentVersionId: remoteVersion.vaultUpdateId, + hash: contentHash, + isDeleted: remoteVersion.isDeleted + }); + + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + source: SyncSource.PULL, + relativePath: remoteVersion.relativePath, + message: `Successfully updated remotely updated file locally`, + type: SyncType.UPDATE + }); } ); } - public async executeWhileHoldingFileLock( + public async executeSync( lockedPaths: RelativePath[], syncType: SyncType, syncSource: SyncSource, - fn: () => Promise - ): Promise { + fn: () => Promise + ): Promise { const relativePath = lockedPaths[lockedPaths.length - 1]; if (!this.settings.getSettings().isSyncEnabled) { @@ -498,31 +493,47 @@ export class UnrestrictedSyncer { ); return; } + if (!this.operations.isFileEligibleForSync(relativePath)) { - this.logger.info( - `File ${relativePath} is not eligible for syncing` - ); + this.history.addHistoryEntry({ + status: SyncStatus.ERROR, + relativePath, + message: `File ${relativePath} is not eligible for syncing`, + type: syncType + }); return; } + this.logger.debug( `Syncing ${relativePath} (${syncSource} - ${syncType})` ); - await Promise.all( - lockedPaths.map(this.locks.waitForDocumentLock.bind(this.locks)) - ); try { - await fn(); + if ( + (await this.operations.exists(relativePath)) && + (await this.operations.getFileSize(relativePath)) / // this can throw FileNotFoundError + 1024 / + 1024 > + this.settings.getSettings().maxFileSizeMB + ) { + this.history.addHistoryEntry({ + status: SyncStatus.ERROR, + relativePath, + message: `File size exceeds the maximum file size limit of ${ + this.settings.getSettings().maxFileSizeMB + }MB`, + type: syncType + }); + return; + } + + return await fn(); } catch (e) { if (e instanceof FileNotFoundError) { // A subsequent sync operation must have been creating to deal with this - this.history.addHistoryEntry({ - status: SyncStatus.NO_OP, - relativePath, - message: `Skip ${syncSource.toLocaleLowerCase()} file because it no longer exists when trying to ${syncType.toLocaleLowerCase()} it`, - type: syncType, - source: syncSource - }); + this.logger.info( + `Skip ${syncSource.toLocaleLowerCase()} file because it no longer exists when trying to ${syncType.toLocaleLowerCase()} it` + ); } else { this.history.addHistoryEntry({ status: SyncStatus.ERROR, @@ -533,8 +544,6 @@ export class UnrestrictedSyncer { }); throw e; } - } finally { - lockedPaths.forEach(this.locks.unlockDocument.bind(this.locks)); } } diff --git a/frontend/test-client/src/agent/mock-agent.ts b/frontend/test-client/src/agent/mock-agent.ts index 2163a503..fad989fb 100644 --- a/frontend/test-client/src/agent/mock-agent.ts +++ b/frontend/test-client/src/agent/mock-agent.ts @@ -46,27 +46,31 @@ export class MockAgent extends MockClient { ? "(online) " : "(offline)"; const formatted = `[${this.name} ${state}] ${logLine.timestamp.toISOString()} ${logLine.level} ${logLine.message}`; + + // HACK: we have to ensure the file has been synced if we want to change it offline without data loss + const historyEntry = /.*History entry: (.*.md).*/.exec( + logLine.message + ); + + if (historyEntry) { + this.doNotTouchWhileOffline = + this.doNotTouchWhileOffline.filter( + (file) => file !== historyEntry[1] + ); + } switch (logLine.level) { case LogLevel.ERROR: console.error(formatted); + // Let's not ignore errors - process.exit(1); + // eslint-disable-next-line @typescript-eslint/no-floating-promises + sleep(1000).then(() => process.exit(1)); + break; case LogLevel.WARNING: console.warn(formatted); break; case LogLevel.INFO: - // HACK: we have to ensure the file has been synced if we want to change it offline without data loss - const result = /.*History entry: (.*.md).*/.exec( - logLine.message - ); - if (result) { - this.doNotTouchWhileOffline = - this.doNotTouchWhileOffline.filter( - (file) => file !== result[1] - ); - } - console.info(formatted); break; case LogLevel.DEBUG: @@ -79,16 +83,17 @@ export class MockAgent extends MockClient { } public async act(): Promise { + this.assertAllContentIsPresentOnce(); + const options: (() => Promise)[] = [ this.createFileAction.bind(this), this.changeFetchChangesUpdateIntervalMsAction.bind(this) ]; - if ( - this.client.settings.getSettings().isSyncEnabled && - this.doNotTouchWhileOffline.length === 0 - ) { - options.push(this.disableSyncAction.bind(this)); + if (this.client.settings.getSettings().isSyncEnabled) { + if (this.doNotTouchWhileOffline.length === 0) { + options.push(this.disableSyncAction.bind(this)); + } } else { options.push(this.enableSyncAction.bind(this)); } diff --git a/frontend/test-client/src/cli.ts b/frontend/test-client/src/cli.ts index 7f8b29a4..26a0f23f 100644 --- a/frontend/test-client/src/cli.ts +++ b/frontend/test-client/src/cli.ts @@ -38,6 +38,8 @@ async function runTest({ ) ); } + // for debugging + (globalThis as any).clients = clients; try { await Promise.all(clients.map(async (client) => client.init())); @@ -78,34 +80,32 @@ async function runTest({ console.info(`Content check for ${client.name} passed`); }); - console.info(`Test passed with ${settings}`); + console.info(`Test passed ${settings}`); } catch (err) { - console.error(`Test failed with ${settings}`); + console.error(`Test failed ${settings}`); throw err; } } async function runTests(): Promise { const agentCounts = [2, 10]; - const jitterScaleInSeconds = [0.5, 3, 0]; + const jitterScaleInSeconds = [0, 0.5, 3]; const concurrencies = [1, 16]; const iterations = [50, 300]; - const doDeletes = [false, true]; + const doDeletes = [false]; for (const agentCount of agentCounts) { for (const concurrency of concurrencies) { for (const jitter of jitterScaleInSeconds) { for (const iteration of iterations) { for (const deleteFiles of doDeletes) { - while (true) { - await runTest({ - agentCount, - concurrency, - iterations: iteration, - doDeletes: deleteFiles, - jitterScaleInSeconds: jitter - }); - } + await runTest({ + agentCount, + concurrency, + iterations: iteration, + doDeletes: deleteFiles, + jitterScaleInSeconds: jitter + }); } } } @@ -113,11 +113,24 @@ async function runTests(): Promise { } } +process.on("uncaughtException", async (error) => { + console.error("Uncaught Exception:", error); + await sleep(1000); + process.exit(1); +}); + +process.on("unhandledRejection", async (reason, promise) => { + console.error("Unhandled Rejection:", reason); + await sleep(1000); + process.exit(1); +}); + runTests() .then(() => { process.exit(0); }) - .catch((err: unknown) => { + .catch(async (err: unknown) => { console.error(err); + await sleep(1000); process.exit(1); }); From a93c17711c74b733b7dc714504b8870a250a5c80 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 2 Mar 2025 14:24:22 +0000 Subject: [PATCH 02/58] Add debug --- backend/reconcile/src/diffs/myers.rs | 6 +++--- backend/reconcile/src/diffs/raw_operation.rs | 4 ++-- .../src/operation_transformation/merge_context.rs | 8 ++++---- backend/reconcile/src/tokenizer/token.rs | 6 +++--- backend/reconcile/src/utils/ordered_operation.rs | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/backend/reconcile/src/diffs/myers.rs b/backend/reconcile/src/diffs/myers.rs index e2f44989..cce51d54 100644 --- a/backend/reconcile/src/diffs/myers.rs +++ b/backend/reconcile/src/diffs/myers.rs @@ -38,7 +38,7 @@ use crate::{ /// execution time permitted before it bails and falls back to an approximation. pub fn diff(old: &[Token], new: &[Token]) -> Vec> where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { let max_d = (old.len() + new.len()).div_ceil(2) + 1; let mut vb = V::new(max_d); @@ -124,7 +124,7 @@ fn find_middle_snake( vb: &mut V, ) -> Option<(usize, usize)> where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { let n = old_range.len(); let m = new_range.len(); @@ -230,7 +230,7 @@ fn conquer( vb: &mut V, result: &mut Vec>, ) where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { // Check for common prefix let common_prefix_len = common_prefix_len(old, old_range.clone(), new, new_range.clone()); diff --git a/backend/reconcile/src/diffs/raw_operation.rs b/backend/reconcile/src/diffs/raw_operation.rs index 280460f6..bf970062 100644 --- a/backend/reconcile/src/diffs/raw_operation.rs +++ b/backend/reconcile/src/diffs/raw_operation.rs @@ -3,7 +3,7 @@ use crate::tokenizer::token::Token; #[derive(Debug, Clone, PartialEq)] pub enum RawOperation where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { Insert(Vec>), Delete(Vec>), @@ -12,7 +12,7 @@ where impl RawOperation where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { pub fn tokens(&self) -> &Vec> { match self { diff --git a/backend/reconcile/src/operation_transformation/merge_context.rs b/backend/reconcile/src/operation_transformation/merge_context.rs index 0bc3c34a..980389df 100644 --- a/backend/reconcile/src/operation_transformation/merge_context.rs +++ b/backend/reconcile/src/operation_transformation/merge_context.rs @@ -5,7 +5,7 @@ use crate::operation_transformation::Operation; #[derive(Clone)] pub struct MergeContext where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { last_operation: Option>, pub shift: i64, @@ -13,7 +13,7 @@ where impl Default for MergeContext where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { fn default() -> Self { MergeContext { @@ -25,7 +25,7 @@ where impl Debug for MergeContext where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("MergeContext") @@ -37,7 +37,7 @@ where impl MergeContext where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { pub fn last_operation(&self) -> Option<&Operation> { self.last_operation.as_ref() } diff --git a/backend/reconcile/src/tokenizer/token.rs b/backend/reconcile/src/tokenizer/token.rs index f723a2c2..b867bb20 100644 --- a/backend/reconcile/src/tokenizer/token.rs +++ b/backend/reconcile/src/tokenizer/token.rs @@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize}; #[derive(Debug, Clone)] pub struct Token where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { normalised: T, original: String, @@ -25,7 +25,7 @@ impl From<&str> for Token { impl Token where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { pub fn new(normalised: T, original: String) -> Self { Token { @@ -43,7 +43,7 @@ where impl PartialEq for Token where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { fn eq(&self, other: &Self) -> bool { self.normalised == other.normalised } } diff --git a/backend/reconcile/src/utils/ordered_operation.rs b/backend/reconcile/src/utils/ordered_operation.rs index 17229d2e..116b6372 100644 --- a/backend/reconcile/src/utils/ordered_operation.rs +++ b/backend/reconcile/src/utils/ordered_operation.rs @@ -7,7 +7,7 @@ use crate::operation_transformation::Operation; #[derive(Debug, Clone, PartialEq)] pub struct OrderedOperation where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { pub order: usize, pub operation: Operation, From d7ae0a781df02d5c063fef141b2e421c8f10bc75 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 2 Mar 2025 14:54:57 +0000 Subject: [PATCH 03/58] Dedupe inserts --- .../reconcile/src/operation_transformation.rs | 25 ++++- .../src/operation_transformation/operation.rs | 37 ++++--- backend/reconcile/src/utils.rs | 2 +- .../src/utils/find_common_overlap.rs | 71 ------------ .../find_longest_prefix_contained_within.rs | 103 ++++++++++++++++++ 5 files changed, 145 insertions(+), 93 deletions(-) delete mode 100644 backend/reconcile/src/utils/find_common_overlap.rs create mode 100644 backend/reconcile/src/utils/find_longest_prefix_contained_within.rs diff --git a/backend/reconcile/src/operation_transformation.rs b/backend/reconcile/src/operation_transformation.rs index ef9a5e81..1f34fa12 100644 --- a/backend/reconcile/src/operation_transformation.rs +++ b/backend/reconcile/src/operation_transformation.rs @@ -37,7 +37,7 @@ pub fn reconcile_with_tokenizer( tokenizer: &Tokenizer, ) -> String where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { let left_operations = EditedText::from_strings_with_tokenizer(original, left, tokenizer); let right_operations = EditedText::from_strings_with_tokenizer(original, right, tokenizer); @@ -120,9 +120,6 @@ mod test { "hi, my friend!", ); - // test_merge_both_ways("hello world", "world !", "hi hello world", "hi world - // !"); - test_merge_both_ways( "both delete the same word", "both the same word", @@ -147,7 +144,25 @@ mod test { ); } - #[ignore = "it's too slow"] + #[test] + fn test_reconcile_idempotent_inserts() { + // Both inserted the same prefix; this should get deduped + test_merge_both_ways( + "hi ", + "hi there ", + "hi there my friend", + "hi there my friend", + ); + + // The prefix of the 2nd appears on the 1st so it shouldn't get duplicated + test_merge_both_ways( + "hi ", + "hi there you ", + "hi there my friend", + "hi there you my friend", + ); + } + #[test_matrix( [ "pride_and_prejudice.txt", "romeo_and_juliet.txt", diff --git a/backend/reconcile/src/operation_transformation/operation.rs b/backend/reconcile/src/operation_transformation/operation.rs index c19265b5..5de0141b 100644 --- a/backend/reconcile/src/operation_transformation/operation.rs +++ b/backend/reconcile/src/operation_transformation/operation.rs @@ -2,14 +2,18 @@ use core::{ fmt::{Debug, Display}, ops::Range, }; +use std::cmp::min; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; use super::merge_context::MergeContext; use crate::{ + utils::{ + find_longest_prefix_contained_within::find_longest_prefix_contained_within, + string_builder::StringBuilder, + }, Token, - utils::{find_common_overlap::find_common_overlap, string_builder::StringBuilder}, }; /// Represents a change that can be applied to a text document. @@ -19,7 +23,7 @@ use crate::{ #[derive(Clone, PartialEq)] pub enum Operation where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { Insert { index: usize, @@ -37,7 +41,7 @@ where impl Operation where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { /// Creates an insert operation with the given index and text. /// If the text is empty (meaning that the operation would be a no-op), @@ -212,17 +216,20 @@ where .. }), ) => { - let offset_in_tokens = find_common_overlap(previous_inserted_text, &text); - let trimmed_length_in_tokens = previous_inserted_text.len() - offset_in_tokens; - let trimmed_length = previous_inserted_text + // In case the current insert's prefix appears in the previously inserted text, + // we can trim the current insert to only include the non-overlapping part. + // This way, we don't end up duplicating text. + let offset_in_tokens = + find_longest_prefix_contained_within(previous_inserted_text, &text); + let offset_in_length = text .iter() - .skip(offset_in_tokens) + .take(offset_in_tokens) .map(Token::get_original_length) .sum::(); let trimmed_operation = - Operation::create_insert(index, text[trimmed_length_in_tokens..].to_vec()); + Operation::create_insert(index, text[offset_in_tokens..].to_vec()); - affecting_context.shift -= trimmed_length as i64; + affecting_context.shift -= offset_in_length as i64; produced_context.shift += trimmed_operation .as_ref() .map(Operation::len) @@ -297,7 +304,7 @@ where impl Display for Operation where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { match self { @@ -341,7 +348,7 @@ where impl Debug for Operation where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!(f, "{self}") } } @@ -355,11 +362,9 @@ mod tests { #[test] #[should_panic] fn test_shifting_error() { - insta::assert_debug_snapshot!( - Operation::create_insert(1, vec!["hi".into()]) - .unwrap() - .with_shifted_index(-2) - ); + insta::assert_debug_snapshot!(Operation::create_insert(1, vec!["hi".into()]) + .unwrap() + .with_shifted_index(-2)); } #[test] diff --git a/backend/reconcile/src/utils.rs b/backend/reconcile/src/utils.rs index c0c3c33d..8461b5ff 100644 --- a/backend/reconcile/src/utils.rs +++ b/backend/reconcile/src/utils.rs @@ -1,6 +1,6 @@ pub mod common_prefix_len; pub mod common_suffix_len; -pub mod find_common_overlap; +pub mod find_longest_prefix_contained_within; pub mod merge_iters; pub mod ordered_operation; pub mod side; diff --git a/backend/reconcile/src/utils/find_common_overlap.rs b/backend/reconcile/src/utils/find_common_overlap.rs deleted file mode 100644 index ac586b81..00000000 --- a/backend/reconcile/src/utils/find_common_overlap.rs +++ /dev/null @@ -1,71 +0,0 @@ -use crate::Token; - -/// Given two lists of tokens, returns the offset in the first (old) list from -/// which the two lists have the same tokens until the end of the first list. -/// Thus, the suffix of the old list from the offset to the end is equal to a -/// prefix of the new list. -/// -/// If there is no overlap, the function returns the maxmium offset, the length -/// of the old list. -/// -/// ## Example -/// -/// ```not_rust -/// old: [0, 1, 9, 0, 2, 5] -/// new: [9, 0, 2, 5, 1] -/// ``` -/// > results in an offset of 2 -pub fn find_common_overlap(old: &[Token], new: &[Token]) -> usize -where - T: PartialEq + Clone, -{ - let minimum_offset = old.len().saturating_sub(new.len()); - for offset in minimum_offset..old.len() { - if old.iter().skip(offset).zip(new.iter()).all(|(a, b)| a == b) { - return offset; - } - } - - old.len() -} - -#[cfg(test)] -mod tests { - use pretty_assertions::assert_eq; - - use super::*; - - #[test] - fn test_common_overlap() { - assert_eq!(find_common_overlap(&["".into()], &["".into()]), 0); - - assert_eq!( - find_common_overlap( - &["a".into(), "b".into(), "c".into()], - &["b".into(), "c".into(), "a".into()] - ), - 1 - ); - - assert_eq!( - find_common_overlap( - &["a".into(), "a".into(), "a".into()], - &["a".into(), "b".into(), "c".into()] - ), - 2 - ); - - assert_eq!( - find_common_overlap( - &["a".into(), "b".into(), "c".into()], - &["d".into(), "e".into(), "a".into()] - ), - 3 - ); - - assert_eq!( - find_common_overlap(&["a".into(), "a".into()], &["a".into()]), - 1 - ); - } -} diff --git a/backend/reconcile/src/utils/find_longest_prefix_contained_within.rs b/backend/reconcile/src/utils/find_longest_prefix_contained_within.rs new file mode 100644 index 00000000..eb4b8264 --- /dev/null +++ b/backend/reconcile/src/utils/find_longest_prefix_contained_within.rs @@ -0,0 +1,103 @@ +use crate::Token; + +/// Given two lists of tokens, returns `length` where `old` list somewhere +/// within contains the `length` prefix of the `new` list. +/// +/// ## Example +/// +/// ```not_rust +/// old: [0, 1, 9, 0, 2, 5] +/// new: [9, 0, 2, 5, 1] +/// ``` +/// > results in an length of 4 +/// +/// +/// ```not_rust +/// old: [0, 1, 9, 0, 2, 5] +/// new: [0, 2] +/// ``` +/// > results in an length of 2 +/// +/// ```not_rust +/// old: [0, 1, 9, 0, 2, 5] +/// new: [0, 4] +/// ``` +/// > results in an length of 1 +pub fn find_longest_prefix_contained_within(old: &[Token], new: &[Token]) -> usize +where + T: PartialEq + Clone + std::fmt::Debug, +{ + let max_possible = new.len().min(old.len()); + + for len in (1..=max_possible).rev() { + let prefix = &new[..len]; + if old.windows(len).any(|window| window == prefix) { + return len; + } + } + + 0 +} + +#[cfg(test)] +mod tests { + use pretty_assertions::assert_eq; + + use super::*; + + #[test] + fn test_common_overlap() { + assert_eq!( + find_longest_prefix_contained_within(&["".into()], &["".into()]), + 1 + ); + + assert_eq!( + find_longest_prefix_contained_within( + &["a".into(), "b".into(), "c".into()], + &["b".into(), "c".into(), "a".into()] + ), + 2 + ); + + assert_eq!( + find_longest_prefix_contained_within( + &["a".into(), "b".into(), "c".into()], + &["b".into(), "c".into()] + ), + 2 + ); + + assert_eq!( + find_longest_prefix_contained_within( + &["a".into(), "b".into(), "c".into()], + &["b".into()] + ), + 1 + ); + + assert_eq!( + find_longest_prefix_contained_within( + &["a".into(), "b".into(), "c".into(), "b".into(), "a".into()], + &["b".into(), "a".into()] + ), + 2 + ); + + assert_eq!( + find_longest_prefix_contained_within( + &["a".into(), "a".into(), "a".into()], + &["a".into(), "b".into(), "c".into()] + ), + 1 + ); + + assert_eq!( + find_longest_prefix_contained_within( + &["a".into(), "b".into(), "c".into()], + &["d".into(), "e".into(), "a".into()] + ), + 0 + ); + } +} From 667b324a88c27d2bb7abbcfcdb51f28e7811ecbc Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 2 Mar 2025 15:10:15 +0000 Subject: [PATCH 04/58] Add deterministic ordering --- .../reconcile/src/operation_transformation.rs | 8 ++-- .../operation_transformation/edited_text.rs | 11 +++-- .../src/operation_transformation/operation.rs | 41 ++++++++++++++++--- 3 files changed, 47 insertions(+), 13 deletions(-) diff --git a/backend/reconcile/src/operation_transformation.rs b/backend/reconcile/src/operation_transformation.rs index 1f34fa12..3f83197a 100644 --- a/backend/reconcile/src/operation_transformation.rs +++ b/backend/reconcile/src/operation_transformation.rs @@ -150,16 +150,16 @@ mod test { test_merge_both_ways( "hi ", "hi there ", - "hi there my friend", - "hi there my friend", + "hi there my friend ", + "hi there my friend ", ); // The prefix of the 2nd appears on the 1st so it shouldn't get duplicated test_merge_both_ways( "hi ", "hi there you ", - "hi there my friend", - "hi there you my friend", + "hi there my friend ", + "hi there you my friend ", ); } diff --git a/backend/reconcile/src/operation_transformation/edited_text.rs b/backend/reconcile/src/operation_transformation/edited_text.rs index 32bda1b2..4052485c 100644 --- a/backend/reconcile/src/operation_transformation/edited_text.rs +++ b/backend/reconcile/src/operation_transformation/edited_text.rs @@ -25,7 +25,7 @@ use crate::{ #[derive(Debug, Clone, PartialEq, Default)] pub struct EditedText<'a, T> where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { text: &'a str, operations: Vec>, @@ -46,7 +46,7 @@ impl<'a> EditedText<'a, String> { impl<'a, T> EditedText<'a, T> where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { /// Create an `EditedText` from the given original (old) and updated (new) /// strings. The returned `EditedText` represents the changes from the @@ -207,9 +207,12 @@ where |(operation, _)| { ( operation.order, - // Operations on left and right must come in the same order so that + // Operations on the left and right must come in the same order so that // inserts can be merged with other inserts and deletes with deletes. usize::from(matches!(operation.operation, Operation::Delete { .. })), + // Make sure that the ordering is deterministic regardless which text + // is left or right. + operation.operation.get_hash(), ) }, ) @@ -282,7 +285,7 @@ mod tests { let original = "hello world! ..."; let left = "Hello world! I'm Andras."; let right = "Hello world! How are you?"; - let expected = "Hello world! I'm Andras.How are you?"; + let expected = "Hello world! How are you?I'm Andras."; let operations_1 = EditedText::from_strings(original, left); let operations_2 = EditedText::from_strings(original, right); diff --git a/backend/reconcile/src/operation_transformation/operation.rs b/backend/reconcile/src/operation_transformation/operation.rs index 5de0141b..a985ad7b 100644 --- a/backend/reconcile/src/operation_transformation/operation.rs +++ b/backend/reconcile/src/operation_transformation/operation.rs @@ -2,18 +2,18 @@ use core::{ fmt::{Debug, Display}, ops::Range, }; -use std::cmp::min; +use std::hash::{DefaultHasher, Hash, Hasher}; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; use super::merge_context::MergeContext; use crate::{ + Token, utils::{ find_longest_prefix_contained_within::find_longest_prefix_contained_within, string_builder::StringBuilder, }, - Token, }; /// Represents a change that can be applied to a text document. @@ -39,6 +39,28 @@ where }, } +impl Hash for Operation +where + T: PartialEq + Clone + std::fmt::Debug, +{ + fn hash(&self, state: &mut H) { + match self { + Operation::Insert { index, text } => { + index.hash(state); + text.iter().for_each(|token| token.original().hash(state)); + } + Operation::Delete { + index, + deleted_character_count, + .. + } => { + index.hash(state); + deleted_character_count.hash(state); + } + }; + } +} + impl Operation where T: PartialEq + Clone + std::fmt::Debug, @@ -300,6 +322,13 @@ where } } } + + /// Gets the hash of the operation based on the indexes and original text. + pub fn get_hash(&self) -> u64 { + let mut hasher = DefaultHasher::new(); + self.hash(&mut hasher); + hasher.finish() + } } impl Display for Operation @@ -362,9 +391,11 @@ mod tests { #[test] #[should_panic] fn test_shifting_error() { - insta::assert_debug_snapshot!(Operation::create_insert(1, vec!["hi".into()]) - .unwrap() - .with_shifted_index(-2)); + insta::assert_debug_snapshot!( + Operation::create_insert(1, vec!["hi".into()]) + .unwrap() + .with_shifted_index(-2) + ); } #[test] From bf8d00c5e24a5381cfb8246d8f179d78a7e91ee8 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 2 Mar 2025 17:53:21 +0000 Subject: [PATCH 05/58] Fix whitespaces --- .../reconcile/src/operation_transformation.rs | 5 +- .../operation_transformation/edited_text.rs | 5 +- .../src/operation_transformation/operation.rs | 11 +---- backend/reconcile/src/tokenizer/token.rs | 7 +-- .../reconcile/src/tokenizer/word_tokenizer.rs | 47 +++++++++++++++++-- 5 files changed, 53 insertions(+), 22 deletions(-) diff --git a/backend/reconcile/src/operation_transformation.rs b/backend/reconcile/src/operation_transformation.rs index 3f83197a..aa891d72 100644 --- a/backend/reconcile/src/operation_transformation.rs +++ b/backend/reconcile/src/operation_transformation.rs @@ -73,7 +73,8 @@ mod test { "original_1 edit_1 original_3", ); - // One deleted a large range, the other deleted subranges and inserted as well + // One deleted a large range, the other deleted subranges and inserted as + // well test_merge_both_ways( "original_1 original_2 original_3 original_4 original_5", "original_1 original_5", @@ -161,6 +162,8 @@ mod test { "hi there my friend ", "hi there you my friend ", ); + + test_merge_both_ways("a", "a b c", "a b c d", "a b c d"); } #[test_matrix( [ diff --git a/backend/reconcile/src/operation_transformation/edited_text.rs b/backend/reconcile/src/operation_transformation/edited_text.rs index 4052485c..87a5df40 100644 --- a/backend/reconcile/src/operation_transformation/edited_text.rs +++ b/backend/reconcile/src/operation_transformation/edited_text.rs @@ -65,7 +65,6 @@ where Self::new( original, - // Self::cook_operations(diff), Self::cook_operations(Self::elongate_operations(diff)).collect(), ) } @@ -191,7 +190,7 @@ where pub fn merge(self, other: Self) -> Self { debug_assert_eq!( self.text, other.text, - "EditedText-s must be derived from the same text to be mergable" + "`EditedText`-s must be derived from the same text to be mergable" ); let mut left_merge_context = MergeContext::default(); @@ -285,7 +284,7 @@ mod tests { let original = "hello world! ..."; let left = "Hello world! I'm Andras."; let right = "Hello world! How are you?"; - let expected = "Hello world! How are you?I'm Andras."; + let expected = "Hello world! I'm Andras. How are you?"; let operations_1 = EditedText::from_strings(original, left); let operations_2 = EditedText::from_strings(original, right); diff --git a/backend/reconcile/src/operation_transformation/operation.rs b/backend/reconcile/src/operation_transformation/operation.rs index a985ad7b..ffc4f7d6 100644 --- a/backend/reconcile/src/operation_transformation/operation.rs +++ b/backend/reconcile/src/operation_transformation/operation.rs @@ -107,15 +107,8 @@ where }) } - /// Tries to apply the operation to the given `ropey::Rope` text, returning - /// the modified text. - /// - /// # Errors - /// - /// Returns a `SyncLibError::OperationApplicationError` if the operation - /// cannot be applied. - /// - /// # Panics + /// Applies the operation to the given `StringBuilder`, returning the + /// modified `StringBuilder`. /// /// When compiled in debug mode, panics if a delete operation is attempted /// on a range of text that does not match the text to be deleted. diff --git a/backend/reconcile/src/tokenizer/token.rs b/backend/reconcile/src/tokenizer/token.rs index b867bb20..ab521a71 100644 --- a/backend/reconcile/src/tokenizer/token.rs +++ b/backend/reconcile/src/tokenizer/token.rs @@ -15,12 +15,7 @@ where } impl From<&str> for Token { - fn from(s: &str) -> Self { - Token { - normalised: s.to_owned(), - original: s.to_owned(), - } - } + fn from(s: &str) -> Self { Token::new(s.trim().to_owned(), s.to_owned()) } } impl Token diff --git a/backend/reconcile/src/tokenizer/word_tokenizer.rs b/backend/reconcile/src/tokenizer/word_tokenizer.rs index 3449cba2..37d748b3 100644 --- a/backend/reconcile/src/tokenizer/word_tokenizer.rs +++ b/backend/reconcile/src/tokenizer/word_tokenizer.rs @@ -1,7 +1,48 @@ use super::token::Token; +/// Splits on whitespace keeping the leading whitespace. +/// +/// +/// ## Example +/// +/// "Hi there!" -> ["Hi", " there!"] pub fn word_tokenizer(text: &str) -> Vec> { - text.split_inclusive(char::is_whitespace) - .map(|s| Token::new(s.to_owned(), s.to_owned())) - .collect() + let mut result: Vec> = Vec::new(); + + let mut last_whitespace = 0; + let mut previous_char_is_whitespace = true; + + for (i, c) in text.char_indices() { + let is_current_char_whitespace = c.is_whitespace(); + if !previous_char_is_whitespace && is_current_char_whitespace { + result.push(text[last_whitespace..i].into()); + last_whitespace = i; + } + + previous_char_is_whitespace = is_current_char_whitespace; + } + + if last_whitespace < text.len() { + result.push(text[last_whitespace..].into()); + } + + result +} + +#[cfg(test)] +mod tests { + use insta::assert_debug_snapshot; + + use super::*; + + #[test] + fn test_with_snapshots() { + assert_debug_snapshot!(word_tokenizer("Hi there!")); + + assert_debug_snapshot!(word_tokenizer("")); + + assert_debug_snapshot!(word_tokenizer(" what? ")); + + assert_debug_snapshot!(word_tokenizer(" hello, \nwhere are you?")); + } } From 24206cabfeaf90f52c8885b6cdf19403a4d08745 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 2 Mar 2025 17:53:36 +0000 Subject: [PATCH 06/58] Update insta --- ...ted_text__tests__calculate_operations.snap | 12 +++++----- ...rd_tokenizer__tests__with_snapshots-2.snap | 6 +++++ ...rd_tokenizer__tests__with_snapshots-3.snap | 15 ++++++++++++ ...rd_tokenizer__tests__with_snapshots-4.snap | 23 +++++++++++++++++++ ...word_tokenizer__tests__with_snapshots.snap | 15 ++++++++++++ 5 files changed, 65 insertions(+), 6 deletions(-) create mode 100644 backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots-2.snap create mode 100644 backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots-3.snap create mode 100644 backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots-4.snap create mode 100644 backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots.snap diff --git a/backend/reconcile/src/operation_transformation/snapshots/reconcile__operation_transformation__edited_text__tests__calculate_operations.snap b/backend/reconcile/src/operation_transformation/snapshots/reconcile__operation_transformation__edited_text__tests__calculate_operations.snap index e8a04870..0630f986 100644 --- a/backend/reconcile/src/operation_transformation/snapshots/reconcile__operation_transformation__edited_text__tests__calculate_operations.snap +++ b/backend/reconcile/src/operation_transformation/snapshots/reconcile__operation_transformation__edited_text__tests__calculate_operations.snap @@ -8,19 +8,19 @@ EditedText { operations: [ OrderedOperation { order: 0, - operation: , + operation: , }, OrderedOperation { order: 0, - operation: , + operation: , }, OrderedOperation { - order: 21, - operation: , + order: 20, + operation: , }, OrderedOperation { - order: 21, - operation: , + order: 20, + operation: , }, ], } diff --git a/backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots-2.snap b/backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots-2.snap new file mode 100644 index 00000000..892e524c --- /dev/null +++ b/backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots-2.snap @@ -0,0 +1,6 @@ +--- +source: reconcile/src/tokenizer/word_tokenizer.rs +expression: "word_tokenizer(\"\")" +snapshot_kind: text +--- +[] diff --git a/backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots-3.snap b/backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots-3.snap new file mode 100644 index 00000000..58d749ef --- /dev/null +++ b/backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots-3.snap @@ -0,0 +1,15 @@ +--- +source: reconcile/src/tokenizer/word_tokenizer.rs +expression: "word_tokenizer(\" what? \")" +snapshot_kind: text +--- +[ + Token { + normalised: "what?", + original: " what?", + }, + Token { + normalised: "", + original: " ", + }, +] diff --git a/backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots-4.snap b/backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots-4.snap new file mode 100644 index 00000000..4c28a7f3 --- /dev/null +++ b/backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots-4.snap @@ -0,0 +1,23 @@ +--- +source: reconcile/src/tokenizer/word_tokenizer.rs +expression: "word_tokenizer(\" hello, \\nwhere are you?\")" +snapshot_kind: text +--- +[ + Token { + normalised: "hello,", + original: " hello,", + }, + Token { + normalised: "where", + original: " \nwhere", + }, + Token { + normalised: "are", + original: " are", + }, + Token { + normalised: "you?", + original: " you?", + }, +] diff --git a/backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots.snap b/backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots.snap new file mode 100644 index 00000000..206c7fee --- /dev/null +++ b/backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots.snap @@ -0,0 +1,15 @@ +--- +source: reconcile/src/tokenizer/word_tokenizer.rs +expression: "word_tokenizer(\"Hi there!\")" +snapshot_kind: text +--- +[ + Token { + normalised: "Hi", + original: "Hi", + }, + Token { + normalised: "there!", + original: " there!", + }, +] From 8cdad79160c8b0b02e2e86cf425c79bd9182141f Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 2 Mar 2025 17:53:49 +0000 Subject: [PATCH 07/58] Add integration test script --- frontend/test-client/run.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100755 frontend/test-client/run.sh diff --git a/frontend/test-client/run.sh b/frontend/test-client/run.sh new file mode 100755 index 00000000..892be317 --- /dev/null +++ b/frontend/test-client/run.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e + +npm run build + +pids=() +for i in {1..10}; do + node dist/cli.js 2>&1 | tee "log_${i}.log" & + pids+=($!) +done + +trap 'kill ${pids[@]} 2>/dev/null' SIGINT SIGTERM + +for pid in ${pids[@]}; do + if ! wait $pid; then + kill ${pids[@]} 2>/dev/null + echo "Process $pid failed, see log_$(echo ${pids[@]} | tr ' ' '\n' | grep -n "^$pid$" | cut -d: -f1).log" + exit 1 + fi +done From c250e82998d13ccdcf7095ba0fb472e8ef49adbf Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 2 Mar 2025 18:27:02 +0000 Subject: [PATCH 08/58] Rename --- ...-matching-file-based-on-hash.ts => find-matching-file.ts} | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) rename frontend/sync-client/src/utils/{find-matching-file-based-on-hash.ts => find-matching-file.ts} (59%) diff --git a/frontend/sync-client/src/utils/find-matching-file-based-on-hash.ts b/frontend/sync-client/src/utils/find-matching-file.ts similarity index 59% rename from frontend/sync-client/src/utils/find-matching-file-based-on-hash.ts rename to frontend/sync-client/src/utils/find-matching-file.ts index 6a247f5f..27a4c875 100644 --- a/frontend/sync-client/src/utils/find-matching-file-based-on-hash.ts +++ b/frontend/sync-client/src/utils/find-matching-file.ts @@ -1,7 +1,8 @@ -import type { DocumentMetadata, RelativePath } from "src/persistence/database"; +import type { DocumentMetadata, RelativePath } from "../persistence/database"; import { EMPTY_HASH } from "./hash"; -export function findMatchingFileBasedOnHash( +// TODO: make this smarter so that offline files can be renamed & edited at the same time +export function findMatchingFile( contentHash: string, candidates: [RelativePath, DocumentMetadata][] ): [RelativePath, DocumentMetadata] | undefined { From ec54d0fdb345b5ec775890eee76f76d79c90fd37 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 2 Mar 2025 21:39:09 +0000 Subject: [PATCH 09/58] Add test --- backend/reconcile/src/operation_transformation.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/backend/reconcile/src/operation_transformation.rs b/backend/reconcile/src/operation_transformation.rs index aa891d72..30e32502 100644 --- a/backend/reconcile/src/operation_transformation.rs +++ b/backend/reconcile/src/operation_transformation.rs @@ -164,6 +164,12 @@ mod test { ); test_merge_both_ways("a", "a b c", "a b c d", "a b c d"); + + test_merge_both_ways( + " |7ca2b36d-6ee7-49eb-8eb1-d77e4cc1a001| ", + " |7ca2b36d-6ee7-49eb-8eb1-d77e4cc1a001| |cd9195cc-103a-4f13-90c8-4fba0ba421ee| |d39156cc-cfd6-42a8-b70a-75020896069d| |fbad794c-9c47-41f2-a343-490284ecb5a0| |dup| ", + " |7ca2b36d-6ee7-49eb-8eb1-d77e4cc1a001| |cd9195cc-103a-4f13-90c8-4fba0ba421ee| |dup| ", + " |7ca2b36d-6ee7-49eb-8eb1-d77e4cc1a001| |cd9195cc-103a-4f13-90c8-4fba0ba421ee| |d39156cc-cfd6-42a8-b70a-75020896069d| |fbad794c-9c47-41f2-a343-490284ecb5a0| |dup| |dup| "); } #[test_matrix( [ From 054d109ef8f5d8920238116b4309d36584c35a63 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 9 Mar 2025 09:07:18 +0000 Subject: [PATCH 10/58] Working for non-deletes --- .../file-operations/file-operations.test.ts | 11 +- .../src/file-operations/file-operations.ts | 45 ++- .../sync-client/src/persistence/database.ts | 310 +++++++------- .../sync-client/src/services/sync-service.ts | 3 + frontend/sync-client/src/sync-client.ts | 2 +- .../sync-client/src/sync-operations/syncer.ts | 321 ++++++++------- .../sync-operations/unrestricted-syncer.ts | 378 ++++++------------ .../sync-client/src/utils/create-promise.ts | 15 + .../src/utils/find-matching-file.ts | 8 +- frontend/test-client/run.sh | 62 ++- frontend/test-client/src/agent/mock-agent.ts | 2 +- frontend/test-client/src/cli.ts | 20 +- 12 files changed, 574 insertions(+), 603 deletions(-) create mode 100644 frontend/sync-client/src/utils/create-promise.ts diff --git a/frontend/sync-client/src/file-operations/file-operations.test.ts b/frontend/sync-client/src/file-operations/file-operations.test.ts index 9d2945d5..b1299098 100644 --- a/frontend/sync-client/src/file-operations/file-operations.test.ts +++ b/frontend/sync-client/src/file-operations/file-operations.test.ts @@ -2,6 +2,7 @@ import type { FileSystemOperations } from "sync-client"; import type { Database, DocumentMetadata, + DocumentRecord, RelativePath } from "../persistence/database"; import { FileOperations } from "./file-operations"; @@ -10,16 +11,16 @@ import { assertSetContainsExactly } from "../utils/assert-set-contains-exactly"; describe("File operations", () => { class MockDatabase { - public async move( + public move( _oldRelativePath: RelativePath, _newRelativePath: RelativePath - ): Promise { + ): void { // this is called but irrelevant for this mock } - public getResolvedDocument( - _relativePath: RelativePath | undefined - ): DocumentMetadata | undefined { + public getDocumentByRelativePath( + _find: RelativePath + ): DocumentRecord | undefined { return undefined; } } diff --git a/frontend/sync-client/src/file-operations/file-operations.ts b/frontend/sync-client/src/file-operations/file-operations.ts index 73818786..34ed19d9 100644 --- a/frontend/sync-client/src/file-operations/file-operations.ts +++ b/frontend/sync-client/src/file-operations/file-operations.ts @@ -71,27 +71,30 @@ export class FileOperations { `Didn't expect ${path} to exist, deconflicting by moving it to '${deconflictedPath}'` ); - const existingMetadata = this.database.getResolvedDocument(path); + const document = this.database.getDocumentByRelativePath(path); this.logger.debug( - `Existing metadata for ${path}: ${JSON.stringify(existingMetadata)}` + `Existing metadata for ${path}: ${JSON.stringify(document?.metadata)}` ); + + this.logger.debug( + `We need to save what's at ${path} to ${deconflictedPath}` + ); + if ( - existingMetadata === undefined || - existingMetadata.isDeleted || - existingMetadata.documentId !== documentId || - !documentId + document?.metadata !== undefined && + document.metadata.documentId === documentId ) { - this.logger.debug( - `We need to save what's at ${path} to ${deconflictedPath}` - ); - await this.move(path, deconflictedPath, documentId); - await this.database.move(path, deconflictedPath); - } else { // This can happen if the document got moved both locally and remotely // to the same file path. In this case, we shouldn't deconflict, however, // we also can't overwrite otherwise we'd lose changes. throw new FileNotFoundError(path); } + + this.logger.debug( + `We need to save what's at ${path} to ${deconflictedPath}` + ); + await this.move(path, deconflictedPath, documentId); + // this.database.move(path, deconflictedPath); } else { await this.createParentDirectories(path); } @@ -135,7 +138,7 @@ export class FileOperations { currentText = currentText.replace(/\r\n/g, "\n"); if (currentText !== expectedText) { this.logger.debug( - `Performing a 3-way merge for ${path} with the expected content` + `Performing a 3-way merge for ${path} with the expected content:\n${expectedText}` ); return mergeText(expectedText, currentText, newText); @@ -174,21 +177,21 @@ export class FileOperations { this.logger.debug( `Conflict when moving '${oldPath}' to '${newPath}', the latter already exists, deconflicting by moving it to '${deconflictedPath}'` ); - const existingMetadata = this.database.getResolvedDocument(newPath); + + const document = this.database.getDocumentByRelativePath(newPath); + if ( - existingMetadata === undefined || - existingMetadata.isDeleted || - existingMetadata.documentId !== documentId || - !documentId + document?.metadata !== undefined && + document.metadata.documentId === documentId ) { - await this.move(newPath, deconflictedPath, documentId); - await this.database.move(oldPath, newPath); - } else { // This can happen if the document got moved both locally and remotely // to the same file path. In this case, we shouldn't deconflict, however, // we also can't overwrite otherwise we'd lose changes. throw new FileNotFoundError(newPath); } + + await this.move(newPath, deconflictedPath, documentId); + // this.database.move(oldPath, newPath); } else { await this.createParentDirectories(newPath); } diff --git a/frontend/sync-client/src/persistence/database.ts b/frontend/sync-client/src/persistence/database.ts index 705f3aea..c529889c 100644 --- a/frontend/sync-client/src/persistence/database.ts +++ b/frontend/sync-client/src/persistence/database.ts @@ -1,3 +1,5 @@ +import type { Logger } from "../tracing/logger"; + export type VaultUpdateId = number; export type DocumentId = string; export type RelativePath = string; @@ -8,20 +10,28 @@ export interface DocumentMetadata { hash: string; isDeleted: boolean; } - -import type { Logger } from "src/tracing/logger"; +export interface StoredDocumentMetadata { + relativePath: RelativePath; + parentVersionId: VaultUpdateId; + documentId: DocumentId; + hash: string; + isDeleted: boolean; +} export interface StoredDatabase { - documents: Record; + documents: StoredDocumentMetadata[]; lastSeenUpdateId: VaultUpdateId | undefined; } -export class Database { - private documents = new Map< - RelativePath, - DocumentMetadata | Promise - >(); +export interface DocumentRecord { + identity: symbol; + relativePath: RelativePath; + metadata: DocumentMetadata | undefined; + updates: Promise[]; +} +export class Database { + private documents: DocumentRecord[]; private lastSeenUpdateId: VaultUpdateId | undefined; public constructor( @@ -30,16 +40,17 @@ export class Database { private readonly saveData: (data: StoredDatabase) => Promise ) { initialState ??= {}; - if (initialState.documents) { - for (const [relativePath, metadata] of Object.entries( - initialState.documents - )) { - this.documents.set(relativePath, metadata); - } - } - this.ensureConsistency(); - this.logger.debug(`Loaded ${this.documents.size} documents`); + this.documents = + initialState.documents?.map(({ relativePath, ...metadata }) => ({ + relativePath, + identity: Symbol(), + metadata, + updates: [] + })) ?? []; + + this.ensureConsistency(); + this.logger.debug(`Loaded ${this.documents.length} documents`); this.lastSeenUpdateId = initialState.lastSeenUpdateId; this.logger.debug( @@ -48,62 +59,29 @@ export class Database { } public get length(): number { - return this.documents.size; + return this.documents.length; } - public get resolvedDocuments(): [RelativePath, DocumentMetadata][] { - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion - return Array.from(this.documents.entries()).filter( - ([_, metadata]) => !(metadata instanceof Promise) - ) as [RelativePath, DocumentMetadata][]; + public get resolvedDocuments(): DocumentRecord[] { + return this.documents.filter(({ metadata }) => metadata !== undefined); } public getLastSeenUpdateId(): VaultUpdateId | undefined { return this.lastSeenUpdateId; } - public async setLastSeenUpdateId( - value: VaultUpdateId | undefined - ): Promise { + public setLastSeenUpdateId(value: VaultUpdateId | undefined): void { this.lastSeenUpdateId = value; - await this.save(); + this.save(); } - public async resetSyncState(): Promise { - this.documents = new Map(); + public resetSyncState(): void { + this.documents = []; this.lastSeenUpdateId = 0; - await this.save(); + this.save(); } - public getDocumentByDocumentId( - documentId: DocumentId - ): [RelativePath, DocumentMetadata] | undefined { - return this.resolvedDocuments.find( - ([_, metadata]) => metadata.documentId === documentId - ); - } - - public getDocumentByIdentity( - document: - | DocumentMetadata - | Promise - | undefined - ): - | [ - RelativePath, - DocumentMetadata | Promise - ] - | undefined { - if (document === undefined) { - return undefined; - } - - return Array.from(this.documents.entries()).find( - ([_, metadata]) => metadata === document - ); - } - - public async setDocument({ + public setDocument({ documentId, relativePath, parentVersionId, @@ -115,84 +93,142 @@ export class Database { parentVersionId: VaultUpdateId; hash: string; isDeleted: boolean; - }): Promise { - this.documents.set(relativePath, { - documentId, - parentVersionId, - hash, - isDeleted - }); - await this.save(); - } + }): void { + const entry = this.getDocumentByRelativePath(relativePath); - public async setDocumentPromise({ - relativePath, - promise - }: { - relativePath: RelativePath; - promise: Promise; - }): Promise { - this.documents.set(relativePath, promise); - // No need to save as Promises don't get serialized - // and a crash would only result in the document being - // creatied again. - } - - public getResolvedDocument( - relativePath: RelativePath | undefined - ): DocumentMetadata | undefined { - if (relativePath == undefined) { - return undefined; - } - - const metadata = this.documents.get(relativePath); - if (metadata instanceof Promise) { - return undefined; - } - - return metadata; - } - - public getDocument( - relativePath: RelativePath | undefined - ): Promise | DocumentMetadata | undefined { - if (relativePath == undefined) { - return undefined; - } - - return this.documents.get(relativePath); - } - - public async move( - oldRelativePath: RelativePath, - newRelativePath: RelativePath - ): Promise { - const document = this.documents.get(oldRelativePath); - if (!document) { - return; - } - - const resolvedDocument = this.getResolvedDocument(oldRelativePath); - if ( - this.documents.has(newRelativePath) && - resolvedDocument != undefined && - resolvedDocument.isDeleted - ) { - throw new Error( - `Cannot update physical path to path that is already in use: ${newRelativePath}` + if (entry !== undefined) { + this.documents = this.documents.filter( + ({ identity }) => identity !== entry.identity ); } - this.documents.delete(oldRelativePath); - this.documents.set(newRelativePath, document); + this.documents.push({ + // `entry` might be undefined if the document is new + identity: entry?.identity ?? Symbol(), + relativePath, + metadata: { + documentId, + parentVersionId, + hash, + isDeleted + }, + updates: entry?.updates ?? [] + }); - await this.save(); + this.save(); } - private async save(): Promise { + public removeDocumentPromise(promise: Promise): void { + const entry = this.getDocumentByUpdatePromise(promise); + entry.updates = entry.updates.filter((update) => update !== promise); + // No need to save as Promises don't get serialized + } + + public getDocumentByRelativePath( + find: RelativePath + ): DocumentRecord | undefined { + return this.documents.find(({ relativePath }) => relativePath === find); + } + + public async getResolvedDocumentByRelativePath( + relativePath: RelativePath, + promise: Promise + ): Promise { + let entry = this.getDocumentByRelativePath(relativePath); + + if (entry === undefined) { + entry = { + relativePath, + identity: Symbol(), + metadata: undefined, + updates: [] + }; + + this.documents.push(entry); + } + + const currentPromises = entry.updates; + entry.updates = [...currentPromises, promise]; + await Promise.all(currentPromises); + + // Refetch the document as it might have been updated + return this.getDocumentByIdentity(entry.identity); + } + + public getDocumentByUpdatePromise(promise: Promise): DocumentRecord { + const result = this.documents.find(({ updates }) => + updates.includes(promise) + ); + + if (result === undefined) { + throw new Error("Document not found by update promise"); + } + + return result; + } + + public getDocumentByDocumentId( + documentId: DocumentId + ): DocumentRecord | undefined { + return this.documents.find( + ({ metadata }) => metadata?.documentId === documentId + ); + } + + public getDocumentByIdentity(find: symbol): DocumentRecord { + const result = this.documents.find(({ identity }) => identity === find); + + if (result === undefined) { + throw new Error("Document not found by identity symbol"); + } + + return result; + } + + public move( + oldRelativePath: RelativePath, + newRelativePath: RelativePath + ): void { + const oldDocument = this.getDocumentByRelativePath(oldRelativePath); + if (oldDocument === undefined) { + throw new Error( + `Document to be moved not found: ${oldRelativePath}` + ); + } + + const newDocument = this.getDocumentByRelativePath(newRelativePath); + if ( + newDocument !== undefined && + newDocument.metadata?.isDeleted === false + ) { + throw new Error( + `Cannot move document to existing path: ${newRelativePath}` + ); + } + + this.documents = this.documents.filter( + ({ identity }) => + identity !== oldDocument.identity && + identity !== newDocument?.identity + ); + + this.documents.push({ + ...oldDocument, + relativePath: newRelativePath + }); + + this.save(); + } + + private save(): void { this.ensureConsistency(); - await this.saveData({ - documents: Object.fromEntries(this.resolvedDocuments), + void this.saveData({ + documents: this.resolvedDocuments.map( + ({ relativePath, metadata }) => ({ + relativePath, + ...metadata + }) + ) as StoredDocumentMetadata[], lastSeenUpdateId: this.lastSeenUpdateId }); } @@ -200,12 +236,16 @@ export class Database { private ensureConsistency(): void { const idToPath = new Map(); - this.resolvedDocuments.forEach(([name, metadata]) => { - idToPath.set(metadata.documentId, [ - ...(idToPath.get(metadata.documentId) ?? []), - name - ]); - }); + this.resolvedDocuments + .filter(({ metadata }) => metadata !== undefined) + .forEach(({ metadata, relativePath }) => { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + idToPath.set(metadata!.documentId, [ + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + ...(idToPath.get(metadata!.documentId) ?? []), + relativePath + ]); + }); const duplicates = Array.from(idToPath.entries()) .filter(([_, paths]) => paths.length > 1) diff --git a/frontend/sync-client/src/services/sync-service.ts b/frontend/sync-client/src/services/sync-service.ts index 6f5b52e7..ff7ec8fd 100644 --- a/frontend/sync-client/src/services/sync-service.ts +++ b/frontend/sync-client/src/services/sync-service.ts @@ -109,6 +109,9 @@ export class SyncService { contentBytes: Uint8Array; createdDate: Date; }): Promise { + this.logger.debug( + `Updating document ${documentId} with parent version ${parentVersionId} & ${new TextDecoder().decode(contentBytes)} & ${relativePath}` + ); const formData = new FormData(); formData.append("parent_version_id", parentVersionId.toString()); formData.append("created_date", createdDate.toISOString()); diff --git a/frontend/sync-client/src/sync-client.ts b/frontend/sync-client/src/sync-client.ts index 54ba6b45..b4d6118a 100644 --- a/frontend/sync-client/src/sync-client.ts +++ b/frontend/sync-client/src/sync-client.ts @@ -148,7 +148,7 @@ export class SyncClient { this.stop(); await this._syncer.reset(); this._history.reset(); - await this._database.resetSyncState(); + this._database.resetSyncState(); this.logger.reset(); } diff --git a/frontend/sync-client/src/sync-operations/syncer.ts b/frontend/sync-client/src/sync-operations/syncer.ts index 193848c4..2b86fdbc 100644 --- a/frontend/sync-client/src/sync-operations/syncer.ts +++ b/frontend/sync-client/src/sync-operations/syncer.ts @@ -12,9 +12,10 @@ import { hash } from "src/utils/hash"; import type { components } from "src/services/types"; import type { Settings } from "src/persistence/settings"; import type { FileOperations } from "src/file-operations/file-operations"; -import { findMatchingFileBasedOnHash } from "src/utils/find-matching-file-based-on-hash"; +import { findMatchingFile } from "src/utils/find-matching-file"; import { UnrestrictedSyncer } from "./unrestricted-syncer"; import { FileNotFoundError } from "src/file-operations/safe-filesystem-operations"; +import { createPromise } from "src/utils/create-promise"; export class Syncer { private readonly remainingOperationsListeners: (( @@ -74,9 +75,10 @@ export class Syncer { logger.debug( `File has been deleted or moved before we had a chance to inspect it, skipping` ); - } else { - throw e; + return undefined; } + + throw e; } } @@ -88,77 +90,95 @@ export class Syncer { public async syncLocallyCreatedFile( relativePath: RelativePath, - updateTime: Date + updateTime?: Date ): Promise { - let resolve: - | undefined - | ((metadata: DocumentMetadata | undefined) => void) = undefined; + const [promise, resolve, reject] = createPromise(); - const creationPromise = new Promise( - (r) => (resolve = r) + // Most likely, we're waiting for the previous delete to finish on the file at this path + const document = await this.database.getResolvedDocumentByRelativePath( + relativePath, + promise ); - await this.database.setDocumentPromise({ - relativePath, - promise: creationPromise - }); - - await this.syncQueue.add(async () => { - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - resolve!( - await this.internalSyncer.unrestrictedSyncLocallyCreatedFile( - relativePath, + try { + await this.syncQueue.add(async () => + this.internalSyncer.unrestrictedSyncLocallyCreatedFile( + () => + this.database.getDocumentByIdentity(document.identity), updateTime ) ); - }); + + resolve(); + } catch (e) { + reject(e); + } finally { + this.database.removeDocumentPromise(promise); + } } public async syncLocallyDeletedFile( relativePath: RelativePath ): Promise { - let metadata = this.database.getDocument(relativePath); - if (metadata !== undefined && !(metadata instanceof Promise)) { - metadata = Promise.resolve(metadata); - } + const [promise, resolve, reject] = createPromise(); - await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncLocallyDeletedFile( - relativePath, - metadata - ) + const document = await this.database.getResolvedDocumentByRelativePath( + relativePath, + promise ); + + try { + await this.syncQueue.add(async () => + this.internalSyncer.unrestrictedSyncLocallyDeletedFile(() => + this.database.getDocumentByIdentity(document.identity) + ) + ); + + resolve(); + } catch (e) { + reject(e); + } finally { + this.database.removeDocumentPromise(promise); + } } public async syncLocallyUpdatedFile(args: { oldPath?: RelativePath; relativePath: RelativePath; - updateTime: Date; + updateTime?: Date; }): Promise { - if (args.oldPath === args.relativePath) { - throw new Error( - `Old path and new path are the same: ${args.oldPath}` - ); - } - if (args.oldPath !== undefined) { - await this.database.move(args.oldPath, args.relativePath); + if (args.oldPath === args.relativePath) { + throw new Error( + `Old path and new path are the same: ${args.oldPath}` + ); + } + + this.database.move(args.oldPath, args.relativePath); } - let metadata = this.database.getDocument(args.relativePath); - if (metadata !== undefined && !(metadata instanceof Promise)) { - metadata = Promise.resolve(metadata); - } - await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncLocallyUpdatedFile({ - ...args, - metadata - }) + const [promise, resolve, reject] = createPromise(); + + const metadata = await this.database.getResolvedDocumentByRelativePath( + args.relativePath, + promise ); - } - public async waitForSyncQueue(): Promise { - return this.syncQueue.onEmpty(); + try { + await this.syncQueue.add(async () => + this.internalSyncer.unrestrictedSyncLocallyUpdatedFile({ + ...args, + getLatestDocument: () => + this.database.getDocumentByIdentity(metadata.identity) + }) + ); + + resolve(); + } catch (e) { + reject(e); + } finally { + this.database.removeDocumentPromise(promise); + } } public async scheduleSyncForOfflineChanges(): Promise { @@ -217,6 +237,10 @@ export class Syncer { } } + public async waitForSyncQueue(): Promise { + return this.syncQueue.onEmpty(); + } + public async reset(): Promise { this.syncQueue.clear(); await this.syncQueue.onEmpty(); @@ -229,53 +253,67 @@ export class Syncer { private async syncRemotelyUpdatedFile( remoteVersion: components["schemas"]["DocumentVersionWithoutContent"] ): Promise { - await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile( - remoteVersion - ) + let document = this.database.getDocumentByDocumentId( + remoteVersion.documentId ); + + if (document === undefined) { + await this.syncQueue.add(async () => + this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile( + remoteVersion + ) + ); + + return; + } + + const [promise, resolve, reject] = createPromise(); + + document = await this.database.getResolvedDocumentByRelativePath( + document.relativePath, + promise + ); + + try { + await this.syncQueue.add(async () => + this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile( + remoteVersion, + () => this.database.getDocumentByIdentity(document.identity) + ) + ); + + resolve(); + } catch (e) { + reject(e); + } finally { + this.database.removeDocumentPromise(promise); + } } private async internalScheduleSyncForOfflineChanges(): Promise { const allLocalFiles = await this.operations.listAllFiles(); - // This includes renamed files for now let locallyPossiblyDeletedFiles = [ ...this.database.resolvedDocuments - ].filter(([path, _]) => !allLocalFiles.includes(path)); + ].filter(({ relativePath }) => !allLocalFiles.includes(relativePath)); const updates = Promise.all( - allLocalFiles.map(async (relativePath) => - this.syncQueue.add(async () => { - const metadata = - this.database.getResolvedDocument(relativePath); + allLocalFiles.map(async (relativePath) => { + if ( + this.database.getDocumentByRelativePath(relativePath) + ?.metadata !== undefined + ) { + this.logger.debug( + `Document ${relativePath} might have been updated locally, scheduling sync to validate and update it` + ); - if (metadata) { - this.logger.debug( - `Document ${relativePath} might have been updated locally, scheduling sync to validate and update it` - ); - const updateTime = - await Syncer.forgivingFileNotFoundWrapper( - async () => - this.operations.getModificationTime( - relativePath - ), - this.logger - ); - if (updateTime === undefined) { - return; - } + return this.syncLocallyUpdatedFile({ + relativePath + }); + } - return this.internalSyncer.unrestrictedSyncLocallyUpdatedFile( - { - relativePath, - updateTime, - metadata: Promise.resolve(metadata) - } - ); - } - - // Perhaps the file has been moved. Let's check by looking at the deleted files + // Perhaps the file has been moved; let's check by looking at the deleted files + const contentHash = await this.syncQueue.add(async () => { const contentBytes = await Syncer.forgivingFileNotFoundWrapper( async () => this.operations.read(relativePath), @@ -284,90 +322,51 @@ export class Syncer { if (contentBytes === undefined) { return; } + return hash(contentBytes); + }); - const contentHash = hash(contentBytes); + if (contentHash == undefined) { + // The file was deleted before we had a chance to read it, no need to sync it here + return; + } - // todo: make this smarter so that offline files can be renamed & edited at the same time - const originalFile = findMatchingFileBasedOnHash( - contentHash, - locallyPossiblyDeletedFiles - ); - if (originalFile !== undefined) { - // `originalFile` hasn't been deleted but it got moved instead - locallyPossiblyDeletedFiles = - locallyPossiblyDeletedFiles.filter( - (item) => item[0] !== originalFile[0] - ); - - this.logger.debug( - `Document '${originalFile[0]}' was not found under its current path in the database but was found under a different path (${relativePath}), scheduling sync to move it` + const originalFile = findMatchingFile( + contentHash, + locallyPossiblyDeletedFiles + ); + if (originalFile !== undefined) { + // `originalFile` hasn't been deleted but it got moved instead + locallyPossiblyDeletedFiles = + locallyPossiblyDeletedFiles.filter( + (item) => + item.relativePath !== originalFile.relativePath ); - const updateTime = - await Syncer.forgivingFileNotFoundWrapper( - async () => - this.operations.getModificationTime( - relativePath - ), - this.logger - ); - if (updateTime === undefined) { - return; - } - - return this.internalSyncer.unrestrictedSyncLocallyUpdatedFile( - { - oldPath: originalFile[0], - relativePath, - updateTime, - metadata: Promise.resolve( - this.database.getResolvedDocument( - relativePath - ) - ), - optimisations: { - contentBytes, - contentHash - } - } - ); - } - this.logger.debug( - `Document ${relativePath} not found in database, scheduling sync to create it` + `Document '${originalFile.relativePath}' was not found under its current path in the database but was found under a different path (${relativePath}), scheduling sync to move it` ); - const updateTime = - await Syncer.forgivingFileNotFoundWrapper( - async () => - this.operations.getModificationTime( - relativePath - ), - this.logger - ); - if (updateTime === undefined) { - return; - } - return this.internalSyncer.unrestrictedSyncLocallyCreatedFile( - relativePath, - updateTime - ); - }) - ) + + // We're outside of the pqueue, so we need to call the public wrapper + return this.syncLocallyUpdatedFile({ + oldPath: originalFile.relativePath, + relativePath + }); + } + + this.logger.debug( + `Document ${relativePath} not found in database, scheduling sync to create it` + ); + // We're outside of the pqueue, so we need to call the public wrapper + return this.syncLocallyCreatedFile(relativePath); + }) ); const deletes = Promise.all( - locallyPossiblyDeletedFiles.map(async ([relativePath, _]) => { + locallyPossiblyDeletedFiles.map(async ({ relativePath }) => { this.logger.debug( `Document ${relativePath} has been deleted locally, scheduling sync to delete it` ); - if (await this.operations.exists(relativePath)) { - this.logger.debug( - `Document ${relativePath} actually exists locally, skipping` - ); - return Promise.resolve(); - } - // We're outside of the pqueue, so we need to call the public wrapper return this.syncLocallyDeletedFile(relativePath); }) @@ -389,15 +388,7 @@ export class Syncer { this.logger.info("Applying remote changes locally"); await Promise.all( - remote.latestDocuments - .filter( - (remoteDocument) => - remoteDocument.vaultUpdateId > - (this.database.getDocumentByDocumentId( - remoteDocument.documentId - )?.[1].parentVersionId ?? -1) - ) - .map(this.syncRemotelyUpdatedFile.bind(this)) + remote.latestDocuments.map(this.syncRemotelyUpdatedFile.bind(this)) ); const lastSeenUpdateId = this.database.getLastSeenUpdateId(); @@ -405,7 +396,7 @@ export class Syncer { lastSeenUpdateId === undefined || remote.lastUpdateId > lastSeenUpdateId ) { - await this.database.setLastSeenUpdateId(remote.lastUpdateId); + this.database.setLastSeenUpdateId(remote.lastUpdateId); } } diff --git a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts index c7b6f044..b1951f89 100644 --- a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts +++ b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts @@ -1,11 +1,12 @@ import type { Database, DocumentMetadata, + DocumentRecord, RelativePath } from "../persistence/database"; import type { SyncService } from "src/services/sync-service"; -import type { Logger } from "src/tracing/logger"; +import { Logger } from "src/tracing/logger"; import type { SyncHistory } from "src/tracing/sync-history"; import { SyncSource, SyncStatus, SyncType } from "src/tracing/sync-history"; import { EMPTY_HASH, hash } from "src/utils/hash"; @@ -31,37 +32,28 @@ export class UnrestrictedSyncer { } public async unrestrictedSyncLocallyCreatedFile( - relativePath: RelativePath, - updateTime: Date, - optimisations?: { - contentBytes?: Uint8Array; - contentHash?: string; - } - ): Promise { + getLatestDocument: () => DocumentRecord, + updateTime?: Date + ): Promise { + const { relativePath, metadata } = getLatestDocument(); + return this.executeSync( [relativePath], SyncType.CREATE, SyncSource.PUSH, async () => { - const localMetadata = this.database.getDocument(relativePath); - - if ( - !(localMetadata instanceof Promise) && - localMetadata && - !localMetadata.isDeleted - ) { + if (metadata !== undefined && !metadata.isDeleted) { this.logger.debug( - `Document metadata already exists for ${relativePath}, it must have been downloaded from the server` + `Document ${relativePath} already exists in the database, no need to create it again` ); - return; } - const contentBytes = - optimisations?.contentBytes ?? - (await this.operations.read(relativePath)); // this can throw FileNotFoundError - const contentHash = - optimisations?.contentHash ?? hash(contentBytes); + const contentBytes = await this.operations.read(relativePath); // this can throw FileNotFoundError + const contentHash = hash(contentBytes); + + updateTime ??= + await this.operations.getModificationTime(relativePath); // this can throw FileNotFoundError const response = await this.syncService.create({ relativePath, @@ -69,95 +61,71 @@ export class UnrestrictedSyncer { createdDate: updateTime }); - const currentMetadata = - this.database.getDocumentByIdentity(localMetadata); - if (!currentMetadata) { - throw new Error( - `Document metadata for ${relativePath} not found after creation` - ); - } + const { relativePath: currentRelativePath } = + getLatestDocument(); this.history.addHistoryEntry({ status: SyncStatus.SUCCESS, source: SyncSource.PUSH, - relativePath: currentMetadata[0], + relativePath, message: `Successfully uploaded locally created file`, type: SyncType.CREATE }); const newMetadata = { + relativePath: currentRelativePath, documentId: response.documentId, parentVersionId: response.vaultUpdateId, hash: contentHash, isDeleted: false }; - await this.database.setDocument({ - relativePath: currentMetadata[0], - ...newMetadata - }); + this.database.setDocument(newMetadata); - await this.tryIncrementVaultUpdateId(response.vaultUpdateId); - - return newMetadata; + this.tryIncrementVaultUpdateId(response.vaultUpdateId); } ); } public async unrestrictedSyncLocallyDeletedFile( - relativePath: RelativePath, - metadata: Promise | undefined + getLatestDocument: () => DocumentRecord ): Promise { + let document = getLatestDocument(); await this.executeSync( - [relativePath], + [document.relativePath], SyncType.DELETE, SyncSource.PUSH, async () => { - const localMetadata = - metadata !== undefined - ? await metadata - : this.database.getResolvedDocument(relativePath); - - if (!localMetadata || localMetadata.isDeleted) { - this.logger.info( - `Locally deleted file hasn't been uploaded yet, so there's no need to delete it on the remote server` + if ( + document.metadata === undefined || + document.metadata.isDeleted + ) { + this.logger.debug( + `Document ${document.relativePath} has been already deleted, no need to delete it again` ); - return; } const response = await this.syncService.delete({ - documentId: localMetadata.documentId, - relativePath, - createdDate: new Date() // We got the event now, so it must have been deleted just now + documentId: document.metadata.documentId, + relativePath: document.relativePath, + createdDate: new Date() // We've got the event now, so it must have been deleted just now }); this.history.addHistoryEntry({ status: SyncStatus.SUCCESS, source: SyncSource.PUSH, - relativePath, + relativePath: document.relativePath, message: `Successfully deleted locally deleted file on the remote server`, type: SyncType.DELETE }); - const currentMetadata = this.database.getDocumentByDocumentId( - localMetadata.documentId - ); - - if (!currentMetadata || currentMetadata[1].isDeleted) { - this.logger.info( - `No metadata found for deleted file, '${relativePath}' must have been deleted by another operation` - ); - - return; - } - - await this.operations.delete(currentMetadata[0]); + document = getLatestDocument(); // We have to have a record of the delete in case there's an in-flight update for the same // document which finishes after the delete has succeeded and would introduce a phantom metadata record. - await this.database.setDocument({ - relativePath: currentMetadata[0], + this.database.setDocument({ + relativePath: document.relativePath, documentId: response.documentId, parentVersionId: response.vaultUpdateId, hash: EMPTY_HASH, @@ -169,84 +137,64 @@ export class UnrestrictedSyncer { public async unrestrictedSyncLocallyUpdatedFile({ oldPath, - relativePath, - metadata, - updateTime, - optimisations + getLatestDocument, + updateTime }: { oldPath?: RelativePath; - relativePath: RelativePath; - metadata: Promise | undefined; - updateTime: Date; - optimisations?: { - contentBytes?: Uint8Array; - contentHash?: string; - }; + getLatestDocument: () => DocumentRecord; + updateTime?: Date; }): Promise { + let document = getLatestDocument(); + await this.executeSync( - [oldPath, relativePath].filter((path) => path !== undefined), + [oldPath, document.relativePath].filter( + (path) => path !== undefined + ), SyncType.UPDATE, SyncSource.PUSH, async () => { - const localMetadata = - metadata !== undefined - ? await metadata - : this.database.getResolvedDocument(relativePath); - - if (!localMetadata || localMetadata.isDeleted) { - // It's fine, a subsequent sync operation must have dealt with this - return; - } - - const contentBytes = - optimisations?.contentBytes ?? - (await this.operations.read(relativePath)); // this can throw FileNotFoundError - - let contentHash = - optimisations?.contentHash ?? hash(contentBytes); - if ( - localMetadata.hash === contentHash && - oldPath === undefined + document.metadata === undefined || + document.metadata.isDeleted ) { this.logger.debug( - `File hash of ${relativePath} matches with last synced version and the path hasn't changed; no need to sync` + `Document ${document.relativePath} has been already deleted, no need to update it, ${JSON.stringify(document)}, ${document.metadata?.isDeleted}` ); return; } - // Re-fetch based on the documentId instead of the relativePath because - // the relativePath might have changed since this operation was scheduled - let latestMetadata = this.database.getDocumentByDocumentId( - localMetadata.documentId - ); - if (!latestMetadata || latestMetadata[1].isDeleted) { - // It's fine, a subsequent sync operation must have dealt with this + const contentBytes = await this.operations.read( + document.relativePath + ); // this can throw FileNotFoundError + let contentHash = hash(contentBytes); + + if ( + document.metadata.hash === contentHash && + oldPath === undefined + ) { + this.logger.debug( + `File hash of ${document.relativePath} matches with last synced version and the path hasn't changed; no need to sync` + ); return; } + updateTime ??= await this.operations.getModificationTime( + document.relativePath + ); // this can throw FileNotFoundError; + const response = await this.syncService.put({ - documentId: latestMetadata[1].documentId, - parentVersionId: latestMetadata[1].parentVersionId, - relativePath: latestMetadata[0], + documentId: document.metadata.documentId, + parentVersionId: document.metadata.parentVersionId, + relativePath: document.relativePath, contentBytes, createdDate: updateTime }); - latestMetadata = this.database.getDocumentByDocumentId( - response.documentId - ); - - if (!latestMetadata || latestMetadata[1].isDeleted) { - // The document has been deleted since this operation was scheduled - return; - } - if ( - latestMetadata[1].parentVersionId >= response.vaultUpdateId + document.metadata.parentVersionId >= response.vaultUpdateId ) { this.logger.debug( - `Document ${relativePath} is already more up to date than the fetched version` + `Document ${document.relativePath} is already more up to date than the fetched version` ); return; } @@ -254,50 +202,42 @@ export class UnrestrictedSyncer { this.history.addHistoryEntry({ status: SyncStatus.SUCCESS, source: SyncSource.PUSH, - relativePath, + relativePath: document.relativePath, message: `Successfully uploaded locally updated file to the remote server`, type: SyncType.UPDATE }); + // Update relativePath which is the only property that can change while this is running (due to a move) + document = getLatestDocument(); + if (response.isDeleted) { - await this.operations.delete(relativePath); + await this.operations.delete(document.relativePath); this.history.addHistoryEntry({ status: SyncStatus.SUCCESS, source: SyncSource.PULL, - relativePath, + relativePath: document.relativePath, message: "The file we tried to update had been deleted remotely, therefore, we have deleted it locally", type: SyncType.DELETE }); - await this.database.setDocument({ + this.database.setDocument({ documentId: response.documentId, - relativePath: latestMetadata[0], + relativePath: document.relativePath, parentVersionId: response.vaultUpdateId, hash: EMPTY_HASH, isDeleted: true }); - await this.tryIncrementVaultUpdateId( - response.vaultUpdateId - ); + this.tryIncrementVaultUpdateId(response.vaultUpdateId); return; } - if ( - latestMetadata[1].parentVersionId >= response.vaultUpdateId - ) { - this.logger.debug( - `Document ${relativePath} is already more up to date than the fetched version` - ); - return; - } - - if (response.relativePath != relativePath) { + if (response.relativePath != document.relativePath) { await this.operations.move( - latestMetadata[0], + document.relativePath, response.relativePath, response.documentId ); // this can throw FileNotFoundError @@ -316,155 +256,85 @@ export class UnrestrictedSyncer { this.history.addHistoryEntry({ status: SyncStatus.SUCCESS, source: SyncSource.PULL, - relativePath, + relativePath: document.relativePath, message: `The file we updated had been updated remotely, so we downloaded the merged version`, type: SyncType.UPDATE }); } - await this.database.setDocument({ + this.database.setDocument({ documentId: response.documentId, relativePath: - response.relativePath != relativePath + response.relativePath != document.relativePath ? response.relativePath - : latestMetadata[0], + : document.relativePath, parentVersionId: response.vaultUpdateId, hash: contentHash, isDeleted: response.isDeleted }); - await this.tryIncrementVaultUpdateId(response.vaultUpdateId); + this.tryIncrementVaultUpdateId(response.vaultUpdateId); } ); } public async unrestrictedSyncRemotelyUpdatedFile( - remoteVersion: components["schemas"]["DocumentVersionWithoutContent"] + remoteVersion: components["schemas"]["DocumentVersionWithoutContent"], + getLatestDocument?: () => DocumentRecord ): Promise { await this.executeSync( [remoteVersion.relativePath], SyncType.UPDATE, SyncSource.PULL, async () => { + const localMetadata = + getLatestDocument?.() ?? + this.database.getDocumentByDocumentId( + remoteVersion.documentId + ); + + if ( + localMetadata?.metadata !== undefined && + !localMetadata.metadata.isDeleted + ) { + // If the file exists locally, let's pretend the user has updated it + // and deal with remote update/deletion within `unrestrictedSyncLocallyUpdatedFile` + if ( + localMetadata.metadata.parentVersionId >= + remoteVersion.vaultUpdateId + ) { + this.logger.debug( + `Document ${remoteVersion.relativePath} is already more up to date than the fetched version` + ); + return; + } + + return this.unrestrictedSyncLocallyUpdatedFile({ + getLatestDocument: () => + this.database.getDocumentByIdentity( + localMetadata.identity + ) + }); + } + const content = ( await this.syncService.get({ documentId: remoteVersion.documentId }) ).contentBase64; const contentBytes = deserialize(content); - const contentHash = hash(contentBytes); - const localMetadata = this.database.getDocumentByDocumentId( - remoteVersion.documentId - ); - if ( - localMetadata?.[1].documentId === - remoteVersion.documentId && - localMetadata[1].parentVersionId > - remoteVersion.vaultUpdateId - ) { - this.logger.info( - `Document ${remoteVersion.relativePath} is already up to date` - ); - return; - } - - const localBytes = await this.operations.read( - remoteVersion.relativePath - ); // this can throw FileNotFoundError - const localHash = hash(localBytes); - - if (localHash !== localMetadata?.[1].hash) { - this.logger.info( - `Document ${remoteVersion.relativePath} has pending local changes, so we shouldn't update it here` - ); - return; - } - - if (!localMetadata || localMetadata[1].isDeleted) { - if (remoteVersion.isDeleted) { - this.logger.info( - `Remotely deleted file hasn't been synced yet, so there's no need to delete it locally` - ); - return; - } - - await this.operations.create( - remoteVersion.relativePath, - contentBytes, - remoteVersion.documentId - ); - - await this.database.setDocument({ - documentId: remoteVersion.documentId, - relativePath: remoteVersion.relativePath, - parentVersionId: remoteVersion.vaultUpdateId, - hash: hash(contentBytes), - isDeleted: remoteVersion.isDeleted - }); - - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - source: SyncSource.PULL, - relativePath: remoteVersion.relativePath, - message: `Successfully downloaded remote file which hadn't existed locally`, - type: SyncType.CREATE - }); - return; - } - - const [relativePath, metadata] = localMetadata; - if (remoteVersion.vaultUpdateId <= metadata.parentVersionId) { - this.logger.debug( - `Document ${relativePath} is already up to date` - ); - return; - } - - if (remoteVersion.isDeleted) { - await this.operations.delete(relativePath); - - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - source: SyncSource.PULL, - relativePath: remoteVersion.relativePath, - message: `Successfully deleted remotely deleted file locally`, - type: SyncType.DELETE - }); - - await this.database.setDocument({ - documentId: remoteVersion.documentId, - relativePath: relativePath, - parentVersionId: remoteVersion.vaultUpdateId, - hash: EMPTY_HASH, - isDeleted: true - }); - - return; - } - - if (relativePath !== remoteVersion.relativePath) { - // TODO: this can fail, that's bad - await this.operations.move( - // this can throw FileNotFoundError - relativePath, - remoteVersion.relativePath, - remoteVersion.documentId - ); - } - - // todo: why await this.operations.create( remoteVersion.relativePath, contentBytes, remoteVersion.documentId ); - await this.database.setDocument({ + this.database.setDocument({ documentId: remoteVersion.documentId, relativePath: remoteVersion.relativePath, parentVersionId: remoteVersion.vaultUpdateId, - hash: contentHash, + hash: hash(contentBytes), isDeleted: remoteVersion.isDeleted }); @@ -472,8 +342,8 @@ export class UnrestrictedSyncer { status: SyncStatus.SUCCESS, source: SyncSource.PULL, relativePath: remoteVersion.relativePath, - message: `Successfully updated remotely updated file locally`, - type: SyncType.UPDATE + message: `Successfully downloaded remote file which hadn't existed locally`, + type: SyncType.CREATE }); } ); @@ -551,11 +421,9 @@ export class UnrestrictedSyncer { this.locks.reset(); } - private async tryIncrementVaultUpdateId( - responseVaultUpdateId: number - ): Promise { + private tryIncrementVaultUpdateId(responseVaultUpdateId: number): void { if (this.database.getLastSeenUpdateId() === responseVaultUpdateId - 1) { - await this.database.setLastSeenUpdateId(responseVaultUpdateId); + this.database.setLastSeenUpdateId(responseVaultUpdateId); } } } diff --git a/frontend/sync-client/src/utils/create-promise.ts b/frontend/sync-client/src/utils/create-promise.ts new file mode 100644 index 00000000..056c169c --- /dev/null +++ b/frontend/sync-client/src/utils/create-promise.ts @@ -0,0 +1,15 @@ +export function createPromise(): [ + Promise, + (value: T) => void, + (error: unknown) => void +] { + let resolve: undefined | ((resolved: T) => void) = undefined; + let reject: undefined | ((error: unknown) => void) = undefined; + + const creationPromise = new Promise( + (resolve_, reject_) => ((resolve = resolve_), (reject = reject_)) + ); + + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + return [creationPromise, resolve!, reject!]; +} diff --git a/frontend/sync-client/src/utils/find-matching-file.ts b/frontend/sync-client/src/utils/find-matching-file.ts index 27a4c875..10545f2c 100644 --- a/frontend/sync-client/src/utils/find-matching-file.ts +++ b/frontend/sync-client/src/utils/find-matching-file.ts @@ -1,14 +1,14 @@ -import type { DocumentMetadata, RelativePath } from "../persistence/database"; +import type { DocumentRecord } from "../persistence/database"; import { EMPTY_HASH } from "./hash"; // TODO: make this smarter so that offline files can be renamed & edited at the same time export function findMatchingFile( contentHash: string, - candidates: [RelativePath, DocumentMetadata][] -): [RelativePath, DocumentMetadata] | undefined { + candidates: DocumentRecord[] +): DocumentRecord | undefined { if (contentHash === EMPTY_HASH) { return undefined; } - return candidates.find(([_, metadata]) => metadata.hash === contentHash); + return candidates.find(({ metadata }) => metadata?.hash === contentHash); } diff --git a/frontend/test-client/run.sh b/frontend/test-client/run.sh index 892be317..6effc147 100755 --- a/frontend/test-client/run.sh +++ b/frontend/test-client/run.sh @@ -1,21 +1,71 @@ #!/bin/bash set -e +set -o pipefail + +# Check if the argument is provided +if [ $# -eq 0 ]; then + echo "Usage: $0 " + exit 1 +fi + +# Get the number of processes from the first argument +process_count=$1 npm run build pids=() -for i in {1..10}; do +for i in $(seq 1 $process_count); do node dist/cli.js 2>&1 | tee "log_${i}.log" & pids+=($!) done -trap 'kill ${pids[@]} 2>/dev/null' SIGINT SIGTERM +print_failed_log() { + for i in $(seq 1 $process_count); do + if [ -n "${pids[$i-1]}" ] && ! kill -0 ${pids[$i-1]} 2>/dev/null; then + # Get the exit code of the process + wait ${pids[$i-1]} + exit_code=$? + + # Only consider non-zero exit codes as failures + if [ $exit_code -ne 0 ]; then + echo "Process ${pids[$i-1]} failed with exit code $exit_code. Log file: $(pwd)/log_${i}.log" + return 0 + else + echo "Process ${pids[$i-1]} completed successfully with exit code 0" + # Mark this PID as processed by setting it to empty + pids[$i-1]="" + fi + fi + done + return 1 +} -for pid in ${pids[@]}; do - if ! wait $pid; then - kill ${pids[@]} 2>/dev/null - echo "Process $pid failed, see log_$(echo ${pids[@]} | tr ' ' '\n' | grep -n "^$pid$" | cut -d: -f1).log" +# Monitor processes +while true; do + if print_failed_log; then + # Kill remaining processes + for pid in "${pids[@]}"; do + if [ -n "$pid" ]; then + kill $pid 2>/dev/null || true + fi + done exit 1 fi + + # Check if all processes have completed + all_done=true + for pid in "${pids[@]}"; do + if [ -n "$pid" ] && kill -0 $pid 2>/dev/null; then + all_done=false + break + fi + done + + if $all_done; then + echo "All processes completed successfully" + exit 0 + fi + + sleep 0.2 done diff --git a/frontend/test-client/src/agent/mock-agent.ts b/frontend/test-client/src/agent/mock-agent.ts index fad989fb..06c0f10e 100644 --- a/frontend/test-client/src/agent/mock-agent.ts +++ b/frontend/test-client/src/agent/mock-agent.ts @@ -64,7 +64,7 @@ export class MockAgent extends MockClient { // Let's not ignore errors // eslint-disable-next-line @typescript-eslint/no-floating-promises - sleep(1000).then(() => process.exit(1)); + sleep(100).then(() => process.exit(1)); break; case LogLevel.WARNING: diff --git a/frontend/test-client/src/cli.ts b/frontend/test-client/src/cli.ts index 26a0f23f..e87666f3 100644 --- a/frontend/test-client/src/cli.ts +++ b/frontend/test-client/src/cli.ts @@ -38,7 +38,9 @@ async function runTest({ ) ); } + // for debugging + // eslint-disable-next-line (globalThis as any).clients = clients; try { @@ -88,11 +90,11 @@ async function runTest({ } async function runTests(): Promise { - const agentCounts = [2, 10]; - const jitterScaleInSeconds = [0, 0.5, 3]; - const concurrencies = [1, 16]; - const iterations = [50, 300]; - const doDeletes = [false]; + const agentCounts = [2, 8]; + const jitterScaleInSeconds = [0.5, 0, 2]; + const concurrencies = [1]; + const iterations = [50, 200]; + const doDeletes = [true, false]; for (const agentCount of agentCounts) { for (const concurrency of concurrencies) { @@ -106,6 +108,7 @@ async function runTests(): Promise { doDeletes: deleteFiles, jitterScaleInSeconds: jitter }); + return; } } } @@ -113,15 +116,13 @@ async function runTests(): Promise { } } -process.on("uncaughtException", async (error) => { +process.on("uncaughtException", (error) => { console.error("Uncaught Exception:", error); - await sleep(1000); process.exit(1); }); -process.on("unhandledRejection", async (reason, promise) => { +process.on("unhandledRejection", (reason, _promise) => { console.error("Unhandled Rejection:", reason); - await sleep(1000); process.exit(1); }); @@ -131,6 +132,5 @@ runTests() }) .catch(async (err: unknown) => { console.error(err); - await sleep(1000); process.exit(1); }); From d23c1a8dbc5d96c2afd195d2d476095a08f45644 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Mon, 10 Mar 2025 22:49:51 +0000 Subject: [PATCH 11/58] omg it mostly works for deletes --- .../file-operations/file-operations.test.ts | 10 +- .../src/file-operations/file-operations.ts | 11 +- .../sync-client/src/persistence/database.ts | 157 +++++++++++++----- .../sync-client/src/sync-operations/syncer.ts | 103 +++++++----- .../sync-operations/unrestricted-syncer.ts | 99 ++++++----- frontend/test-client/run.sh | 2 +- 6 files changed, 243 insertions(+), 139 deletions(-) diff --git a/frontend/sync-client/src/file-operations/file-operations.test.ts b/frontend/sync-client/src/file-operations/file-operations.test.ts index b1299098..26ae3267 100644 --- a/frontend/sync-client/src/file-operations/file-operations.test.ts +++ b/frontend/sync-client/src/file-operations/file-operations.test.ts @@ -1,7 +1,6 @@ import type { FileSystemOperations } from "sync-client"; import type { Database, - DocumentMetadata, DocumentRecord, RelativePath } from "../persistence/database"; @@ -11,14 +10,7 @@ import { assertSetContainsExactly } from "../utils/assert-set-contains-exactly"; describe("File operations", () => { class MockDatabase { - public move( - _oldRelativePath: RelativePath, - _newRelativePath: RelativePath - ): void { - // this is called but irrelevant for this mock - } - - public getDocumentByRelativePath( + public getLatestDocumentByRelativePath( _find: RelativePath ): DocumentRecord | undefined { return undefined; diff --git a/frontend/sync-client/src/file-operations/file-operations.ts b/frontend/sync-client/src/file-operations/file-operations.ts index 34ed19d9..ef9dda55 100644 --- a/frontend/sync-client/src/file-operations/file-operations.ts +++ b/frontend/sync-client/src/file-operations/file-operations.ts @@ -71,15 +71,12 @@ export class FileOperations { `Didn't expect ${path} to exist, deconflicting by moving it to '${deconflictedPath}'` ); - const document = this.database.getDocumentByRelativePath(path); + const document = + this.database.getLatestDocumentByRelativePath(path); this.logger.debug( `Existing metadata for ${path}: ${JSON.stringify(document?.metadata)}` ); - this.logger.debug( - `We need to save what's at ${path} to ${deconflictedPath}` - ); - if ( document?.metadata !== undefined && document.metadata.documentId === documentId @@ -94,7 +91,6 @@ export class FileOperations { `We need to save what's at ${path} to ${deconflictedPath}` ); await this.move(path, deconflictedPath, documentId); - // this.database.move(path, deconflictedPath); } else { await this.createParentDirectories(path); } @@ -178,7 +174,8 @@ export class FileOperations { `Conflict when moving '${oldPath}' to '${newPath}', the latter already exists, deconflicting by moving it to '${deconflictedPath}'` ); - const document = this.database.getDocumentByRelativePath(newPath); + const document = + this.database.getLatestDocumentByRelativePath(newPath); if ( document?.metadata !== undefined && diff --git a/frontend/sync-client/src/persistence/database.ts b/frontend/sync-client/src/persistence/database.ts index c529889c..0bbbd5b1 100644 --- a/frontend/sync-client/src/persistence/database.ts +++ b/frontend/sync-client/src/persistence/database.ts @@ -10,6 +10,7 @@ export interface DocumentMetadata { hash: string; isDeleted: boolean; } + export interface StoredDocumentMetadata { relativePath: RelativePath; parentVersionId: VaultUpdateId; @@ -25,6 +26,7 @@ export interface StoredDatabase { export interface DocumentRecord { identity: symbol; + parallelVersion: number; relativePath: RelativePath; metadata: DocumentMetadata | undefined; updates: Promise[]; @@ -46,7 +48,8 @@ export class Database { relativePath, identity: Symbol(), metadata, - updates: [] + updates: [], + parallelVersion: 0 })) ?? []; this.ensureConsistency(); @@ -63,7 +66,38 @@ export class Database { } public get resolvedDocuments(): DocumentRecord[] { - return this.documents.filter(({ metadata }) => metadata !== undefined); + const paths = new Map(); + this.documents + .filter( + ({ metadata }) => metadata !== undefined && !metadata.isDeleted + ) + .forEach((record) => + paths.set(record.relativePath, [ + record, + ...(paths.get(record.relativePath) ?? []) + ]) + ); + + return Array.from(paths.values()).map((records) => { + records.sort( + (a, b) => b.parallelVersion - a.parallelVersion // descending + ); + + if ( + records.length > 1 && + records.some((current, i) => + i === 0 + ? false + : records[i - 1].parallelVersion === + current.parallelVersion + ) + ) { + throw new Error( + `Multiple documents with the same parallel version and path at ${records[0].relativePath}` + ); + } + return records[0]; + }); } public getLastSeenUpdateId(): VaultUpdateId | undefined { @@ -81,25 +115,53 @@ export class Database { this.save(); } - public setDocument({ - documentId, - relativePath, - parentVersionId, - hash, - isDeleted - }: { - documentId: DocumentId; - relativePath: RelativePath; - parentVersionId: VaultUpdateId; - hash: string; - isDeleted: boolean; - }): void { - const entry = this.getDocumentByRelativePath(relativePath); + public setDocument( + { + documentId, + relativePath, + parentVersionId, + hash, + isDeleted + }: { + documentId: DocumentId; + relativePath: RelativePath; + parentVersionId: VaultUpdateId; + hash: string; + isDeleted: boolean; + }, + identity?: symbol + ): void { + let entry: DocumentRecord | undefined; + if (identity !== undefined) { + entry = this.getDocumentByIdentity(identity); - if (entry !== undefined) { - this.documents = this.documents.filter( - ({ identity }) => identity !== entry.identity - ); + if (entry !== undefined) { + this.documents = this.documents.filter( + ({ identity }) => identity !== entry!.identity + ); + } + } else { + entry = this.getLatestDocumentByRelativePath(relativePath); + if ( + entry?.metadata?.documentId !== undefined && + entry.metadata.documentId !== documentId + ) { + this.documents.push({ + // `entry` might be undefined if the document is new + identity: Symbol(), + relativePath, + metadata: { + documentId, + parentVersionId, + hash, + isDeleted + }, + updates: [], + parallelVersion: entry?.parallelVersion + 1 + }); + } + this.save(); + return; } this.documents.push({ @@ -112,7 +174,8 @@ export class Database { hash, isDeleted }, - updates: entry?.updates ?? [] + updates: entry?.updates ?? [], + parallelVersion: entry?.parallelVersion ?? 0 }); this.save(); @@ -124,24 +187,29 @@ export class Database { // No need to save as Promises don't get serialized } - public getDocumentByRelativePath( + public getLatestDocumentByRelativePath( find: RelativePath ): DocumentRecord | undefined { - return this.documents.find(({ relativePath }) => relativePath === find); + const candidates = this.documents.filter( + ({ relativePath }) => relativePath === find + ); + candidates.sort((a, b) => b.parallelVersion - a.parallelVersion); // descending + return candidates[0]; } public async getResolvedDocumentByRelativePath( relativePath: RelativePath, promise: Promise - ): Promise { - let entry = this.getDocumentByRelativePath(relativePath); + ): Promise { + let entry = this.getLatestDocumentByRelativePath(relativePath); if (entry === undefined) { entry = { relativePath, identity: Symbol(), metadata: undefined, - updates: [] + updates: [], + parallelVersion: 0 }; this.documents.push(entry); @@ -150,9 +218,6 @@ export class Database { const currentPromises = entry.updates; entry.updates = [...currentPromises, promise]; await Promise.all(currentPromises); - - // Refetch the document as it might have been updated - return this.getDocumentByIdentity(entry.identity); } public getDocumentByUpdatePromise(promise: Promise): DocumentRecord { @@ -189,38 +254,40 @@ export class Database { oldRelativePath: RelativePath, newRelativePath: RelativePath ): void { - const oldDocument = this.getDocumentByRelativePath(oldRelativePath); + const oldDocument = + this.getLatestDocumentByRelativePath(oldRelativePath); if (oldDocument === undefined) { + return; throw new Error( `Document to be moved not found: ${oldRelativePath}` ); } - const newDocument = this.getDocumentByRelativePath(newRelativePath); - if ( - newDocument !== undefined && - newDocument.metadata?.isDeleted === false - ) { - throw new Error( - `Cannot move document to existing path: ${newRelativePath}` - ); - } - this.documents = this.documents.filter( - ({ identity }) => - identity !== oldDocument.identity && - identity !== newDocument?.identity + ({ identity }) => identity !== oldDocument.identity ); + let newDocument = this.getLatestDocumentByRelativePath(newRelativePath); + + // It's either an invalid state of newDocument is pending deletion and we have to wait for it to complete this.documents.push({ - ...oldDocument, - relativePath: newRelativePath + identity: oldDocument.identity, + metadata: oldDocument.metadata, + relativePath: newRelativePath, + updates: oldDocument.updates, + // We're in a strange state where the target of the move has just got deleted, + // however, its metadata might already have a bunch of updates queued up for + // the document at the new location. We need to keep these updates. + parallelVersion: + newDocument !== undefined ? newDocument.parallelVersion + 1 : 0 }); this.save(); } private save(): void { + this.logger.debug(JSON.stringify(this.documents, null, 2)); + this.ensureConsistency(); void this.saveData({ documents: this.resolvedDocuments.map( diff --git a/frontend/sync-client/src/sync-operations/syncer.ts b/frontend/sync-client/src/sync-operations/syncer.ts index 2b86fdbc..e61e7c45 100644 --- a/frontend/sync-client/src/sync-operations/syncer.ts +++ b/frontend/sync-client/src/sync-operations/syncer.ts @@ -1,9 +1,4 @@ -import type { - Database, - DocumentMetadata, - RelativePath -} from "../persistence/database"; - +import type { Database, RelativePath } from "../persistence/database"; import type { SyncService } from "src/services/sync-service"; import type { Logger } from "src/tracing/logger"; import type { SyncHistory } from "src/tracing/sync-history"; @@ -24,10 +19,8 @@ export class Syncer { private readonly syncQueue: PQueue; - private runningScheduleSyncForOfflineChanges: Promise | undefined = - undefined; - private runningApplyRemoteChangesLocally: Promise | undefined = - undefined; + private runningScheduleSyncForOfflineChanges: Promise | undefined; + private runningApplyRemoteChangesLocally: Promise | undefined; private readonly internalSyncer: UnrestrictedSyncer; @@ -92,10 +85,17 @@ export class Syncer { relativePath: RelativePath, updateTime?: Date ): Promise { + if (!this.settings.getSettings().isSyncEnabled) { + this.logger.info( + `Syncing is disabled, not syncing '${relativePath}'` + ); + return; + } + const [promise, resolve, reject] = createPromise(); // Most likely, we're waiting for the previous delete to finish on the file at this path - const document = await this.database.getResolvedDocumentByRelativePath( + await this.database.getResolvedDocumentByRelativePath( relativePath, promise ); @@ -103,8 +103,7 @@ export class Syncer { try { await this.syncQueue.add(async () => this.internalSyncer.unrestrictedSyncLocallyCreatedFile( - () => - this.database.getDocumentByIdentity(document.identity), + () => this.database.getDocumentByUpdatePromise(promise), updateTime ) ); @@ -120,18 +119,29 @@ export class Syncer { public async syncLocallyDeletedFile( relativePath: RelativePath ): Promise { + if (!this.settings.getSettings().isSyncEnabled) { + this.logger.info( + `Syncing is disabled, not syncing '${relativePath}'` + ); + return; + } + const [promise, resolve, reject] = createPromise(); - const document = await this.database.getResolvedDocumentByRelativePath( + await this.database.getResolvedDocumentByRelativePath( relativePath, promise ); try { await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncLocallyDeletedFile(() => - this.database.getDocumentByIdentity(document.identity) - ) + this.internalSyncer.unrestrictedSyncLocallyDeletedFile(() => { + this.logger.debug( + `aaaahg ${relativePath} has been deleted locally, syncing to delete it` + ); + + return this.database.getDocumentByUpdatePromise(promise); + }) ); resolve(); @@ -142,34 +152,46 @@ export class Syncer { } } - public async syncLocallyUpdatedFile(args: { + public async syncLocallyUpdatedFile({ + oldPath, + relativePath, + updateTime + }: { oldPath?: RelativePath; relativePath: RelativePath; updateTime?: Date; }): Promise { - if (args.oldPath !== undefined) { - if (args.oldPath === args.relativePath) { - throw new Error( - `Old path and new path are the same: ${args.oldPath}` - ); - } - - this.database.move(args.oldPath, args.relativePath); + if (!this.settings.getSettings().isSyncEnabled) { + this.logger.info( + `Syncing is disabled, not syncing '${relativePath}'` + ); + return; } const [promise, resolve, reject] = createPromise(); - const metadata = await this.database.getResolvedDocumentByRelativePath( - args.relativePath, + if (oldPath !== undefined) { + if (oldPath === relativePath) { + throw new Error( + `Old path and new path are the same: ${oldPath}` + ); + } + + this.database.move(oldPath, relativePath); + } + + await this.database.getResolvedDocumentByRelativePath( + relativePath, promise ); try { await this.syncQueue.add(async () => this.internalSyncer.unrestrictedSyncLocallyUpdatedFile({ - ...args, + oldPath, + updateTime, getLatestDocument: () => - this.database.getDocumentByIdentity(metadata.identity) + this.database.getDocumentByUpdatePromise(promise) }) ); @@ -189,7 +211,7 @@ export class Syncer { return; } - if (this.runningScheduleSyncForOfflineChanges != null) { + if (this.runningScheduleSyncForOfflineChanges !== undefined) { this.logger.debug("Uploading local changes is already in progress"); return this.runningScheduleSyncForOfflineChanges; } @@ -244,9 +266,7 @@ export class Syncer { public async reset(): Promise { this.syncQueue.clear(); await this.syncQueue.onEmpty(); - this.remainingOperationsListeners.forEach((listener) => { - listener(0); - }); + this.remainingOperationsListeners.forEach((listener) => listener(0)); this.internalSyncer.reset(); } @@ -257,6 +277,15 @@ export class Syncer { remoteVersion.documentId ); + if (document === undefined) { + const candidate = this.database.getLatestDocumentByRelativePath( + remoteVersion.relativePath + ); + if (candidate !== undefined && candidate.metadata === undefined) { + document = candidate; + } + } + if (document === undefined) { await this.syncQueue.add(async () => this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile( @@ -269,7 +298,7 @@ export class Syncer { const [promise, resolve, reject] = createPromise(); - document = await this.database.getResolvedDocumentByRelativePath( + await this.database.getResolvedDocumentByRelativePath( document.relativePath, promise ); @@ -278,7 +307,7 @@ export class Syncer { await this.syncQueue.add(async () => this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile( remoteVersion, - () => this.database.getDocumentByIdentity(document.identity) + () => this.database.getDocumentByUpdatePromise(promise) ) ); @@ -300,7 +329,7 @@ export class Syncer { const updates = Promise.all( allLocalFiles.map(async (relativePath) => { if ( - this.database.getDocumentByRelativePath(relativePath) + this.database.getLatestDocumentByRelativePath(relativePath) ?.metadata !== undefined ) { this.logger.debug( diff --git a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts index b1951f89..109cc673 100644 --- a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts +++ b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts @@ -1,6 +1,5 @@ import type { Database, - DocumentMetadata, DocumentRecord, RelativePath } from "../persistence/database"; @@ -61,7 +60,7 @@ export class UnrestrictedSyncer { createdDate: updateTime }); - const { relativePath: currentRelativePath } = + const { relativePath: currentRelativePath, identity } = getLatestDocument(); this.history.addHistoryEntry({ @@ -80,7 +79,7 @@ export class UnrestrictedSyncer { isDeleted: false }; - this.database.setDocument(newMetadata); + this.database.setDocument(newMetadata, identity); this.tryIncrementVaultUpdateId(response.vaultUpdateId); } @@ -101,7 +100,7 @@ export class UnrestrictedSyncer { document.metadata.isDeleted ) { this.logger.debug( - `Document ${document.relativePath} has been already deleted, no need to delete it again` + `Document '${document.relativePath}' has been already deleted, no need to delete it again` ); return; } @@ -124,13 +123,16 @@ export class UnrestrictedSyncer { // We have to have a record of the delete in case there's an in-flight update for the same // document which finishes after the delete has succeeded and would introduce a phantom metadata record. - this.database.setDocument({ - relativePath: document.relativePath, - documentId: response.documentId, - parentVersionId: response.vaultUpdateId, - hash: EMPTY_HASH, - isDeleted: true - }); + this.database.setDocument( + { + relativePath: document.relativePath, + documentId: response.documentId, + parentVersionId: response.vaultUpdateId, + hash: EMPTY_HASH, + isDeleted: true + }, + document.identity + ); } ); } @@ -222,13 +224,16 @@ export class UnrestrictedSyncer { type: SyncType.DELETE }); - this.database.setDocument({ - documentId: response.documentId, - relativePath: document.relativePath, - parentVersionId: response.vaultUpdateId, - hash: EMPTY_HASH, - isDeleted: true - }); + this.database.setDocument( + { + documentId: response.documentId, + relativePath: document.relativePath, + parentVersionId: response.vaultUpdateId, + hash: EMPTY_HASH, + isDeleted: true + }, + document.identity + ); this.tryIncrementVaultUpdateId(response.vaultUpdateId); @@ -262,16 +267,19 @@ export class UnrestrictedSyncer { }); } - this.database.setDocument({ - documentId: response.documentId, - relativePath: - response.relativePath != document.relativePath - ? response.relativePath - : document.relativePath, - parentVersionId: response.vaultUpdateId, - hash: contentHash, - isDeleted: response.isDeleted - }); + this.database.setDocument( + { + documentId: response.documentId, + relativePath: + response.relativePath != document.relativePath + ? response.relativePath + : document.relativePath, + parentVersionId: response.vaultUpdateId, + hash: contentHash, + isDeleted: response.isDeleted + }, + document.identity + ); this.tryIncrementVaultUpdateId(response.vaultUpdateId); } @@ -293,10 +301,7 @@ export class UnrestrictedSyncer { remoteVersion.documentId ); - if ( - localMetadata?.metadata !== undefined && - !localMetadata.metadata.isDeleted - ) { + if (localMetadata?.metadata !== undefined) { // If the file exists locally, let's pretend the user has updated it // and deal with remote update/deletion within `unrestrictedSyncLocallyUpdatedFile` if ( @@ -315,6 +320,11 @@ export class UnrestrictedSyncer { localMetadata.identity ) }); + } else if (remoteVersion.isDeleted) { + this.logger.debug( + `Document ${remoteVersion.relativePath} has been deleted remotely, no need to sync` + ); + return; } const content = ( @@ -330,13 +340,22 @@ export class UnrestrictedSyncer { remoteVersion.documentId ); - this.database.setDocument({ - documentId: remoteVersion.documentId, - relativePath: remoteVersion.relativePath, - parentVersionId: remoteVersion.vaultUpdateId, - hash: hash(contentBytes), - isDeleted: remoteVersion.isDeleted - }); + this.database.setDocument( + { + documentId: remoteVersion.documentId, + relativePath: remoteVersion.relativePath, + parentVersionId: remoteVersion.vaultUpdateId, + hash: hash(contentBytes), + isDeleted: remoteVersion.isDeleted + }, + getLatestDocument?.()?.identity ?? + this.database.getDocumentByDocumentId( + remoteVersion.documentId + )?.identity ?? + this.database.getLatestDocumentByRelativePath( + remoteVersion.relativePath + )?.identity + ); this.history.addHistoryEntry({ status: SyncStatus.SUCCESS, @@ -359,7 +378,7 @@ export class UnrestrictedSyncer { if (!this.settings.getSettings().isSyncEnabled) { this.logger.info( - `Syncing is disabled, not syncing ${relativePath}` + `Syncing is disabled, not syncing '${relativePath}'` ); return; } diff --git a/frontend/test-client/run.sh b/frontend/test-client/run.sh index 6effc147..5bf19a63 100755 --- a/frontend/test-client/run.sh +++ b/frontend/test-client/run.sh @@ -16,7 +16,7 @@ npm run build pids=() for i in $(seq 1 $process_count); do - node dist/cli.js 2>&1 | tee "log_${i}.log" & + node dist/cli.js 2>&1 > "log_${i}.log" & pids+=($!) done From 67532f5d0c08ce6c01c26dba77f1377cd7c2ea03 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Wed, 12 Mar 2025 21:16:40 +0000 Subject: [PATCH 12/58] Isdeleted fix --- .../sync-client/src/persistence/database.ts | 104 ++++++++++-------- .../sync-client/src/sync-operations/syncer.ts | 2 + .../sync-operations/unrestricted-syncer.ts | 103 ++++++++++------- frontend/test-client/run.sh | 2 + frontend/test-client/src/cli.ts | 27 ++--- 5 files changed, 140 insertions(+), 98 deletions(-) diff --git a/frontend/sync-client/src/persistence/database.ts b/frontend/sync-client/src/persistence/database.ts index 0bbbd5b1..4418f55c 100644 --- a/frontend/sync-client/src/persistence/database.ts +++ b/frontend/sync-client/src/persistence/database.ts @@ -8,7 +8,6 @@ export interface DocumentMetadata { parentVersionId: VaultUpdateId; documentId: DocumentId; hash: string; - isDeleted: boolean; } export interface StoredDocumentMetadata { @@ -16,7 +15,6 @@ export interface StoredDocumentMetadata { parentVersionId: VaultUpdateId; documentId: DocumentId; hash: string; - isDeleted: boolean; } export interface StoredDatabase { @@ -26,10 +24,11 @@ export interface StoredDatabase { export interface DocumentRecord { identity: symbol; - parallelVersion: number; relativePath: RelativePath; metadata: DocumentMetadata | undefined; + isDeleted: boolean; updates: Promise[]; + parallelVersion: number; } export class Database { @@ -48,6 +47,7 @@ export class Database { relativePath, identity: Symbol(), metadata, + isDeleted: false, updates: [], parallelVersion: 0 })) ?? []; @@ -68,9 +68,7 @@ export class Database { public get resolvedDocuments(): DocumentRecord[] { const paths = new Map(); this.documents - .filter( - ({ metadata }) => metadata !== undefined && !metadata.isDeleted - ) + .filter(({ metadata }) => metadata !== undefined) .forEach((record) => paths.set(record.relativePath, [ record, @@ -120,62 +118,70 @@ export class Database { documentId, relativePath, parentVersionId, - hash, - isDeleted + hash }: { documentId: DocumentId; relativePath: RelativePath; parentVersionId: VaultUpdateId; hash: string; - isDeleted: boolean; }, identity?: symbol ): void { let entry: DocumentRecord | undefined; if (identity !== undefined) { - entry = this.getDocumentByIdentity(identity); + const entry = this.getDocumentByIdentity(identity); - if (entry !== undefined) { - this.documents = this.documents.filter( - ({ identity }) => identity !== entry!.identity - ); - } - } else { - entry = this.getLatestDocumentByRelativePath(relativePath); - if ( - entry?.metadata?.documentId !== undefined && - entry.metadata.documentId !== documentId - ) { - this.documents.push({ - // `entry` might be undefined if the document is new - identity: Symbol(), - relativePath, - metadata: { - documentId, - parentVersionId, - hash, - isDeleted - }, - updates: [], - parallelVersion: entry?.parallelVersion + 1 - }); - } + this.documents = this.documents.filter( + ({ identity }) => identity !== entry.identity + ); + + this.documents.push({ + ...entry, + relativePath, + metadata: { + documentId, + parentVersionId, + hash + } + }); + + this.save(); + return; + } + + // We find a match based on relative path and we find one with a different document id + // meaning that two documents occupy the same path in terms of in-flight requests so we + // need to create a new parallel version. + entry = this.getLatestDocumentByRelativePath(relativePath); + if (entry && entry.metadata?.documentId !== documentId) { + this.documents.push({ + // `entry` might be undefined if the document is new + identity: Symbol(), + relativePath, + metadata: { + documentId, + parentVersionId, + hash + }, + isDeleted: false, + updates: [], + parallelVersion: entry.parallelVersion + 1 + }); this.save(); return; } this.documents.push({ - // `entry` might be undefined if the document is new - identity: entry?.identity ?? Symbol(), + identity: Symbol(), relativePath, metadata: { documentId, parentVersionId, - hash, - isDeleted + hash }, - updates: entry?.updates ?? [], - parallelVersion: entry?.parallelVersion ?? 0 + isDeleted: false, + updates: [], + parallelVersion: 0 }); this.save(); @@ -208,6 +214,7 @@ export class Database { relativePath, identity: Symbol(), metadata: undefined, + isDeleted: false, updates: [], parallelVersion: 0 }; @@ -257,10 +264,9 @@ export class Database { const oldDocument = this.getLatestDocumentByRelativePath(oldRelativePath); if (oldDocument === undefined) { + // We can try moving a non-existent document if it hasn't yet got created becasue it's + // the result of an offline event while this move happens online before. return; - throw new Error( - `Document to be moved not found: ${oldRelativePath}` - ); } this.documents = this.documents.filter( @@ -274,6 +280,7 @@ export class Database { identity: oldDocument.identity, metadata: oldDocument.metadata, relativePath: newRelativePath, + isDeleted: oldDocument.isDeleted, updates: oldDocument.updates, // We're in a strange state where the target of the move has just got deleted, // however, its metadata might already have a bunch of updates queued up for @@ -285,6 +292,15 @@ export class Database { this.save(); } + public delete(relativePath: RelativePath): void { + const candidate = this.getLatestDocumentByRelativePath(relativePath); + if (candidate === undefined) { + // it's fine because the document to be deleted might not have been created yet + return; + } + candidate.isDeleted = true; + } + private save(): void { this.logger.debug(JSON.stringify(this.documents, null, 2)); diff --git a/frontend/sync-client/src/sync-operations/syncer.ts b/frontend/sync-client/src/sync-operations/syncer.ts index e61e7c45..531e735a 100644 --- a/frontend/sync-client/src/sync-operations/syncer.ts +++ b/frontend/sync-client/src/sync-operations/syncer.ts @@ -126,6 +126,8 @@ export class Syncer { return; } + this.database.delete(relativePath); + const [promise, resolve, reject] = createPromise(); await this.database.getResolvedDocumentByRelativePath( diff --git a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts index 109cc673..5e4cf754 100644 --- a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts +++ b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts @@ -34,52 +34,57 @@ export class UnrestrictedSyncer { getLatestDocument: () => DocumentRecord, updateTime?: Date ): Promise { - const { relativePath, metadata } = getLatestDocument(); + let latestDocument = getLatestDocument(); return this.executeSync( - [relativePath], + [latestDocument.relativePath], SyncType.CREATE, SyncSource.PUSH, async () => { - if (metadata !== undefined && !metadata.isDeleted) { + if ( + latestDocument.metadata !== undefined && + !latestDocument.isDeleted + ) { this.logger.debug( - `Document ${relativePath} already exists in the database, no need to create it again` + `Document ${latestDocument.relativePath} already exists in the database, no need to create it again` ); return; } - const contentBytes = await this.operations.read(relativePath); // this can throw FileNotFoundError + const contentBytes = await this.operations.read( + latestDocument.relativePath + ); // this can throw FileNotFoundError const contentHash = hash(contentBytes); - updateTime ??= - await this.operations.getModificationTime(relativePath); // this can throw FileNotFoundError + updateTime ??= await this.operations.getModificationTime( + latestDocument.relativePath + ); // this can throw FileNotFoundError const response = await this.syncService.create({ - relativePath, + relativePath: latestDocument.relativePath, contentBytes, createdDate: updateTime }); - const { relativePath: currentRelativePath, identity } = - getLatestDocument(); + latestDocument = getLatestDocument(); this.history.addHistoryEntry({ status: SyncStatus.SUCCESS, source: SyncSource.PUSH, - relativePath, + relativePath: latestDocument.relativePath, message: `Successfully uploaded locally created file`, type: SyncType.CREATE }); const newMetadata = { - relativePath: currentRelativePath, + relativePath: latestDocument.relativePath, documentId: response.documentId, parentVersionId: response.vaultUpdateId, hash: contentHash, isDeleted: false }; - this.database.setDocument(newMetadata, identity); + this.database.setDocument(newMetadata, latestDocument.identity); this.tryIncrementVaultUpdateId(response.vaultUpdateId); } @@ -95,12 +100,9 @@ export class UnrestrictedSyncer { SyncType.DELETE, SyncSource.PUSH, async () => { - if ( - document.metadata === undefined || - document.metadata.isDeleted - ) { + if (document.metadata === undefined) { this.logger.debug( - `Document '${document.relativePath}' has been already deleted, no need to delete it again` + `Document '${document.relativePath}' has been created yet so deleting it remotely can be skipped` ); return; } @@ -128,8 +130,7 @@ export class UnrestrictedSyncer { relativePath: document.relativePath, documentId: response.documentId, parentVersionId: response.vaultUpdateId, - hash: EMPTY_HASH, - isDeleted: true + hash: EMPTY_HASH }, document.identity ); @@ -155,12 +156,9 @@ export class UnrestrictedSyncer { SyncType.UPDATE, SyncSource.PUSH, async () => { - if ( - document.metadata === undefined || - document.metadata.isDeleted - ) { + if (document.metadata === undefined || document.isDeleted) { this.logger.debug( - `Document ${document.relativePath} has been already deleted, no need to update it, ${JSON.stringify(document)}, ${document.metadata?.isDeleted}` + `Document ${document.relativePath} has been already deleted, no need to update it` ); return; } @@ -192,6 +190,22 @@ export class UnrestrictedSyncer { createdDate: updateTime }); + // Update relativePath which is the only property that can change while this is running (due to a move) + document = getLatestDocument(); + + if (document.isDeleted) { + this.logger.info( + `Document ${document.relativePath} has been deleted before we could finish updating it` + ); + return; + } + + if (!document.metadata) { + throw new Error( + `Document ${document.relativePath} no longer has metadata after updating it` + ); + } + if ( document.metadata.parentVersionId >= response.vaultUpdateId ) { @@ -209,9 +223,6 @@ export class UnrestrictedSyncer { type: SyncType.UPDATE }); - // Update relativePath which is the only property that can change while this is running (due to a move) - document = getLatestDocument(); - if (response.isDeleted) { await this.operations.delete(document.relativePath); @@ -224,13 +235,13 @@ export class UnrestrictedSyncer { type: SyncType.DELETE }); + this.database.delete(document.relativePath); this.database.setDocument( { documentId: response.documentId, relativePath: document.relativePath, parentVersionId: response.vaultUpdateId, - hash: EMPTY_HASH, - isDeleted: true + hash: EMPTY_HASH }, document.identity ); @@ -267,6 +278,8 @@ export class UnrestrictedSyncer { }); } + document = getLatestDocument(); + this.database.setDocument( { documentId: response.documentId, @@ -275,8 +288,7 @@ export class UnrestrictedSyncer { ? response.relativePath : document.relativePath, parentVersionId: response.vaultUpdateId, - hash: contentHash, - isDeleted: response.isDeleted + hash: contentHash }, document.identity ); @@ -321,6 +333,8 @@ export class UnrestrictedSyncer { ) }); } else if (remoteVersion.isDeleted) { + // Either the doc hasn't made it to us before and therefore we don't need to delete it, + // or we already have it, in which case the preceeding if will deal with it this.logger.debug( `Document ${remoteVersion.relativePath} has been deleted remotely, no need to sync` ); @@ -332,6 +346,20 @@ export class UnrestrictedSyncer { documentId: remoteVersion.documentId }) ).contentBase64; + + const latestDocument = + getLatestDocument?.() ?? + this.database.getDocumentByDocumentId( + remoteVersion.documentId + ); + + if (latestDocument?.isDeleted) { + this.logger.info( + `Document ${remoteVersion.relativePath} has been deleted locally before we could finish updating it` + ); + return; + } + const contentBytes = deserialize(content); await this.operations.create( @@ -345,16 +373,9 @@ export class UnrestrictedSyncer { documentId: remoteVersion.documentId, relativePath: remoteVersion.relativePath, parentVersionId: remoteVersion.vaultUpdateId, - hash: hash(contentBytes), - isDeleted: remoteVersion.isDeleted + hash: hash(contentBytes) }, - getLatestDocument?.()?.identity ?? - this.database.getDocumentByDocumentId( - remoteVersion.documentId - )?.identity ?? - this.database.getLatestDocumentByRelativePath( - remoteVersion.relativePath - )?.identity + latestDocument?.identity ); this.history.addHistoryEntry({ diff --git a/frontend/test-client/run.sh b/frontend/test-client/run.sh index 5bf19a63..5bb39e94 100755 --- a/frontend/test-client/run.sh +++ b/frontend/test-client/run.sh @@ -41,6 +41,8 @@ print_failed_log() { return 1 } +echo "Monitoring $process_count processes" + # Monitor processes while true; do if print_failed_log; then diff --git a/frontend/test-client/src/cli.ts b/frontend/test-client/src/cli.ts index e87666f3..27c59f75 100644 --- a/frontend/test-client/src/cli.ts +++ b/frontend/test-client/src/cli.ts @@ -96,19 +96,20 @@ async function runTests(): Promise { const iterations = [50, 200]; const doDeletes = [true, false]; - for (const agentCount of agentCounts) { - for (const concurrency of concurrencies) { - for (const jitter of jitterScaleInSeconds) { - for (const iteration of iterations) { - for (const deleteFiles of doDeletes) { - await runTest({ - agentCount, - concurrency, - iterations: iteration, - doDeletes: deleteFiles, - jitterScaleInSeconds: jitter - }); - return; + for (let i = 0; i < 10; i++) { + for (const agentCount of agentCounts) { + for (const concurrency of concurrencies) { + for (const jitter of jitterScaleInSeconds) { + for (const iteration of iterations) { + for (const deleteFiles of doDeletes) { + await runTest({ + agentCount, + concurrency, + iterations: iteration, + doDeletes: deleteFiles, + jitterScaleInSeconds: jitter + }); + } } } } From 53b9b51f5fb9881c35c9713387fbd4164bd19863 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Wed, 12 Mar 2025 21:16:53 +0000 Subject: [PATCH 13/58] remove created dates --- backend/sync_server/src/database.rs | 9 +-------- .../src/database/migrations/20241207143519_bootstrap.sql | 1 - backend/sync_server/src/database/models.rs | 5 ----- backend/sync_server/src/server/create_document.rs | 5 ----- backend/sync_server/src/server/delete_document.rs | 1 - backend/sync_server/src/server/requests.rs | 6 ------ backend/sync_server/src/server/update_document.rs | 5 ----- 7 files changed, 1 insertion(+), 31 deletions(-) diff --git a/backend/sync_server/src/database.rs b/backend/sync_server/src/database.rs index 52305629..89fa8229 100644 --- a/backend/sync_server/src/database.rs +++ b/backend/sync_server/src/database.rs @@ -84,7 +84,6 @@ impl Database { vault_update_id, document_id as "document_id: uuid::Uuid", relative_path, - created_date as "created_date: chrono::DateTime", updated_date as "updated_date: chrono::DateTime", is_deleted from latest_document_versions @@ -118,7 +117,6 @@ impl Database { vault_update_id, document_id as "document_id: uuid::Uuid", relative_path, - created_date as "created_date: chrono::DateTime", updated_date as "updated_date: chrono::DateTime", is_deleted from latest_document_versions @@ -176,7 +174,6 @@ impl Database { vault_update_id, document_id as "document_id: uuid::Uuid", relative_path, - created_date as "created_date: chrono::DateTime", updated_date as "updated_date: chrono::DateTime", content, is_deleted @@ -213,7 +210,6 @@ impl Database { vault_update_id, document_id as "document_id: uuid::Uuid", relative_path, - created_date as "created_date: chrono::DateTime", updated_date as "updated_date: chrono::DateTime", content, is_deleted @@ -246,7 +242,6 @@ impl Database { vault_update_id, document_id as "document_id: uuid::Uuid", relative_path, - created_date as "created_date: chrono::DateTime", updated_date as "updated_date: chrono::DateTime", content, is_deleted @@ -276,18 +271,16 @@ impl Database { vault_update_id, document_id, relative_path, - created_date, updated_date, content, is_deleted ) - values (?, ?, ?, ?, ?, ?, ?, ?) + values (?, ?, ?, ?, ?, ?, ?) "#, version.vault_id, version.vault_update_id, version.document_id, version.relative_path, - version.created_date, version.updated_date, version.content, version.is_deleted diff --git a/backend/sync_server/src/database/migrations/20241207143519_bootstrap.sql b/backend/sync_server/src/database/migrations/20241207143519_bootstrap.sql index 360b34d2..62002b58 100644 --- a/backend/sync_server/src/database/migrations/20241207143519_bootstrap.sql +++ b/backend/sync_server/src/database/migrations/20241207143519_bootstrap.sql @@ -3,7 +3,6 @@ CREATE TABLE IF NOT EXISTS documents ( vault_update_id INTEGER NOT NULL, document_id TEXT NOT NULL, relative_path TEXT NOT NULL, - created_date TIMESTAMP NOT NULL, updated_date TIMESTAMP NOT NULL, content BLOB NOT NULL, is_deleted BOOLEAN NOT NULL, diff --git a/backend/sync_server/src/database/models.rs b/backend/sync_server/src/database/models.rs index d8f743a9..9ba1832b 100644 --- a/backend/sync_server/src/database/models.rs +++ b/backend/sync_server/src/database/models.rs @@ -13,7 +13,6 @@ pub struct StoredDocumentVersion { pub vault_update_id: VaultUpdateId, pub document_id: DocumentId, pub relative_path: String, - pub created_date: DateTime, pub updated_date: DateTime, pub content: Vec, pub is_deleted: bool, @@ -32,7 +31,6 @@ pub struct DocumentVersionWithoutContent { pub vault_update_id: VaultUpdateId, pub document_id: DocumentId, pub relative_path: String, - pub created_date: DateTime, pub updated_date: DateTime, pub is_deleted: bool, } @@ -44,7 +42,6 @@ impl From for DocumentVersionWithoutContent { vault_update_id: value.vault_update_id, document_id: value.document_id, relative_path: value.relative_path, - created_date: value.created_date, updated_date: value.updated_date, is_deleted: value.is_deleted, } @@ -58,7 +55,6 @@ pub struct DocumentVersion { pub vault_update_id: VaultUpdateId, pub document_id: DocumentId, pub relative_path: String, - pub created_date: DateTime, pub updated_date: DateTime, pub content_base64: String, pub is_deleted: bool, @@ -71,7 +67,6 @@ impl From for DocumentVersion { vault_update_id: value.vault_update_id, document_id: value.document_id, relative_path: value.relative_path, - created_date: value.created_date, updated_date: value.updated_date, content_base64: bytes_to_base64(&value.content), is_deleted: value.is_deleted, diff --git a/backend/sync_server/src/server/create_document.rs b/backend/sync_server/src/server/create_document.rs index a2567939..e432cb5a 100644 --- a/backend/sync_server/src/server/create_document.rs +++ b/backend/sync_server/src/server/create_document.rs @@ -6,7 +6,6 @@ use axum_extra::{ headers::{Authorization, authorization::Bearer}, }; use axum_jsonschema::Json; -use chrono::{DateTime, Utc}; use schemars::JsonSchema; use serde::Deserialize; use sync_lib::base64_to_bytes; @@ -45,7 +44,6 @@ pub async fn create_document_multipart( state, vault_id, request.relative_path, - request.created_date, request.content.contents.to_vec(), ) .await @@ -70,7 +68,6 @@ pub async fn create_document_json( state, vault_id, request.relative_path, - request.created_date, content_bytes, ) .await @@ -81,7 +78,6 @@ async fn internal_create_document( state: AppState, vault_id: VaultId, relative_path: String, - created_date: DateTime, content: Vec, ) -> Result, SyncServerError> { auth(&state, auth_header.token())?; @@ -106,7 +102,6 @@ async fn internal_create_document( document_id: uuid::Uuid::new_v4(), relative_path: sanitized_relative_path, content, - created_date, updated_date: chrono::Utc::now(), is_deleted: false, }; diff --git a/backend/sync_server/src/server/delete_document.rs b/backend/sync_server/src/server/delete_document.rs index afef37a7..25901e84 100644 --- a/backend/sync_server/src/server/delete_document.rs +++ b/backend/sync_server/src/server/delete_document.rs @@ -52,7 +52,6 @@ pub async fn delete_document( document_id, relative_path: sanitize_path(&request.relative_path), content: vec![], - created_date: request.created_date, updated_date: chrono::Utc::now(), is_deleted: true, }; diff --git a/backend/sync_server/src/server/requests.rs b/backend/sync_server/src/server/requests.rs index 1720f96f..b55d1c47 100644 --- a/backend/sync_server/src/server/requests.rs +++ b/backend/sync_server/src/server/requests.rs @@ -1,7 +1,6 @@ use aide_axum_typed_multipart::FieldData; use axum::body::Bytes; use axum_typed_multipart::TryFromMultipart; -use chrono::{DateTime, Utc}; use schemars::JsonSchema; use serde::{self, Deserialize}; @@ -11,14 +10,12 @@ use crate::database::models::VaultUpdateId; #[serde(rename_all = "camelCase")] pub struct CreateDocumentVersion { pub relative_path: String, - pub created_date: DateTime, pub content_base64: String, } #[derive(Debug, TryFromMultipart, JsonSchema)] pub struct CreateDocumentVersionMultipart { pub relative_path: String, - pub created_date: DateTime, #[form_data(limit = "unlimited")] pub content: FieldData, } @@ -28,7 +25,6 @@ pub struct CreateDocumentVersionMultipart { pub struct UpdateDocumentVersion { pub parent_version_id: VaultUpdateId, pub relative_path: String, - pub created_date: DateTime, pub content_base64: String, } @@ -37,7 +33,6 @@ pub struct UpdateDocumentVersion { pub struct UpdateDocumentVersionMultipart { pub parent_version_id: VaultUpdateId, pub relative_path: String, - pub created_date: DateTime, #[form_data(limit = "unlimited")] pub content: FieldData, } @@ -46,5 +41,4 @@ pub struct UpdateDocumentVersionMultipart { #[serde(rename_all = "camelCase")] pub struct DeleteDocumentVersion { pub relative_path: String, - pub created_date: DateTime, } diff --git a/backend/sync_server/src/server/update_document.rs b/backend/sync_server/src/server/update_document.rs index 17a647ae..316b06f4 100644 --- a/backend/sync_server/src/server/update_document.rs +++ b/backend/sync_server/src/server/update_document.rs @@ -6,7 +6,6 @@ use axum_extra::{ headers::{Authorization, authorization::Bearer}, }; use axum_jsonschema::Json; -use chrono::{DateTime, Utc}; use log::info; use schemars::JsonSchema; use serde::Deserialize; @@ -50,7 +49,6 @@ pub async fn update_document_multipart( document_id, request.parent_version_id, request.relative_path, - request.created_date, request.content.contents.to_vec(), ) .await @@ -77,7 +75,6 @@ pub async fn update_document_json( document_id, request.parent_version_id, request.relative_path, - request.created_date, content_bytes, ) .await @@ -91,7 +88,6 @@ async fn internal_update_document( document_id: DocumentId, parent_version_id: VaultUpdateId, relative_path: String, - created_date: DateTime, content: Vec, ) -> Result, SyncServerError> { auth(&state, auth_header.token())?; @@ -205,7 +201,6 @@ async fn internal_update_document( vault_update_id: last_update_id + 1, relative_path: new_relative_path, content: merged_content, - created_date, updated_date: chrono::Utc::now(), is_deleted: false, }; From d5ff50a1b0447fbe2aedb01de5954da2984dce4a Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Wed, 12 Mar 2025 21:17:14 +0000 Subject: [PATCH 14/58] update api --- .../src/file-operations/file-operations.ts | 4 ---- .../file-operations/filesystem-operations.ts | 1 - .../safe-filesystem-operations.ts | 11 ----------- frontend/sync-client/src/services/types.ts | 18 ------------------ frontend/test-client/src/agent/mock-client.ts | 18 ++++-------------- 5 files changed, 4 insertions(+), 48 deletions(-) diff --git a/frontend/sync-client/src/file-operations/file-operations.ts b/frontend/sync-client/src/file-operations/file-operations.ts index ef9dda55..c3584de5 100644 --- a/frontend/sync-client/src/file-operations/file-operations.ts +++ b/frontend/sync-client/src/file-operations/file-operations.ts @@ -49,10 +49,6 @@ export class FileOperations { return this.fs.getFileSize(path); } - public async getModificationTime(path: RelativePath): Promise { - return this.fs.getModificationTime(path); - } - public async exists(path: RelativePath): Promise { return this.fs.exists(path); } diff --git a/frontend/sync-client/src/file-operations/filesystem-operations.ts b/frontend/sync-client/src/file-operations/filesystem-operations.ts index b58d3c23..7c51ec78 100644 --- a/frontend/sync-client/src/file-operations/filesystem-operations.ts +++ b/frontend/sync-client/src/file-operations/filesystem-operations.ts @@ -9,7 +9,6 @@ export interface FileSystemOperations { updater: (currentContent: string) => string ) => Promise; getFileSize: (path: RelativePath) => Promise; - getModificationTime: (path: RelativePath) => Promise; exists: (path: RelativePath) => Promise; createDirectory: (path: RelativePath) => Promise; delete: (path: RelativePath) => Promise; diff --git a/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts b/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts index f1036073..a2f7d111 100644 --- a/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts +++ b/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts @@ -69,17 +69,6 @@ export class SafeFileSystemOperations implements FileSystemOperations { ); } - public async getModificationTime(path: RelativePath): Promise { - this.logger.debug(`Getting modification time: ${path}`); - return this.safeOperation( - path, - this.decorateToHoldLock(path, async () => - this.fs.getModificationTime(path) - ), - "getModificationTime" - ); - } - public async exists(path: RelativePath): Promise { this.logger.debug(`Checking if file exists: ${path}`); return this.decorateToHoldLock(path, async () => diff --git a/frontend/sync-client/src/services/types.ts b/frontend/sync-client/src/services/types.ts index c1c4446e..eba3d9f0 100644 --- a/frontend/sync-client/src/services/types.ts +++ b/frontend/sync-client/src/services/types.ts @@ -452,25 +452,17 @@ export interface components { Array_of_uint8: number[]; CreateDocumentVersion: { contentBase64: string; - /** Format: date-time */ - createdDate: string; relativePath: string; }; CreateDocumentVersionMultipart: { content: components["schemas"]["Array_of_uint8"]; - /** Format: date-time */ - created_date: string; relative_path: string; }; DeleteDocumentVersion: { - /** Format: date-time */ - createdDate: string; relativePath: string; }; /** @description Response to an update document request. */ DocumentUpdateResponse: { - /** Format: date-time */ - createdDate: string; /** Format: uuid */ documentId: string; isDeleted: boolean; @@ -484,8 +476,6 @@ export interface components { vaultUpdateId: number; } | { contentBase64: string; - /** Format: date-time */ - createdDate: string; /** Format: uuid */ documentId: string; isDeleted: boolean; @@ -500,8 +490,6 @@ export interface components { }; DocumentVersion: { contentBase64: string; - /** Format: date-time */ - createdDate: string; /** Format: uuid */ documentId: string; isDeleted: boolean; @@ -513,8 +501,6 @@ export interface components { vaultUpdateId: number; }; DocumentVersionWithoutContent: { - /** Format: date-time */ - createdDate: string; /** Format: uuid */ documentId: string; isDeleted: boolean; @@ -586,16 +572,12 @@ export interface components { }; UpdateDocumentVersion: { contentBase64: string; - /** Format: date-time */ - createdDate: string; /** Format: int64 */ parentVersionId: number; relativePath: string; }; UpdateDocumentVersionMultipart: { content: components["schemas"]["Array_of_uint8"]; - /** Format: date-time */ - createdDate: string; /** Format: int64 */ parentVersionId: number; relativePath: string; diff --git a/frontend/test-client/src/agent/mock-client.ts b/frontend/test-client/src/agent/mock-client.ts index e627eb78..ec077778 100644 --- a/frontend/test-client/src/agent/mock-client.ts +++ b/frontend/test-client/src/agent/mock-client.ts @@ -46,13 +46,6 @@ export class MockClient implements FileSystemOperations { return (await this.read(path)).length; } - public async getModificationTime(path: RelativePath): Promise { - if (!this.localFiles.has(path)) { - throw new Error(`File ${path} does not exist`); - } - return new Date(); - } - public async exists(path: RelativePath): Promise { return this.localFiles.has(path); } @@ -68,7 +61,7 @@ export class MockClient implements FileSystemOperations { `Creating file ${path} with content ${new TextDecoder().decode(newContent)}` ); this.localFiles.set(path, newContent); - void this.client.syncer.syncLocallyCreatedFile(path, new Date()); + void this.client.syncer.syncLocallyCreatedFile(path); } public async createDirectory(_path: RelativePath): Promise { @@ -93,8 +86,7 @@ export class MockClient implements FileSystemOperations { ); void this.client.syncer.syncLocallyUpdatedFile({ - relativePath: path, - updateTime: new Date() + relativePath: path }); return newContent; @@ -108,8 +100,7 @@ export class MockClient implements FileSystemOperations { ); void this.client.syncer.syncLocallyUpdatedFile({ - relativePath: path, - updateTime: new Date() + relativePath: path }); } @@ -140,8 +131,7 @@ export class MockClient implements FileSystemOperations { void this.client.syncer.syncLocallyUpdatedFile({ oldPath, - relativePath: newPath, - updateTime: new Date() + relativePath: newPath }); } } From f894cd6bd8e6d4c44ef8e26cfb72c7037ee00843 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 15 Mar 2025 09:24:22 +0000 Subject: [PATCH 15/58] Take document id --- .../sync_server/src/server/create_document.rs | 26 +++++++++++++++++-- backend/sync_server/src/server/requests.rs | 8 +++++- frontend/sync-client/src/services/types.ts | 4 +++ 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/backend/sync_server/src/server/create_document.rs b/backend/sync_server/src/server/create_document.rs index e432cb5a..4d17effc 100644 --- a/backend/sync_server/src/server/create_document.rs +++ b/backend/sync_server/src/server/create_document.rs @@ -16,7 +16,7 @@ use super::{ requests::{CreateDocumentVersion, CreateDocumentVersionMultipart}, }; use crate::{ - database::models::{DocumentVersionWithoutContent, StoredDocumentVersion, VaultId}, + database::models::{DocumentId, DocumentVersionWithoutContent, StoredDocumentVersion, VaultId}, errors::{SyncServerError, client_error, server_error}, utils::sanitize_path, }; @@ -43,6 +43,7 @@ pub async fn create_document_multipart( auth_header, state, vault_id, + request.document_id, request.relative_path, request.content.contents.to_vec(), ) @@ -67,6 +68,7 @@ pub async fn create_document_json( auth_header, state, vault_id, + request.document_id, request.relative_path, content_bytes, ) @@ -77,6 +79,7 @@ async fn internal_create_document( auth_header: Authorization, state: AppState, vault_id: VaultId, + document_id: Option, relative_path: String, content: Vec, ) -> Result, SyncServerError> { @@ -88,6 +91,25 @@ async fn internal_create_document( .await .map_err(server_error)?; + let document_id = match document_id { + Some(document_id) => { + let existing_version = state + .database + .get_latest_document(&vault_id, &document_id, Some(&mut transaction)) + .await + .map_err(server_error)?; + + if existing_version.is_some() { + return Err(client_error(anyhow::anyhow!( + "Document with the same ID already exists" + ))); + } + + document_id + } + None => uuid::Uuid::new_v4(), + }; + let last_update_id = state .database .get_max_update_id_in_vault(&vault_id, Some(&mut transaction)) @@ -99,7 +121,7 @@ async fn internal_create_document( let new_version = StoredDocumentVersion { vault_id, vault_update_id: last_update_id + 1, - document_id: uuid::Uuid::new_v4(), + document_id, relative_path: sanitized_relative_path, content, updated_date: chrono::Utc::now(), diff --git a/backend/sync_server/src/server/requests.rs b/backend/sync_server/src/server/requests.rs index b55d1c47..3c888266 100644 --- a/backend/sync_server/src/server/requests.rs +++ b/backend/sync_server/src/server/requests.rs @@ -4,17 +4,23 @@ use axum_typed_multipart::TryFromMultipart; use schemars::JsonSchema; use serde::{self, Deserialize}; -use crate::database::models::VaultUpdateId; +use crate::database::models::{DocumentId, VaultUpdateId}; #[derive(Debug, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct CreateDocumentVersion { + /// The client can decide the document id (if it wishes to) in order + /// to help with syncing. If the client does not provide a document id, + /// the server will generate one. If the client provides a document id + /// it must not already exist in the database. + pub document_id: Option, pub relative_path: String, pub content_base64: String, } #[derive(Debug, TryFromMultipart, JsonSchema)] pub struct CreateDocumentVersionMultipart { + pub document_id: Option, pub relative_path: String, #[form_data(limit = "unlimited")] pub content: FieldData, diff --git a/frontend/sync-client/src/services/types.ts b/frontend/sync-client/src/services/types.ts index eba3d9f0..642fd6c2 100644 --- a/frontend/sync-client/src/services/types.ts +++ b/frontend/sync-client/src/services/types.ts @@ -452,10 +452,14 @@ export interface components { Array_of_uint8: number[]; CreateDocumentVersion: { contentBase64: string; + /** Format: uuid */ + documentId?: string | null; relativePath: string; }; CreateDocumentVersionMultipart: { content: components["schemas"]["Array_of_uint8"]; + /** Format: uuid */ + document_id?: string | null; relative_path: string; }; DeleteDocumentVersion: { From 408afa3626295ac26f618c557602c86be0cacb05 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 15 Mar 2025 09:24:54 +0000 Subject: [PATCH 16/58] No max attempt --- frontend/package-lock.json | 1 + frontend/sync-client/package.json | 3 ++- frontend/sync-client/src/utils/retried-fetch.ts | 1 - 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index e41f20a4..fe3e6b70 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -6737,6 +6737,7 @@ "ts-loader": "^9.5.2", "tslib": "2.8.1", "typescript": "5.7.3", + "uuid": "^11.1.0", "webpack": "^5.98.0", "webpack-cli": "^6.0.1" } diff --git a/frontend/sync-client/package.json b/frontend/sync-client/package.json index 94340f33..1d3618ca 100644 --- a/frontend/sync-client/package.json +++ b/frontend/sync-client/package.json @@ -12,6 +12,7 @@ "devDependencies": { "tslib": "2.8.1", "typescript": "5.7.3", + "uuid": "^11.1.0", "sync_lib": "file:../../backend/sync_lib/pkg", "@types/jest": "^29.5.14", "@types/node": "^22.13.5", @@ -26,4 +27,4 @@ "webpack": "^5.98.0", "webpack-cli": "^6.0.1" } -} +} \ No newline at end of file diff --git a/frontend/sync-client/src/utils/retried-fetch.ts b/frontend/sync-client/src/utils/retried-fetch.ts index e4c47f07..1a49c704 100644 --- a/frontend/sync-client/src/utils/retried-fetch.ts +++ b/frontend/sync-client/src/utils/retried-fetch.ts @@ -31,7 +31,6 @@ export function retriedFetchFactory( } return false; }, - retries: 6, retryDelay: (attempt) => Math.pow(1.5, attempt) * 500, ...init }); From e3196c2dc0988b4ddd0fbbd85c74576a54a85f77 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 15 Mar 2025 09:25:09 +0000 Subject: [PATCH 17/58] works --- .../src/file-operations/file-operations.ts | 24 +- .../sync-client/src/persistence/database.ts | 116 ++++++---- .../src/services/connected-state.ts | 52 +++++ .../sync-client/src/services/sync-service.ts | 38 +-- frontend/sync-client/src/sync-client.ts | 24 +- .../sync-client/src/sync-operations/syncer.ts | 218 ++++++++---------- .../sync-operations/unrestricted-syncer.ts | 137 +++++------ frontend/test-client/src/agent/mock-agent.ts | 2 - frontend/test-client/src/agent/mock-client.ts | 13 ++ frontend/test-client/src/cli.ts | 15 +- 10 files changed, 338 insertions(+), 301 deletions(-) create mode 100644 frontend/sync-client/src/services/connected-state.ts diff --git a/frontend/sync-client/src/file-operations/file-operations.ts b/frontend/sync-client/src/file-operations/file-operations.ts index c3584de5..8071a0f5 100644 --- a/frontend/sync-client/src/file-operations/file-operations.ts +++ b/frontend/sync-client/src/file-operations/file-operations.ts @@ -38,7 +38,7 @@ export class FileOperations { const decoder = new TextDecoder("utf-8"); - // Normalize line endings to LF on Windows + // Normalize line-endings to LF on Windows let text = decoder.decode(content); text = text.replace(/\r\n/g, "\n"); @@ -53,7 +53,7 @@ export class FileOperations { return this.fs.exists(path); } - // Create and write the file if it doesn't exist. Otherwise, it has the same behavior as write. + // Create and write the file if it doesn't exist.Otherwise, it has the same behavior as write. // All parent directories are created if they don't exist. public async create( path: RelativePath, @@ -73,20 +73,15 @@ export class FileOperations { `Existing metadata for ${path}: ${JSON.stringify(document?.metadata)}` ); - if ( - document?.metadata !== undefined && - document.metadata.documentId === documentId - ) { + if (document !== undefined && document.documentId === documentId) { // This can happen if the document got moved both locally and remotely // to the same file path. In this case, we shouldn't deconflict, however, // we also can't overwrite otherwise we'd lose changes. throw new FileNotFoundError(path); } - this.logger.debug( - `We need to save what's at ${path} to ${deconflictedPath}` - ); - await this.move(path, deconflictedPath, documentId); + this.database.move(path, deconflictedPath); + await this.fs.rename(path, deconflictedPath); } else { await this.createParentDirectories(path); } @@ -147,7 +142,7 @@ export class FileOperations { } public async delete(path: RelativePath): Promise { - if (!(await this.exists(path))) { + if (await this.exists(path)) { this.logger.debug(`Deleting file: ${path}`); return this.fs.delete(path); } else { @@ -175,7 +170,7 @@ export class FileOperations { if ( document?.metadata !== undefined && - document.metadata.documentId === documentId + document.documentId === documentId ) { // This can happen if the document got moved both locally and remotely // to the same file path. In this case, we shouldn't deconflict, however, @@ -183,12 +178,13 @@ export class FileOperations { throw new FileNotFoundError(newPath); } - await this.move(newPath, deconflictedPath, documentId); - // this.database.move(oldPath, newPath); + this.database.move(newPath, deconflictedPath); + await this.fs.rename(newPath, deconflictedPath); } else { await this.createParentDirectories(newPath); } + this.database.move(oldPath, newPath); await this.fs.rename(oldPath, newPath); } diff --git a/frontend/sync-client/src/persistence/database.ts b/frontend/sync-client/src/persistence/database.ts index 4418f55c..0987c0de 100644 --- a/frontend/sync-client/src/persistence/database.ts +++ b/frontend/sync-client/src/persistence/database.ts @@ -6,14 +6,13 @@ export type RelativePath = string; export interface DocumentMetadata { parentVersionId: VaultUpdateId; - documentId: DocumentId; hash: string; } export interface StoredDocumentMetadata { relativePath: RelativePath; - parentVersionId: VaultUpdateId; documentId: DocumentId; + parentVersionId: VaultUpdateId; hash: string; } @@ -25,6 +24,7 @@ export interface StoredDatabase { export interface DocumentRecord { identity: symbol; relativePath: RelativePath; + documentId: DocumentId; metadata: DocumentMetadata | undefined; isDeleted: boolean; updates: Promise[]; @@ -43,14 +43,17 @@ export class Database { initialState ??= {}; this.documents = - initialState.documents?.map(({ relativePath, ...metadata }) => ({ - relativePath, - identity: Symbol(), - metadata, - isDeleted: false, - updates: [], - parallelVersion: 0 - })) ?? []; + initialState.documents?.map( + ({ relativePath, documentId, ...metadata }) => ({ + relativePath, + documentId, + identity: Symbol(), + metadata, + isDeleted: false, + updates: [], + parallelVersion: 0 + }) + ) ?? []; this.ensureConsistency(); this.logger.debug(`Loaded ${this.documents.length} documents`); @@ -135,11 +138,17 @@ export class Database { ({ identity }) => identity !== entry.identity ); + if (entry.relativePath !== relativePath) { + throw new Error( + "Document identity does not match the relative path" + ); + } + this.documents.push({ ...entry, relativePath, + documentId, metadata: { - documentId, parentVersionId, hash } @@ -153,13 +162,13 @@ export class Database { // meaning that two documents occupy the same path in terms of in-flight requests so we // need to create a new parallel version. entry = this.getLatestDocumentByRelativePath(relativePath); - if (entry && entry.metadata?.documentId !== documentId) { + if (entry && entry.documentId !== documentId) { this.documents.push({ // `entry` might be undefined if the document is new identity: Symbol(), relativePath, + documentId, metadata: { - documentId, parentVersionId, hash }, @@ -174,8 +183,8 @@ export class Database { this.documents.push({ identity: Symbol(), relativePath, + documentId, metadata: { - documentId, parentVersionId, hash }, @@ -210,16 +219,13 @@ export class Database { let entry = this.getLatestDocumentByRelativePath(relativePath); if (entry === undefined) { - entry = { - relativePath, - identity: Symbol(), - metadata: undefined, - isDeleted: false, - updates: [], - parallelVersion: 0 - }; - - this.documents.push(entry); + throw new Error( + `Document not found by relative path: ${relativePath}, ${JSON.stringify( + this.documents, + null, + 2 + )}` + ); } const currentPromises = entry.updates; @@ -227,6 +233,30 @@ export class Database { await Promise.all(currentPromises); } + public getNewResolvedDocumentByRelativePath( + documentId: DocumentId, + relativePath: RelativePath, + promise: Promise + ): void { + let previousEntry = this.getLatestDocumentByRelativePath(relativePath); + + const entry = { + relativePath, + documentId, + identity: Symbol(), + metadata: undefined, + isDeleted: false, + updates: [promise], + parallelVersion: + previousEntry?.parallelVersion === undefined + ? 0 + : previousEntry.parallelVersion + 1 + }; + + this.documents.push(entry); + this.save(); + } + public getDocumentByUpdatePromise(promise: Promise): DocumentRecord { const result = this.documents.find(({ updates }) => updates.includes(promise) @@ -240,11 +270,9 @@ export class Database { } public getDocumentByDocumentId( - documentId: DocumentId + find: DocumentId ): DocumentRecord | undefined { - return this.documents.find( - ({ metadata }) => metadata?.documentId === documentId - ); + return this.documents.find(({ documentId }) => documentId === find); } public getDocumentByIdentity(find: symbol): DocumentRecord { @@ -263,9 +291,8 @@ export class Database { ): void { const oldDocument = this.getLatestDocumentByRelativePath(oldRelativePath); + if (oldDocument === undefined) { - // We can try moving a non-existent document if it hasn't yet got created becasue it's - // the result of an offline event while this move happens online before. return; } @@ -275,13 +302,11 @@ export class Database { let newDocument = this.getLatestDocumentByRelativePath(newRelativePath); - // It's either an invalid state of newDocument is pending deletion and we have to wait for it to complete + // It's either an invalid state of newDocument is pending deletion and we have + // to wait for it to complete. this.documents.push({ - identity: oldDocument.identity, - metadata: oldDocument.metadata, + ...oldDocument, relativePath: newRelativePath, - isDeleted: oldDocument.isDeleted, - updates: oldDocument.updates, // We're in a strange state where the target of the move has just got deleted, // however, its metadata might already have a bunch of updates queued up for // the document at the new location. We need to keep these updates. @@ -295,8 +320,9 @@ export class Database { public delete(relativePath: RelativePath): void { const candidate = this.getLatestDocumentByRelativePath(relativePath); if (candidate === undefined) { - // it's fine because the document to be deleted might not have been created yet - return; + throw new Error( + `Document not found by relative path: ${relativePath}` + ); } candidate.isDeleted = true; } @@ -319,16 +345,12 @@ export class Database { private ensureConsistency(): void { const idToPath = new Map(); - this.resolvedDocuments - .filter(({ metadata }) => metadata !== undefined) - .forEach(({ metadata, relativePath }) => { - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - idToPath.set(metadata!.documentId, [ - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - ...(idToPath.get(metadata!.documentId) ?? []), - relativePath - ]); - }); + this.resolvedDocuments.forEach(({ relativePath, documentId }) => { + idToPath.set(documentId, [ + ...(idToPath.get(documentId) ?? []), + relativePath + ]); + }); const duplicates = Array.from(idToPath.entries()) .filter(([_, paths]) => paths.length > 1) diff --git a/frontend/sync-client/src/services/connected-state.ts b/frontend/sync-client/src/services/connected-state.ts new file mode 100644 index 00000000..da522816 --- /dev/null +++ b/frontend/sync-client/src/services/connected-state.ts @@ -0,0 +1,52 @@ +import { Syncer } from "../sync-operations/syncer"; +import { Settings } from "../persistence/settings"; +import { Logger } from "../tracing/logger"; +import { createPromise } from "../utils/create-promise"; +import { retriedFetchFactory } from "../utils/retried-fetch"; + +export class ConnectedState { + private resolveIsSyncEnabled: (() => void) | undefined; + private syncIsEnabled: Promise | undefined; + + public constructor( + settings: Settings, + private readonly logger: Logger + ) { + settings.addOnSettingsChangeHandlers((newSettings, oldSettings) => { + if (!oldSettings.isSyncEnabled && newSettings.isSyncEnabled) { + this.handleComingOnline(); + } else if ( + oldSettings.isSyncEnabled && + !newSettings.isSyncEnabled + ) { + this.handleGoingOffline(); + } + }); + } + + private handleComingOnline() { + this.logger.debug("Sync is enabled"); + this.resolveIsSyncEnabled?.(); + } + + private handleGoingOffline() { + this.logger.debug("Sync is disabled"); + [this.syncIsEnabled, this.resolveIsSyncEnabled] = createPromise(); + } + + public getFetchImplementation( + fetch: typeof globalThis.fetch, + { doRetries = true }: { doRetries: boolean } = { doRetries: true } + ): typeof globalThis.fetch { + const retriedFetch = doRetries + ? retriedFetchFactory(this.logger, fetch) + : fetch; + + return async (input: RequestInfo | URL): Promise => { + if (this.syncIsEnabled !== undefined) { + await this.syncIsEnabled; + } + return retriedFetch(input); + }; + } +} diff --git a/frontend/sync-client/src/services/sync-service.ts b/frontend/sync-client/src/services/sync-service.ts index ff7ec8fd..d784aa0d 100644 --- a/frontend/sync-client/src/services/sync-service.ts +++ b/frontend/sync-client/src/services/sync-service.ts @@ -6,20 +6,22 @@ import type { RelativePath, VaultUpdateId } from "../persistence/database"; -import type { Logger } from "src/tracing/logger"; -import { retriedFetchFactory } from "src/utils/retried-fetch"; -import type { Settings } from "src/persistence/settings"; +import type { Logger } from "../tracing/logger"; +import type { Settings } from "../persistence/settings"; +import { ConnectedState } from "./connected-state"; export interface CheckConnectionResult { isSuccessful: boolean; message: string; } + export class SyncService { private client!: Client; private clientWithoutRetries!: Client; private _fetchImplementation: typeof globalThis.fetch = globalThis.fetch; public constructor( + private readonly connectedState: ConnectedState, private readonly settings: Settings, private readonly logger: Logger ) { @@ -52,17 +54,19 @@ export class SyncService { } public async create({ + documentId, relativePath, - contentBytes, - createdDate + contentBytes }: { + documentId?: DocumentId; relativePath: RelativePath; contentBytes: Uint8Array; - createdDate: Date; }): Promise { const formData = new FormData(); + if (documentId !== undefined) { + formData.append("document_id", documentId); + } formData.append("relative_path", relativePath); - formData.append("created_date", createdDate.toISOString()); formData.append("content", new Blob([contentBytes])); const response = await this.client.POST( @@ -100,21 +104,18 @@ export class SyncService { parentVersionId, documentId, relativePath, - contentBytes, - createdDate + contentBytes }: { parentVersionId: VaultUpdateId; documentId: DocumentId; relativePath: RelativePath; contentBytes: Uint8Array; - createdDate: Date; }): Promise { this.logger.debug( `Updating document ${documentId} with parent version ${parentVersionId} & ${new TextDecoder().decode(contentBytes)} & ${relativePath}` ); const formData = new FormData(); formData.append("parent_version_id", parentVersionId.toString()); - formData.append("created_date", createdDate.toISOString()); formData.append("relative_path", relativePath); formData.append("content", new Blob([contentBytes])); @@ -152,12 +153,10 @@ export class SyncService { public async delete({ documentId, - relativePath, - createdDate + relativePath }: { documentId: DocumentId; relativePath: RelativePath; - createdDate: Date; }): Promise { const response = await this.client.DELETE( "/vaults/{vault_id}/documents/{document_id}", @@ -172,7 +171,6 @@ export class SyncService { } }, body: { - createdDate: createdDate.toISOString(), relativePath } } @@ -298,11 +296,17 @@ export class SyncService { private createClient(remoteUri: string): void { this.client = createClient({ baseUrl: remoteUri, - fetch: retriedFetchFactory(this.logger, this._fetchImplementation) + fetch: this.connectedState.getFetchImplementation( + this._fetchImplementation + ) }); this.clientWithoutRetries = createClient({ - baseUrl: remoteUri + baseUrl: remoteUri, + fetch: this.connectedState.getFetchImplementation( + this._fetchImplementation, + { doRetries: false } + ) }); } } diff --git a/frontend/sync-client/src/sync-client.ts b/frontend/sync-client/src/sync-client.ts index b4d6118a..dfd366ca 100644 --- a/frontend/sync-client/src/sync-client.ts +++ b/frontend/sync-client/src/sync-client.ts @@ -12,6 +12,7 @@ import { SyncService } from "./services/sync-service"; import { Syncer } from "./sync-operations/syncer"; import type { FileSystemOperations } from "./file-operations/filesystem-operations"; import { FileOperations } from "./file-operations/file-operations"; +import { ConnectedState } from "./services/connected-state"; export class SyncClient { private remoteListenerIntervalId: NodeJS.Timeout | null = null; @@ -90,7 +91,9 @@ export class SyncClient { } ); - const syncService = new SyncService(settings, logger); + const connectedState = new ConnectedState(settings, logger); + + const syncService = new SyncService(connectedState, settings, logger); const syncer = new Syncer( logger, @@ -117,18 +120,13 @@ export class SyncClient { ); settings.addOnSettingsChangeHandlers((newSettings, oldSettings) => { - client.registerRemoteEventListener( - newSettings.fetchChangesUpdateIntervalMs - ); - - if (!oldSettings.isSyncEnabled && newSettings.isSyncEnabled) { - syncer - .scheduleSyncForOfflineChanges() - .catch((_error: unknown) => { - logger.error( - "Failed to schedule sync for offline changes" - ); - }); + if ( + newSettings.fetchChangesUpdateIntervalMs !== + oldSettings.fetchChangesUpdateIntervalMs + ) { + client.registerRemoteEventListener( + newSettings.fetchChangesUpdateIntervalMs + ); } }); diff --git a/frontend/sync-client/src/sync-operations/syncer.ts b/frontend/sync-client/src/sync-operations/syncer.ts index 531e735a..7cb31aac 100644 --- a/frontend/sync-client/src/sync-operations/syncer.ts +++ b/frontend/sync-client/src/sync-operations/syncer.ts @@ -3,6 +3,7 @@ import type { SyncService } from "src/services/sync-service"; import type { Logger } from "src/tracing/logger"; import type { SyncHistory } from "src/tracing/sync-history"; import PQueue from "p-queue"; +import { v4 as uuidv4 } from "uuid"; import { hash } from "src/utils/hash"; import type { components } from "src/services/types"; import type { Settings } from "src/persistence/settings"; @@ -27,7 +28,7 @@ export class Syncer { public constructor( private readonly logger: Logger, private readonly database: Database, - private readonly settings: Settings, + settings: Settings, private readonly syncService: SyncService, private readonly operations: FileOperations, history: SyncHistory @@ -43,9 +44,11 @@ export class Syncer { this.syncQueue.concurrency = newSettings.syncConcurrency; }); - this.syncQueue.on("active", () => { - this.emitRemainingOperationsChange(this.syncQueue.size); - }); + this.syncQueue.on("active", () => + this.remainingOperationsListeners.forEach((listener) => + listener(this.syncQueue.size) + ) + ); this.internalSyncer = new UnrestrictedSyncer( logger, @@ -82,29 +85,32 @@ export class Syncer { } public async syncLocallyCreatedFile( - relativePath: RelativePath, - updateTime?: Date + relativePath: RelativePath ): Promise { - if (!this.settings.getSettings().isSyncEnabled) { - this.logger.info( - `Syncing is disabled, not syncing '${relativePath}'` + if ( + this.database.getLatestDocumentByRelativePath(relativePath) + ?.isDeleted === false + ) { + this.logger.debug( + `Document ${relativePath} already exists in the database, skipping` ); return; } const [promise, resolve, reject] = createPromise(); + const proposedDocumentId = uuidv4(); - // Most likely, we're waiting for the previous delete to finish on the file at this path - await this.database.getResolvedDocumentByRelativePath( + this.database.getNewResolvedDocumentByRelativePath( + proposedDocumentId, relativePath, promise ); try { - await this.syncQueue.add(async () => + await this.syncQueue.add(() => this.internalSyncer.unrestrictedSyncLocallyCreatedFile( - () => this.database.getDocumentByUpdatePromise(promise), - updateTime + proposedDocumentId, + () => this.database.getDocumentByUpdatePromise(promise) ) ); @@ -119,13 +125,8 @@ export class Syncer { public async syncLocallyDeletedFile( relativePath: RelativePath ): Promise { - if (!this.settings.getSettings().isSyncEnabled) { - this.logger.info( - `Syncing is disabled, not syncing '${relativePath}'` - ); - return; - } - + // We have to have a record of the delete in case there's an in-flight update for the same + // document which finishes after the delete has succeeded and would introduce a phantom metadata record. this.database.delete(relativePath); const [promise, resolve, reject] = createPromise(); @@ -137,13 +138,9 @@ export class Syncer { try { await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncLocallyDeletedFile(() => { - this.logger.debug( - `aaaahg ${relativePath} has been deleted locally, syncing to delete it` - ); - - return this.database.getDocumentByUpdatePromise(promise); - }) + this.internalSyncer.unrestrictedSyncLocallyDeletedFile(() => + this.database.getDocumentByUpdatePromise(promise) + ) ); resolve(); @@ -156,23 +153,22 @@ export class Syncer { public async syncLocallyUpdatedFile({ oldPath, - relativePath, - updateTime + relativePath }: { oldPath?: RelativePath; relativePath: RelativePath; - updateTime?: Date; }): Promise { - if (!this.settings.getSettings().isSyncEnabled) { - this.logger.info( - `Syncing is disabled, not syncing '${relativePath}'` - ); - return; - } - - const [promise, resolve, reject] = createPromise(); - if (oldPath !== undefined) { + if ( + this.database.getLatestDocumentByRelativePath(oldPath) + ?.isDeleted === true + ) { + this.logger.debug( + `Document ${oldPath} has been deleted locally, skipping` + ); + return; + } + if (oldPath === relativePath) { throw new Error( `Old path and new path are the same: ${oldPath}` @@ -182,6 +178,18 @@ export class Syncer { this.database.move(oldPath, relativePath); } + if ( + this.database.getLatestDocumentByRelativePath(relativePath) + ?.isDeleted === true + ) { + this.logger.debug( + `Document ${relativePath} has been deleted locally, skipping` + ); + return; + } + + const [promise, resolve, reject] = createPromise(); + await this.database.getResolvedDocumentByRelativePath( relativePath, promise @@ -191,7 +199,6 @@ export class Syncer { await this.syncQueue.add(async () => this.internalSyncer.unrestrictedSyncLocallyUpdatedFile({ oldPath, - updateTime, getLatestDocument: () => this.database.getDocumentByUpdatePromise(promise) }) @@ -206,13 +213,6 @@ export class Syncer { } public async scheduleSyncForOfflineChanges(): Promise { - if (!this.settings.getSettings().isSyncEnabled) { - this.logger.debug( - `Syncing is disabled, not uploading local changes` - ); - return; - } - if (this.runningScheduleSyncForOfflineChanges !== undefined) { this.logger.debug("Uploading local changes is already in progress"); return this.runningScheduleSyncForOfflineChanges; @@ -234,13 +234,6 @@ export class Syncer { } public async applyRemoteChangesLocally(): Promise { - if (!this.settings.getSettings().isSyncEnabled) { - this.logger.debug( - `Syncing is disabled, not fetching remote changes` - ); - return; - } - if (this.runningApplyRemoteChangesLocally != null) { this.logger.debug( "Applying remote changes locally is already in progress" @@ -272,6 +265,35 @@ export class Syncer { this.internalSyncer.reset(); } + private async internalApplyRemoteChangesLocally(): Promise { + const remote = await this.syncQueue.add(async () => + this.syncService.getAll(this.database.getLastSeenUpdateId()) + ); + + if (!remote) { + throw new Error("Failed to fetch remote changes"); + } + + if (remote.latestDocuments.length === 0) { + this.logger.debug("No remote changes to apply"); + return; + } + + this.logger.info("Applying remote changes locally"); + + await Promise.all( + remote.latestDocuments.map(this.syncRemotelyUpdatedFile.bind(this)) + ); + + const lastSeenUpdateId = this.database.getLastSeenUpdateId(); + if ( + lastSeenUpdateId === undefined || + remote.lastUpdateId > lastSeenUpdateId + ) { + this.database.setLastSeenUpdateId(remote.lastUpdateId); + } + } + private async syncRemotelyUpdatedFile( remoteVersion: components["schemas"]["DocumentVersionWithoutContent"] ): Promise { @@ -279,45 +301,38 @@ export class Syncer { remoteVersion.documentId ); - if (document === undefined) { - const candidate = this.database.getLatestDocumentByRelativePath( - remoteVersion.relativePath - ); - if (candidate !== undefined && candidate.metadata === undefined) { - document = candidate; - } - } - - if (document === undefined) { - await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile( - remoteVersion - ) - ); - - return; - } - const [promise, resolve, reject] = createPromise(); - await this.database.getResolvedDocumentByRelativePath( - document.relativePath, - promise - ); - - try { + if (document === undefined) { await this.syncQueue.add(async () => this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile( remoteVersion, - () => this.database.getDocumentByUpdatePromise(promise) + () => + this.database.getDocumentByDocumentId( + remoteVersion.documentId + ) ) ); + } else { + await this.database.getResolvedDocumentByRelativePath( + document.relativePath, + promise + ); - resolve(); - } catch (e) { - reject(e); - } finally { - this.database.removeDocumentPromise(promise); + try { + await this.syncQueue.add(async () => + this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile( + remoteVersion, + () => this.database.getDocumentByUpdatePromise(promise) + ) + ); + + resolve(); + } catch (e) { + reject(e); + } finally { + this.database.removeDocumentPromise(promise); + } } } @@ -405,35 +420,4 @@ export class Syncer { await Promise.all([updates, deletes]); } - - private async internalApplyRemoteChangesLocally(): Promise { - const remote = await this.syncService.getAll( - this.database.getLastSeenUpdateId() - ); - - if (remote.latestDocuments.length === 0) { - this.logger.debug("No remote changes to apply"); - return; - } - - this.logger.info("Applying remote changes locally"); - - await Promise.all( - remote.latestDocuments.map(this.syncRemotelyUpdatedFile.bind(this)) - ); - - const lastSeenUpdateId = this.database.getLastSeenUpdateId(); - if ( - lastSeenUpdateId === undefined || - remote.lastUpdateId > lastSeenUpdateId - ) { - this.database.setLastSeenUpdateId(remote.lastUpdateId); - } - } - - private emitRemainingOperationsChange(remainingOperations: number): void { - this.remainingOperationsListeners.forEach((listener) => { - listener(remainingOperations); - }); - } } diff --git a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts index 5e4cf754..84b4f070 100644 --- a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts +++ b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts @@ -1,5 +1,6 @@ import type { Database, + DocumentId, DocumentRecord, RelativePath } from "../persistence/database"; @@ -15,6 +16,7 @@ import type { Settings } from "src/persistence/settings"; import type { FileOperations } from "src/file-operations/file-operations"; import { FileNotFoundError } from "src/file-operations/safe-filesystem-operations"; import { DocumentLocks } from "../file-operations/document-locks"; +import { createPromise } from "src/utils/create-promise"; export class UnrestrictedSyncer { private readonly locks: DocumentLocks; @@ -31,8 +33,8 @@ export class UnrestrictedSyncer { } public async unrestrictedSyncLocallyCreatedFile( - getLatestDocument: () => DocumentRecord, - updateTime?: Date + proposedDocumentId: DocumentId, + getLatestDocument: () => DocumentRecord ): Promise { let latestDocument = getLatestDocument(); @@ -41,29 +43,15 @@ export class UnrestrictedSyncer { SyncType.CREATE, SyncSource.PUSH, async () => { - if ( - latestDocument.metadata !== undefined && - !latestDocument.isDeleted - ) { - this.logger.debug( - `Document ${latestDocument.relativePath} already exists in the database, no need to create it again` - ); - return; - } - const contentBytes = await this.operations.read( latestDocument.relativePath ); // this can throw FileNotFoundError const contentHash = hash(contentBytes); - updateTime ??= await this.operations.getModificationTime( - latestDocument.relativePath - ); // this can throw FileNotFoundError - const response = await this.syncService.create({ + documentId: proposedDocumentId, relativePath: latestDocument.relativePath, - contentBytes, - createdDate: updateTime + contentBytes }); latestDocument = getLatestDocument(); @@ -76,15 +64,15 @@ export class UnrestrictedSyncer { type: SyncType.CREATE }); - const newMetadata = { - relativePath: latestDocument.relativePath, - documentId: response.documentId, - parentVersionId: response.vaultUpdateId, - hash: contentHash, - isDeleted: false - }; - - this.database.setDocument(newMetadata, latestDocument.identity); + this.database.setDocument( + { + relativePath: latestDocument.relativePath, + documentId: response.documentId, + parentVersionId: response.vaultUpdateId, + hash: contentHash + }, + latestDocument.identity + ); this.tryIncrementVaultUpdateId(response.vaultUpdateId); } @@ -100,17 +88,9 @@ export class UnrestrictedSyncer { SyncType.DELETE, SyncSource.PUSH, async () => { - if (document.metadata === undefined) { - this.logger.debug( - `Document '${document.relativePath}' has been created yet so deleting it remotely can be skipped` - ); - return; - } - const response = await this.syncService.delete({ - documentId: document.metadata.documentId, - relativePath: document.relativePath, - createdDate: new Date() // We've got the event now, so it must have been deleted just now + documentId: document.documentId, + relativePath: document.relativePath }); this.history.addHistoryEntry({ @@ -123,8 +103,6 @@ export class UnrestrictedSyncer { document = getLatestDocument(); - // We have to have a record of the delete in case there's an in-flight update for the same - // document which finishes after the delete has succeeded and would introduce a phantom metadata record. this.database.setDocument( { relativePath: document.relativePath, @@ -140,12 +118,10 @@ export class UnrestrictedSyncer { public async unrestrictedSyncLocallyUpdatedFile({ oldPath, - getLatestDocument, - updateTime + getLatestDocument }: { oldPath?: RelativePath; getLatestDocument: () => DocumentRecord; - updateTime?: Date; }): Promise { let document = getLatestDocument(); @@ -178,19 +154,13 @@ export class UnrestrictedSyncer { return; } - updateTime ??= await this.operations.getModificationTime( - document.relativePath - ); // this can throw FileNotFoundError; - const response = await this.syncService.put({ - documentId: document.metadata.documentId, + documentId: document.documentId, parentVersionId: document.metadata.parentVersionId, relativePath: document.relativePath, - contentBytes, - createdDate: updateTime + contentBytes }); - // Update relativePath which is the only property that can change while this is running (due to a move) document = getLatestDocument(); if (document.isDeleted) { @@ -252,6 +222,11 @@ export class UnrestrictedSyncer { } if (response.relativePath != document.relativePath) { + // this.database.getNewResolvedDocumentByRelativePath( + // response.relativePath, + // promise + // ); + await this.operations.move( document.relativePath, response.relativePath, @@ -283,10 +258,7 @@ export class UnrestrictedSyncer { this.database.setDocument( { documentId: response.documentId, - relativePath: - response.relativePath != document.relativePath - ? response.relativePath - : document.relativePath, + relativePath: document.relativePath, parentVersionId: response.vaultUpdateId, hash: contentHash }, @@ -300,20 +272,19 @@ export class UnrestrictedSyncer { public async unrestrictedSyncRemotelyUpdatedFile( remoteVersion: components["schemas"]["DocumentVersionWithoutContent"], - getLatestDocument?: () => DocumentRecord + getLatestDocument: () => DocumentRecord | undefined ): Promise { await this.executeSync( [remoteVersion.relativePath], SyncType.UPDATE, SyncSource.PULL, async () => { - const localMetadata = - getLatestDocument?.() ?? - this.database.getDocumentByDocumentId( - remoteVersion.documentId - ); + let localMetadata = getLatestDocument(); - if (localMetadata?.metadata !== undefined) { + if ( + localMetadata !== undefined && + localMetadata?.metadata !== undefined + ) { // If the file exists locally, let's pretend the user has updated it // and deal with remote update/deletion within `unrestrictedSyncLocallyUpdatedFile` if ( @@ -329,7 +300,7 @@ export class UnrestrictedSyncer { return this.unrestrictedSyncLocallyUpdatedFile({ getLatestDocument: () => this.database.getDocumentByIdentity( - localMetadata.identity + localMetadata!.identity ) }); } else if (remoteVersion.isDeleted) { @@ -347,27 +318,26 @@ export class UnrestrictedSyncer { }) ).contentBase64; - const latestDocument = - getLatestDocument?.() ?? - this.database.getDocumentByDocumentId( - remoteVersion.documentId - ); + localMetadata = getLatestDocument(); - if (latestDocument?.isDeleted) { + if (localMetadata?.isDeleted === true) { this.logger.info( `Document ${remoteVersion.relativePath} has been deleted locally before we could finish updating it` ); return; } + if ( + localMetadata?.metadata?.parentVersionId ?? + -1 >= remoteVersion.vaultUpdateId + ) { + this.logger.debug( + `Document ${remoteVersion.relativePath} is already more up to date than the fetched version` + ); + return; + } const contentBytes = deserialize(content); - await this.operations.create( - remoteVersion.relativePath, - contentBytes, - remoteVersion.documentId - ); - this.database.setDocument( { documentId: remoteVersion.documentId, @@ -375,7 +345,13 @@ export class UnrestrictedSyncer { parentVersionId: remoteVersion.vaultUpdateId, hash: hash(contentBytes) }, - latestDocument?.identity + localMetadata?.identity + ); + + await this.operations.create( + remoteVersion.relativePath, + contentBytes, + remoteVersion.documentId ); this.history.addHistoryEntry({ @@ -390,19 +366,12 @@ export class UnrestrictedSyncer { } public async executeSync( - lockedPaths: RelativePath[], + paths: RelativePath[], syncType: SyncType, syncSource: SyncSource, fn: () => Promise ): Promise { - const relativePath = lockedPaths[lockedPaths.length - 1]; - - if (!this.settings.getSettings().isSyncEnabled) { - this.logger.info( - `Syncing is disabled, not syncing '${relativePath}'` - ); - return; - } + const relativePath = paths[paths.length - 1]; if (!this.operations.isFileEligibleForSync(relativePath)) { this.history.addHistoryEntry({ diff --git a/frontend/test-client/src/agent/mock-agent.ts b/frontend/test-client/src/agent/mock-agent.ts index 06c0f10e..94c106da 100644 --- a/frontend/test-client/src/agent/mock-agent.ts +++ b/frontend/test-client/src/agent/mock-agent.ts @@ -83,8 +83,6 @@ export class MockAgent extends MockClient { } public async act(): Promise { - this.assertAllContentIsPresentOnce(); - const options: (() => Promise)[] = [ this.createFileAction.bind(this), this.changeFetchChangesUpdateIntervalMsAction.bind(this) diff --git a/frontend/test-client/src/agent/mock-client.ts b/frontend/test-client/src/agent/mock-client.ts index ec077778..814d3fc1 100644 --- a/frontend/test-client/src/agent/mock-client.ts +++ b/frontend/test-client/src/agent/mock-client.ts @@ -1,3 +1,4 @@ +import { assert } from "../utils/assert"; import type { RelativePath, FileSystemOperations, @@ -81,6 +82,18 @@ export class MockClient implements FileSystemOperations { const newContentUint8Array = new TextEncoder().encode(newContent); this.localFiles.set(path, newContentUint8Array); + const existingPats = currentContent + .split(" ") + .map((part) => part.trim()); + const newParts = newContent.split(" ").map((part) => part.trim()); + existingPats.forEach((part) => + // all changes should be additive + assert( + newParts.includes(part), + `Part ${part} not found in new content` + ) + ); + this.client.logger.info( `Updated file ${path} with:\n current content: ${currentContent}\n new content: ${newContent}` ); diff --git a/frontend/test-client/src/cli.ts b/frontend/test-client/src/cli.ts index 27c59f75..e1a6fd5b 100644 --- a/frontend/test-client/src/cli.ts +++ b/frontend/test-client/src/cli.ts @@ -92,16 +92,16 @@ async function runTest({ async function runTests(): Promise { const agentCounts = [2, 8]; const jitterScaleInSeconds = [0.5, 0, 2]; - const concurrencies = [1]; + const concurrencies = [16, 1]; const iterations = [50, 200]; const doDeletes = [true, false]; - for (let i = 0; i < 10; i++) { - for (const agentCount of agentCounts) { - for (const concurrency of concurrencies) { - for (const jitter of jitterScaleInSeconds) { - for (const iteration of iterations) { - for (const deleteFiles of doDeletes) { + for (const agentCount of agentCounts) { + for (const concurrency of concurrencies) { + for (const jitter of jitterScaleInSeconds) { + for (const iteration of iterations) { + for (const deleteFiles of doDeletes) { + for (let i = 0; i < 10; i++) { await runTest({ agentCount, concurrency, @@ -110,6 +110,7 @@ async function runTests(): Promise { jitterScaleInSeconds: jitter }); } + return; } } } From d799a1da0c8a427836b17b51f6626fbe7141a052 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 15 Mar 2025 10:48:17 +0000 Subject: [PATCH 18/58] Use string uuids --- backend/sync_server/src/database.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/backend/sync_server/src/database.rs b/backend/sync_server/src/database.rs index 89fa8229..e31fc02d 100644 --- a/backend/sync_server/src/database.rs +++ b/backend/sync_server/src/database.rs @@ -7,6 +7,7 @@ use models::{ use sqlx::{sqlite::SqliteConnectOptions, types::chrono::Utc}; pub mod models; use sqlx::{Pool, Sqlite, sqlite::SqlitePoolOptions}; +use uuid::fmt::Hyphenated; use crate::config::database_config::DatabaseConfig; @@ -82,7 +83,7 @@ impl Database { select vault_id, vault_update_id, - document_id as "document_id: uuid::Uuid", + document_id as "document_id: Hyphenated", relative_path, updated_date as "updated_date: chrono::DateTime", is_deleted @@ -115,7 +116,7 @@ impl Database { select vault_id, vault_update_id, - document_id as "document_id: uuid::Uuid", + document_id as "document_id: Hyphenated", relative_path, updated_date as "updated_date: chrono::DateTime", is_deleted @@ -172,7 +173,7 @@ impl Database { select vault_id, vault_update_id, - document_id as "document_id: uuid::Uuid", + document_id as "document_id: Hyphenated", relative_path, updated_date as "updated_date: chrono::DateTime", content, @@ -202,13 +203,14 @@ impl Database { document_id: &DocumentId, transaction: Option<&mut Transaction<'_>>, ) -> Result> { + let document_id = document_id.as_hyphenated(); let query = sqlx::query_as!( StoredDocumentVersion, r#" select vault_id, vault_update_id, - document_id as "document_id: uuid::Uuid", + document_id as "document_id: Hyphenated", relative_path, updated_date as "updated_date: chrono::DateTime", content, @@ -240,7 +242,7 @@ impl Database { select vault_id, vault_update_id, - document_id as "document_id: uuid::Uuid", + document_id as "document_id: Hyphenated", relative_path, updated_date as "updated_date: chrono::DateTime", content, @@ -264,6 +266,7 @@ impl Database { version: &StoredDocumentVersion, transaction: Option<&mut Transaction<'_>>, ) -> Result<()> { + let document_id = version.document_id.as_hyphenated(); let query = sqlx::query!( r#" insert into documents ( @@ -279,7 +282,7 @@ impl Database { "#, version.vault_id, version.vault_update_id, - version.document_id, + document_id, version.relative_path, version.updated_date, version.content, From 24a8def394252a59db07e28978e39093ee713255 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 15 Mar 2025 11:44:56 +0000 Subject: [PATCH 19/58] . --- frontend/sync-client/src/persistence/database.ts | 5 +++++ frontend/sync-client/src/services/connected-state.ts | 1 - frontend/sync-client/src/utils/hash.ts | 2 +- frontend/test-client/src/cli.ts | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/frontend/sync-client/src/persistence/database.ts b/frontend/sync-client/src/persistence/database.ts index 0987c0de..2bcd77bc 100644 --- a/frontend/sync-client/src/persistence/database.ts +++ b/frontend/sync-client/src/persistence/database.ts @@ -301,6 +301,11 @@ export class Database { ); let newDocument = this.getLatestDocumentByRelativePath(newRelativePath); + if (newDocument !== undefined && !newDocument.isDeleted) { + throw new Error( + `Document already exists at new location: ${newRelativePath}` + ); + } // It's either an invalid state of newDocument is pending deletion and we have // to wait for it to complete. diff --git a/frontend/sync-client/src/services/connected-state.ts b/frontend/sync-client/src/services/connected-state.ts index da522816..e21ad005 100644 --- a/frontend/sync-client/src/services/connected-state.ts +++ b/frontend/sync-client/src/services/connected-state.ts @@ -1,4 +1,3 @@ -import { Syncer } from "../sync-operations/syncer"; import { Settings } from "../persistence/settings"; import { Logger } from "../tracing/logger"; import { createPromise } from "../utils/create-promise"; diff --git a/frontend/sync-client/src/utils/hash.ts b/frontend/sync-client/src/utils/hash.ts index 10f20d1d..cd965db5 100644 --- a/frontend/sync-client/src/utils/hash.ts +++ b/frontend/sync-client/src/utils/hash.ts @@ -6,7 +6,7 @@ export function hash(content: Uint8Array): string { result = (result << 5) - result + content[i]; result |= 0; // Convert to 32bit integer } - return Math.abs(result).toString(16); + return Math.abs(result).toString(16).padStart(8, "0"); } export const EMPTY_HASH = hash(new Uint8Array(0)); diff --git a/frontend/test-client/src/cli.ts b/frontend/test-client/src/cli.ts index e1a6fd5b..8e6e711b 100644 --- a/frontend/test-client/src/cli.ts +++ b/frontend/test-client/src/cli.ts @@ -92,7 +92,7 @@ async function runTest({ async function runTests(): Promise { const agentCounts = [2, 8]; const jitterScaleInSeconds = [0.5, 0, 2]; - const concurrencies = [16, 1]; + const concurrencies = [1]; const iterations = [50, 200]; const doDeletes = [true, false]; From e6766fff42e60e11641c487ea259e720f49569f6 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 15 Mar 2025 12:11:25 +0000 Subject: [PATCH 20/58] working!!!! (hopefully) --- .../src/file-operations/file-operations.ts | 67 +++++-------------- .../sync-operations/unrestricted-syncer.ts | 67 ++++++++++++------- frontend/test-client/src/agent/mock-client.ts | 15 +++-- 3 files changed, 70 insertions(+), 79 deletions(-) diff --git a/frontend/sync-client/src/file-operations/file-operations.ts b/frontend/sync-client/src/file-operations/file-operations.ts index 8071a0f5..f74576d5 100644 --- a/frontend/sync-client/src/file-operations/file-operations.ts +++ b/frontend/sync-client/src/file-operations/file-operations.ts @@ -1,10 +1,6 @@ import type { Logger } from "src/tracing/logger"; import type { FileSystemOperations } from "./filesystem-operations"; -import type { - Database, - DocumentId, - RelativePath -} from "src/persistence/database"; +import type { Database, RelativePath } from "src/persistence/database"; import { isBinary, isFileTypeMergable, mergeText } from "sync_lib"; import { FileNotFoundError, @@ -53,12 +49,12 @@ export class FileOperations { return this.fs.exists(path); } - // Create and write the file if it doesn't exist.Otherwise, it has the same behavior as write. + // Create and write the file if it doesn't exist. Otherwise, it has the same behavior as write. // All parent directories are created if they don't exist. public async create( path: RelativePath, newContent: Uint8Array, - documentId?: DocumentId + whatevs?: any ): Promise { this.logger.debug(`Creating file: ${path}`); if (await this.fs.exists(path)) { @@ -67,25 +63,14 @@ export class FileOperations { `Didn't expect ${path} to exist, deconflicting by moving it to '${deconflictedPath}'` ); - const document = - this.database.getLatestDocumentByRelativePath(path); - this.logger.debug( - `Existing metadata for ${path}: ${JSON.stringify(document?.metadata)}` - ); - - if (document !== undefined && document.documentId === documentId) { - // This can happen if the document got moved both locally and remotely - // to the same file path. In this case, we shouldn't deconflict, however, - // we also can't overwrite otherwise we'd lose changes. - throw new FileNotFoundError(path); - } - - this.database.move(path, deconflictedPath); + // this.database.move(path, deconflictedPath); await this.fs.rename(path, deconflictedPath); } else { await this.createParentDirectories(path); } + whatevs?.(); + await this.fs.write(path, newContent); } @@ -152,8 +137,7 @@ export class FileOperations { public async move( oldPath: RelativePath, - newPath: RelativePath, - documentId?: DocumentId + newPath: RelativePath ): Promise { if (oldPath === newPath) { return; @@ -165,26 +149,14 @@ export class FileOperations { `Conflict when moving '${oldPath}' to '${newPath}', the latter already exists, deconflicting by moving it to '${deconflictedPath}'` ); - const document = - this.database.getLatestDocumentByRelativePath(newPath); - - if ( - document?.metadata !== undefined && - document.documentId === documentId - ) { - // This can happen if the document got moved both locally and remotely - // to the same file path. In this case, we shouldn't deconflict, however, - // we also can't overwrite otherwise we'd lose changes. - throw new FileNotFoundError(newPath); - } - - this.database.move(newPath, deconflictedPath); + // this.database.move(newPath, deconflictedPath); + // this.database.move(oldPath, newPath); await this.fs.rename(newPath, deconflictedPath); } else { + // this.database.move(oldPath, newPath); await this.createParentDirectories(newPath); } - this.database.move(oldPath, newPath); await this.fs.rename(oldPath, newPath); } @@ -226,17 +198,12 @@ export class FileOperations { ); stem = stem.replace(FileOperations.PARENTHESES_REGEX, ""); - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - while (true) { - const newName = - currentCount === 0 - ? `${directory}${stem}${extension}` - : `${directory}${stem} (${currentCount})${extension}`; - if (await this.fs.exists(newName)) { - currentCount++; - } else { - return newName; - } - } + let newName; + do { + currentCount++; + newName = `${directory}${stem} (${currentCount})${extension}`; + } while (await this.fs.exists(newName)); + + return newName; } } diff --git a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts index 84b4f070..80949615 100644 --- a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts +++ b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts @@ -36,42 +36,44 @@ export class UnrestrictedSyncer { proposedDocumentId: DocumentId, getLatestDocument: () => DocumentRecord ): Promise { - let latestDocument = getLatestDocument(); + let document = getLatestDocument(); return this.executeSync( - [latestDocument.relativePath], + [document.relativePath], SyncType.CREATE, SyncSource.PUSH, async () => { + document = getLatestDocument(); + const contentBytes = await this.operations.read( - latestDocument.relativePath + document.relativePath ); // this can throw FileNotFoundError const contentHash = hash(contentBytes); const response = await this.syncService.create({ documentId: proposedDocumentId, - relativePath: latestDocument.relativePath, + relativePath: document.relativePath, contentBytes }); - latestDocument = getLatestDocument(); + document = getLatestDocument(); this.history.addHistoryEntry({ status: SyncStatus.SUCCESS, source: SyncSource.PUSH, - relativePath: latestDocument.relativePath, + relativePath: document.relativePath, message: `Successfully uploaded locally created file`, type: SyncType.CREATE }); this.database.setDocument( { - relativePath: latestDocument.relativePath, + relativePath: document.relativePath, documentId: response.documentId, parentVersionId: response.vaultUpdateId, hash: contentHash }, - latestDocument.identity + document.identity ); this.tryIncrementVaultUpdateId(response.vaultUpdateId); @@ -88,6 +90,8 @@ export class UnrestrictedSyncer { SyncType.DELETE, SyncSource.PUSH, async () => { + document = getLatestDocument(); + const response = await this.syncService.delete({ documentId: document.documentId, relativePath: document.relativePath @@ -132,6 +136,9 @@ export class UnrestrictedSyncer { SyncType.UPDATE, SyncSource.PUSH, async () => { + document = getLatestDocument(); + const originalRelativePath = document.relativePath; + if (document.metadata === undefined || document.isDeleted) { this.logger.debug( `Document ${document.relativePath} has been already deleted, no need to update it` @@ -194,8 +201,6 @@ export class UnrestrictedSyncer { }); if (response.isDeleted) { - await this.operations.delete(document.relativePath); - this.history.addHistoryEntry({ status: SyncStatus.SUCCESS, source: SyncSource.PULL, @@ -216,21 +221,25 @@ export class UnrestrictedSyncer { document.identity ); + await this.operations.delete(document.relativePath); + this.tryIncrementVaultUpdateId(response.vaultUpdateId); return; } - if (response.relativePath != document.relativePath) { + let actualPath = document.relativePath; + + if (response.relativePath != originalRelativePath) { // this.database.getNewResolvedDocumentByRelativePath( // response.relativePath, // promise // ); + actualPath = response.relativePath; await this.operations.move( document.relativePath, - response.relativePath, - response.documentId + response.relativePath ); // this can throw FileNotFoundError } @@ -239,7 +248,7 @@ export class UnrestrictedSyncer { contentHash = hash(responseBytes); await this.operations.write( - response.relativePath, + actualPath, contentBytes, responseBytes ); @@ -253,12 +262,10 @@ export class UnrestrictedSyncer { }); } - document = getLatestDocument(); - this.database.setDocument( { documentId: response.documentId, - relativePath: document.relativePath, + relativePath: actualPath, parentVersionId: response.vaultUpdateId, hash: contentHash }, @@ -326,6 +333,7 @@ export class UnrestrictedSyncer { ); return; } + if ( localMetadata?.metadata?.parentVersionId ?? -1 >= remoteVersion.vaultUpdateId @@ -338,6 +346,21 @@ export class UnrestrictedSyncer { const contentBytes = deserialize(content); + const [promise, resolve] = createPromise(); + + await this.operations.create( + remoteVersion.relativePath, + contentBytes, + () => + this.database.getNewResolvedDocumentByRelativePath( + remoteVersion.documentId, + remoteVersion.relativePath, + promise + ) + ); + + const document = + this.database.getDocumentByUpdatePromise(promise); this.database.setDocument( { documentId: remoteVersion.documentId, @@ -345,14 +368,10 @@ export class UnrestrictedSyncer { parentVersionId: remoteVersion.vaultUpdateId, hash: hash(contentBytes) }, - localMetadata?.identity - ); - - await this.operations.create( - remoteVersion.relativePath, - contentBytes, - remoteVersion.documentId + document.identity ); + resolve(); + this.database.removeDocumentPromise(promise); this.history.addHistoryEntry({ status: SyncStatus.SUCCESS, diff --git a/frontend/test-client/src/agent/mock-client.ts b/frontend/test-client/src/agent/mock-client.ts index 814d3fc1..6cbcca06 100644 --- a/frontend/test-client/src/agent/mock-client.ts +++ b/frontend/test-client/src/agent/mock-client.ts @@ -82,11 +82,11 @@ export class MockClient implements FileSystemOperations { const newContentUint8Array = new TextEncoder().encode(newContent); this.localFiles.set(path, newContentUint8Array); - const existingPats = currentContent + const existingParts = currentContent .split(" ") .map((part) => part.trim()); const newParts = newContent.split(" ").map((part) => part.trim()); - existingPats.forEach((part) => + existingParts.forEach((part) => // all changes should be additive assert( newParts.includes(part), @@ -106,15 +106,20 @@ export class MockClient implements FileSystemOperations { } public async write(path: RelativePath, content: Uint8Array): Promise { + const hasExisted = this.localFiles.has(path); this.localFiles.set(path, content); this.client.logger.info( `Updated file ${path} with:\n new content: ${new TextDecoder().decode(content)}` ); - void this.client.syncer.syncLocallyUpdatedFile({ - relativePath: path - }); + if (hasExisted) { + void this.client.syncer.syncLocallyUpdatedFile({ + relativePath: path + }); + } else { + void this.client.syncer.syncLocallyCreatedFile(path); + } } public async delete(path: RelativePath): Promise { From b0814b1b97d9275b01a97d6fa946529811a22720 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 15 Mar 2025 14:11:05 +0000 Subject: [PATCH 21/58] Improve bundling --- frontend/sync-client/webpack.config.js | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/frontend/sync-client/webpack.config.js b/frontend/sync-client/webpack.config.js index cd2c051d..6d03f510 100644 --- a/frontend/sync-client/webpack.config.js +++ b/frontend/sync-client/webpack.config.js @@ -2,24 +2,11 @@ const path = require("path"); module.exports = (_env, _argv) => ({ entry: "./src/index.ts", - devtool: "source-map", - target: "node", module: { rules: [ { test: /\.ts$/, - use: [ - { - loader: "ts-loader", - options: { - compilerOptions: { - declaration: true, - declarationDir: "./dist/types" - }, - transpileOnly: false - } - } - ] + use: ["ts-loader"] }, { test: /\.wasm$/, @@ -28,18 +15,23 @@ module.exports = (_env, _argv) => ({ ] }, optimization: { + // the consuming project should take care of minification minimize: false }, resolve: { - extensions: [".ts", ".js"], + extensions: [".ts"], alias: { root: __dirname, src: path.resolve(__dirname, "src") } }, + performance: { + hints: false // it's a library, no need to warn about its size + }, output: { clean: true, filename: "index.js", + globalObject: "this", library: { name: "SyncClient", type: "umd" From b5260e97e9ad26da93378224c9d95d5753241d60 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 15 Mar 2025 14:11:24 +0000 Subject: [PATCH 22/58] Add module --- frontend/sync-client/package.json | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/frontend/sync-client/package.json b/frontend/sync-client/package.json index 1d3618ca..030d5c3d 100644 --- a/frontend/sync-client/package.json +++ b/frontend/sync-client/package.json @@ -4,26 +4,27 @@ "private": true, "main": "dist/index.js", "types": "dist/types/index.d.ts", + "module": "src/index.js", "scripts": { "dev": "webpack watch --mode development", "build": "webpack --mode production", "test": "NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules\" jest" }, "devDependencies": { + "@types/jest": "^29.5.14", + "@types/node": "^22.13.5", + "byte-base64": "^1.1.0", + "fetch-retry": "^6.0.0", + "jest": "^29.7.0", + "openapi-fetch": "0.13.4", + "openapi-typescript": "7.6.1", + "p-queue": "^8.1.0", + "sync_lib": "file:../../backend/sync_lib/pkg", + "ts-jest": "^29.2.6", + "ts-loader": "^9.5.2", "tslib": "2.8.1", "typescript": "5.7.3", "uuid": "^11.1.0", - "sync_lib": "file:../../backend/sync_lib/pkg", - "@types/jest": "^29.5.14", - "@types/node": "^22.13.5", - "jest": "^29.7.0", - "ts-jest": "^29.2.6", - "p-queue": "^8.1.0", - "fetch-retry": "^6.0.0", - "byte-base64": "^1.1.0", - "openapi-fetch": "0.13.4", - "openapi-typescript": "7.6.1", - "ts-loader": "^9.5.2", "webpack": "^5.98.0", "webpack-cli": "^6.0.1" } From 9ad54eff7ab16daf1f037ee16677e6ad0c761714 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 15 Mar 2025 14:12:04 +0000 Subject: [PATCH 23/58] lint --- .../src/obisidan-event-handler.ts | 11 ++----- .../obsidian-plugin/src/views/history-view.ts | 2 +- .../obsidian-plugin/src/views/logs-view.ts | 2 +- .../obsidian-plugin/src/views/settings-tab.ts | 2 +- .../obsidian-plugin/src/views/status-bar.ts | 2 +- .../src/file-operations/document-locks.ts | 3 ++ .../file-operations/file-operations.test.ts | 2 +- .../src/file-operations/file-operations.ts | 4 +-- .../file-operations/filesystem-operations.ts | 2 +- .../safe-filesystem-operations.ts | 4 +-- .../sync-client/src/persistence/database.ts | 6 ++-- .../sync-client/src/persistence/settings.ts | 4 +-- .../src/services/connected-state.ts | 4 +-- .../sync-client/src/services/sync-service.ts | 2 +- .../sync-client/src/sync-operations/syncer.ts | 32 +++++++++---------- .../sync-operations/unrestricted-syncer.ts | 27 ++++++++-------- .../sync-client/src/tracing/sync-history.ts | 2 +- .../sync-client/src/utils/retried-fetch.ts | 2 +- frontend/sync-client/tsconfig.json | 15 ++++++--- frontend/test-client/src/agent/mock-client.ts | 18 +++++------ frontend/test-client/src/cli.ts | 5 ++- 21 files changed, 76 insertions(+), 75 deletions(-) diff --git a/frontend/obsidian-plugin/src/obisidan-event-handler.ts b/frontend/obsidian-plugin/src/obisidan-event-handler.ts index b2d58b70..9fb5dba4 100644 --- a/frontend/obsidian-plugin/src/obisidan-event-handler.ts +++ b/frontend/obsidian-plugin/src/obisidan-event-handler.ts @@ -9,10 +9,7 @@ export class ObsidianFileEventHandler { if (file instanceof TFile) { this.client.logger.info(`File created: ${file.path}`); - await this.client.syncer.syncLocallyCreatedFile( - file.path, - new Date(file.stat.ctime) - ); + await this.client.syncer.syncLocallyCreatedFile(file.path); } else { this.client.logger.debug(`Folder created: ${file.path}, ignored`); } @@ -34,8 +31,7 @@ export class ObsidianFileEventHandler { await this.client.syncer.syncLocallyUpdatedFile({ oldPath, - relativePath: file.path, - updateTime: new Date(file.stat.ctime) + relativePath: file.path }); } else { this.client.logger.debug( @@ -53,8 +49,7 @@ export class ObsidianFileEventHandler { this.client.logger.info(`File modified: ${file.path}`); await this.client.syncer.syncLocallyUpdatedFile({ - relativePath: file.path, - updateTime: new Date(file.stat.ctime) + relativePath: file.path }); } else { this.client.logger.debug(`Folder modified: ${file.path}, ignored`); diff --git a/frontend/obsidian-plugin/src/views/history-view.ts b/frontend/obsidian-plugin/src/views/history-view.ts index 56c8d539..ba2e18bc 100644 --- a/frontend/obsidian-plugin/src/views/history-view.ts +++ b/frontend/obsidian-plugin/src/views/history-view.ts @@ -60,7 +60,7 @@ export class HistoryView extends ItemView { } element.createEl("span", { - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + text: entry.relativePath }); diff --git a/frontend/obsidian-plugin/src/views/logs-view.ts b/frontend/obsidian-plugin/src/views/logs-view.ts index 66aa30c2..22def86f 100644 --- a/frontend/obsidian-plugin/src/views/logs-view.ts +++ b/frontend/obsidian-plugin/src/views/logs-view.ts @@ -1,6 +1,6 @@ import type { WorkspaceLeaf } from "obsidian"; import { ItemView } from "obsidian"; -import type VaultLinkPlugin from "src/vault-link-plugin"; +import type VaultLinkPlugin from "../vault-link-plugin"; import type { SyncClient } from "sync-client"; export class LogsView extends ItemView { diff --git a/frontend/obsidian-plugin/src/views/settings-tab.ts b/frontend/obsidian-plugin/src/views/settings-tab.ts index b3a39dd7..d37a26dc 100644 --- a/frontend/obsidian-plugin/src/views/settings-tab.ts +++ b/frontend/obsidian-plugin/src/views/settings-tab.ts @@ -1,7 +1,7 @@ import type { App } from "obsidian"; import { Notice, PluginSettingTab, Setting } from "obsidian"; -import type VaultLinkPlugin from "src/vault-link-plugin"; +import type VaultLinkPlugin from "../vault-link-plugin"; import type { StatusDescription } from "./status-description"; import { LogsView } from "./logs-view"; import { HistoryView } from "./history-view"; diff --git a/frontend/obsidian-plugin/src/views/status-bar.ts b/frontend/obsidian-plugin/src/views/status-bar.ts index 9ecd7b87..ba7d7339 100644 --- a/frontend/obsidian-plugin/src/views/status-bar.ts +++ b/frontend/obsidian-plugin/src/views/status-bar.ts @@ -1,5 +1,5 @@ import type { HistoryStats, SyncClient } from "sync-client"; -import type VaultLinkPlugin from "src/vault-link-plugin"; +import type VaultLinkPlugin from "../vault-link-plugin"; export class StatusBar { private readonly statusBarItem: HTMLElement; diff --git a/frontend/sync-client/src/file-operations/document-locks.ts b/frontend/sync-client/src/file-operations/document-locks.ts index 3dc7ec29..522ed02a 100644 --- a/frontend/sync-client/src/file-operations/document-locks.ts +++ b/frontend/sync-client/src/file-operations/document-locks.ts @@ -1,6 +1,9 @@ import type { Logger } from "../tracing/logger"; import type { RelativePath } from "../persistence/database"; +// Manages locks on documents to prevent concurrent modifications +// allowing the client's FileOperations implementation to be simpler. +// Locks are granted in a first-in-first-out order. export class DocumentLocks { private readonly locked = new Set(); private readonly waiters = new Map void)[]>(); diff --git a/frontend/sync-client/src/file-operations/file-operations.test.ts b/frontend/sync-client/src/file-operations/file-operations.test.ts index 26ae3267..e0514ae8 100644 --- a/frontend/sync-client/src/file-operations/file-operations.test.ts +++ b/frontend/sync-client/src/file-operations/file-operations.test.ts @@ -1,4 +1,3 @@ -import type { FileSystemOperations } from "sync-client"; import type { Database, DocumentRecord, @@ -7,6 +6,7 @@ import type { import { FileOperations } from "./file-operations"; import { Logger } from "../tracing/logger"; import { assertSetContainsExactly } from "../utils/assert-set-contains-exactly"; +import type { FileSystemOperations } from "./filesystem-operations"; describe("File operations", () => { class MockDatabase { diff --git a/frontend/sync-client/src/file-operations/file-operations.ts b/frontend/sync-client/src/file-operations/file-operations.ts index f74576d5..74448efa 100644 --- a/frontend/sync-client/src/file-operations/file-operations.ts +++ b/frontend/sync-client/src/file-operations/file-operations.ts @@ -1,6 +1,6 @@ -import type { Logger } from "src/tracing/logger"; +import type { Logger } from "../tracing/logger"; import type { FileSystemOperations } from "./filesystem-operations"; -import type { Database, RelativePath } from "src/persistence/database"; +import type { Database, RelativePath } from "../persistence/database"; import { isBinary, isFileTypeMergable, mergeText } from "sync_lib"; import { FileNotFoundError, diff --git a/frontend/sync-client/src/file-operations/filesystem-operations.ts b/frontend/sync-client/src/file-operations/filesystem-operations.ts index 7c51ec78..3cab9d2d 100644 --- a/frontend/sync-client/src/file-operations/filesystem-operations.ts +++ b/frontend/sync-client/src/file-operations/filesystem-operations.ts @@ -1,4 +1,4 @@ -import type { RelativePath } from "src/persistence/database"; +import type { RelativePath } from "../persistence/database"; export interface FileSystemOperations { listAllFiles: () => Promise; diff --git a/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts b/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts index a2f7d111..c13611ef 100644 --- a/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts +++ b/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts @@ -12,7 +12,8 @@ export class FileNotFoundError extends Error { // Decorate FileSystemOperations replacing errors with FileNotFoundError // if the accessed file doesn't exist. It also ensures that there's only -// ever a single request in-flight for any one file. +// ever a single request in-flight for any one file through the use of +// DocumentLocks. export class SafeFileSystemOperations implements FileSystemOperations { private readonly locks: DocumentLocks; @@ -24,7 +25,6 @@ export class SafeFileSystemOperations implements FileSystemOperations { } public async listAllFiles(): Promise { - this.logger.debug("Listing all files"); return this.fs.listAllFiles(); } diff --git a/frontend/sync-client/src/persistence/database.ts b/frontend/sync-client/src/persistence/database.ts index 2bcd77bc..42b8be86 100644 --- a/frontend/sync-client/src/persistence/database.ts +++ b/frontend/sync-client/src/persistence/database.ts @@ -216,7 +216,7 @@ export class Database { relativePath: RelativePath, promise: Promise ): Promise { - let entry = this.getLatestDocumentByRelativePath(relativePath); + const entry = this.getLatestDocumentByRelativePath(relativePath); if (entry === undefined) { throw new Error( @@ -238,7 +238,7 @@ export class Database { relativePath: RelativePath, promise: Promise ): void { - let previousEntry = this.getLatestDocumentByRelativePath(relativePath); + const previousEntry = this.getLatestDocumentByRelativePath(relativePath); const entry = { relativePath, @@ -300,7 +300,7 @@ export class Database { ({ identity }) => identity !== oldDocument.identity ); - let newDocument = this.getLatestDocumentByRelativePath(newRelativePath); + const newDocument = this.getLatestDocumentByRelativePath(newRelativePath); if (newDocument !== undefined && !newDocument.isDeleted) { throw new Error( `Document already exists at new location: ${newRelativePath}` diff --git a/frontend/sync-client/src/persistence/settings.ts b/frontend/sync-client/src/persistence/settings.ts index 29a77ff7..dbeb8c15 100644 --- a/frontend/sync-client/src/persistence/settings.ts +++ b/frontend/sync-client/src/persistence/settings.ts @@ -1,5 +1,5 @@ -import type { Logger } from "src/tracing/logger"; -import { LogLevel } from "src/tracing/logger"; +import type { Logger } from "../tracing/logger"; +import { LogLevel } from "../tracing/logger"; export interface SyncSettings { remoteUri: string; diff --git a/frontend/sync-client/src/services/connected-state.ts b/frontend/sync-client/src/services/connected-state.ts index e21ad005..7e3a28e7 100644 --- a/frontend/sync-client/src/services/connected-state.ts +++ b/frontend/sync-client/src/services/connected-state.ts @@ -1,5 +1,5 @@ -import { Settings } from "../persistence/settings"; -import { Logger } from "../tracing/logger"; +import type { Settings } from "../persistence/settings"; +import type { Logger } from "../tracing/logger"; import { createPromise } from "../utils/create-promise"; import { retriedFetchFactory } from "../utils/retried-fetch"; diff --git a/frontend/sync-client/src/services/sync-service.ts b/frontend/sync-client/src/services/sync-service.ts index d784aa0d..92455643 100644 --- a/frontend/sync-client/src/services/sync-service.ts +++ b/frontend/sync-client/src/services/sync-service.ts @@ -8,7 +8,7 @@ import type { } from "../persistence/database"; import type { Logger } from "../tracing/logger"; import type { Settings } from "../persistence/settings"; -import { ConnectedState } from "./connected-state"; +import type { ConnectedState } from "./connected-state"; export interface CheckConnectionResult { isSuccessful: boolean; diff --git a/frontend/sync-client/src/sync-operations/syncer.ts b/frontend/sync-client/src/sync-operations/syncer.ts index 7cb31aac..d30f8457 100644 --- a/frontend/sync-client/src/sync-operations/syncer.ts +++ b/frontend/sync-client/src/sync-operations/syncer.ts @@ -1,17 +1,17 @@ import type { Database, RelativePath } from "../persistence/database"; -import type { SyncService } from "src/services/sync-service"; -import type { Logger } from "src/tracing/logger"; -import type { SyncHistory } from "src/tracing/sync-history"; +import type { SyncService } from "../services/sync-service"; +import type { Logger } from "../tracing/logger"; +import type { SyncHistory } from "../tracing/sync-history"; import PQueue from "p-queue"; import { v4 as uuidv4 } from "uuid"; -import { hash } from "src/utils/hash"; -import type { components } from "src/services/types"; -import type { Settings } from "src/persistence/settings"; -import type { FileOperations } from "src/file-operations/file-operations"; -import { findMatchingFile } from "src/utils/find-matching-file"; +import { hash } from "../utils/hash"; +import type { components } from "../services/types"; +import type { Settings } from "../persistence/settings"; +import type { FileOperations } from "../file-operations/file-operations"; +import { findMatchingFile } from "../utils/find-matching-file"; import { UnrestrictedSyncer } from "./unrestricted-syncer"; -import { FileNotFoundError } from "src/file-operations/safe-filesystem-operations"; -import { createPromise } from "src/utils/create-promise"; +import { FileNotFoundError } from "../file-operations/safe-filesystem-operations"; +import { createPromise } from "../utils/create-promise"; export class Syncer { private readonly remainingOperationsListeners: (( @@ -45,9 +45,9 @@ export class Syncer { }); this.syncQueue.on("active", () => - this.remainingOperationsListeners.forEach((listener) => - listener(this.syncQueue.size) - ) + { this.remainingOperationsListeners.forEach((listener) => + { listener(this.syncQueue.size); } + ); } ); this.internalSyncer = new UnrestrictedSyncer( @@ -107,7 +107,7 @@ export class Syncer { ); try { - await this.syncQueue.add(() => + await this.syncQueue.add(async () => this.internalSyncer.unrestrictedSyncLocallyCreatedFile( proposedDocumentId, () => this.database.getDocumentByUpdatePromise(promise) @@ -261,7 +261,7 @@ export class Syncer { public async reset(): Promise { this.syncQueue.clear(); await this.syncQueue.onEmpty(); - this.remainingOperationsListeners.forEach((listener) => listener(0)); + this.remainingOperationsListeners.forEach((listener) => { listener(0); }); this.internalSyncer.reset(); } @@ -297,7 +297,7 @@ export class Syncer { private async syncRemotelyUpdatedFile( remoteVersion: components["schemas"]["DocumentVersionWithoutContent"] ): Promise { - let document = this.database.getDocumentByDocumentId( + const document = this.database.getDocumentByDocumentId( remoteVersion.documentId ); diff --git a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts index 80949615..1734e25b 100644 --- a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts +++ b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts @@ -5,18 +5,18 @@ import type { RelativePath } from "../persistence/database"; -import type { SyncService } from "src/services/sync-service"; -import { Logger } from "src/tracing/logger"; -import type { SyncHistory } from "src/tracing/sync-history"; -import { SyncSource, SyncStatus, SyncType } from "src/tracing/sync-history"; -import { EMPTY_HASH, hash } from "src/utils/hash"; -import type { components } from "src/services/types"; -import { deserialize } from "src/utils/deserialize"; -import type { Settings } from "src/persistence/settings"; -import type { FileOperations } from "src/file-operations/file-operations"; -import { FileNotFoundError } from "src/file-operations/safe-filesystem-operations"; +import type { SyncService } from "../services/sync-service"; +import type { Logger } from "../tracing/logger"; +import type { SyncHistory } from "../tracing/sync-history"; +import { SyncSource, SyncStatus, SyncType } from "../tracing/sync-history"; +import { EMPTY_HASH, hash } from "../utils/hash"; +import type { components } from "../services/types"; +import { deserialize } from "../utils/deserialize"; +import type { Settings } from "../persistence/settings"; +import type { FileOperations } from "../file-operations/file-operations"; +import { FileNotFoundError } from "../file-operations/safe-filesystem-operations"; import { DocumentLocks } from "../file-operations/document-locks"; -import { createPromise } from "src/utils/create-promise"; +import { createPromise } from "../utils/create-promise"; export class UnrestrictedSyncer { private readonly locks: DocumentLocks; @@ -289,7 +289,6 @@ export class UnrestrictedSyncer { let localMetadata = getLatestDocument(); if ( - localMetadata !== undefined && localMetadata?.metadata !== undefined ) { // If the file exists locally, let's pretend the user has updated it @@ -352,11 +351,11 @@ export class UnrestrictedSyncer { remoteVersion.relativePath, contentBytes, () => - this.database.getNewResolvedDocumentByRelativePath( + { this.database.getNewResolvedDocumentByRelativePath( remoteVersion.documentId, remoteVersion.relativePath, promise - ) + ); } ); const document = diff --git a/frontend/sync-client/src/tracing/sync-history.ts b/frontend/sync-client/src/tracing/sync-history.ts index ea87bcac..ec8841e9 100644 --- a/frontend/sync-client/src/tracing/sync-history.ts +++ b/frontend/sync-client/src/tracing/sync-history.ts @@ -1,4 +1,4 @@ -import type { RelativePath } from "src/persistence/database"; +import type { RelativePath } from "../persistence/database"; import type { Logger } from "./logger"; export interface CommonHistoryEntry { diff --git a/frontend/sync-client/src/utils/retried-fetch.ts b/frontend/sync-client/src/utils/retried-fetch.ts index 1a49c704..a3856f8d 100644 --- a/frontend/sync-client/src/utils/retried-fetch.ts +++ b/frontend/sync-client/src/utils/retried-fetch.ts @@ -1,6 +1,6 @@ import * as fetchRetryFactory from "fetch-retry"; import type { RequestInitRetryParams } from "fetch-retry"; -import type { Logger } from "src/tracing/logger"; +import type { Logger } from "../tracing/logger"; function getUrlFromInput(input: RequestInfo | URL): string { if (input instanceof URL) { diff --git a/frontend/sync-client/tsconfig.json b/frontend/sync-client/tsconfig.json index 6db72fcc..40647521 100644 --- a/frontend/sync-client/tsconfig.json +++ b/frontend/sync-client/tsconfig.json @@ -1,12 +1,17 @@ { "compilerOptions": { - "baseUrl": ".", "module": "ESNext", "target": "ESNext", "strict": true, - "moduleResolution": "bundler", "allowSyntheticDefaultImports": true, - "lib": ["DOM", "ESNext"] + "moduleResolution": "bundler", + "lib": [ + "DOM", // to get "fetch" + ], + "declaration": true, + "declarationDir": "./dist/types" }, - "exclude": ["./dist"] -} + "exclude": [ + "./dist" + ] +} \ No newline at end of file diff --git a/frontend/test-client/src/agent/mock-client.ts b/frontend/test-client/src/agent/mock-client.ts index 6cbcca06..261d4323 100644 --- a/frontend/test-client/src/agent/mock-client.ts +++ b/frontend/test-client/src/agent/mock-client.ts @@ -1,10 +1,10 @@ import { assert } from "../utils/assert"; -import type { - RelativePath, - FileSystemOperations, - SyncSettings +import { + type RelativePath, + type FileSystemOperations, + type SyncSettings, + SyncClient } from "sync-client"; -import { SyncClient } from "sync-client"; export class MockClient implements FileSystemOperations { protected readonly localFiles = new Map(); @@ -24,8 +24,8 @@ export class MockClient implements FileSystemOperations { await Promise.all( Object.keys(this.initialSettings).map(async (key) => { return this.client.settings.setSetting( - key as keyof SyncSettings, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - this.initialSettings[key as keyof SyncSettings] // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion + key as keyof SyncSettings, + this.initialSettings[key as keyof SyncSettings]! ); }) ); @@ -88,10 +88,10 @@ export class MockClient implements FileSystemOperations { const newParts = newContent.split(" ").map((part) => part.trim()); existingParts.forEach((part) => // all changes should be additive - assert( + { assert( newParts.includes(part), `Part ${part} not found in new content` - ) + ); } ); this.client.logger.info( diff --git a/frontend/test-client/src/cli.ts b/frontend/test-client/src/cli.ts index 8e6e711b..f714dadb 100644 --- a/frontend/test-client/src/cli.ts +++ b/frontend/test-client/src/cli.ts @@ -92,7 +92,7 @@ async function runTest({ async function runTests(): Promise { const agentCounts = [2, 8]; const jitterScaleInSeconds = [0.5, 0, 2]; - const concurrencies = [1]; + const concurrencies = [16, 1]; const iterations = [50, 200]; const doDeletes = [true, false]; @@ -101,7 +101,7 @@ async function runTests(): Promise { for (const jitter of jitterScaleInSeconds) { for (const iteration of iterations) { for (const deleteFiles of doDeletes) { - for (let i = 0; i < 10; i++) { + for (let i = 0; i < 20; i++) { await runTest({ agentCount, concurrency, @@ -110,7 +110,6 @@ async function runTests(): Promise { jitterScaleInSeconds: jitter }); } - return; } } } From 15a09dff957efe426d8a8e32cfb310cecdd32d8b Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 15 Mar 2025 15:06:06 +0000 Subject: [PATCH 24/58] . --- frontend/package-lock.json | 59 +++++--------------------- frontend/sync-client/package.json | 25 ++++++----- frontend/sync-client/webpack.config.js | 39 +++++++++++------ frontend/test-client/run.sh | 4 +- frontend/test-client/src/cli.ts | 4 +- frontend/test-client/webpack.config.js | 3 +- 6 files changed, 59 insertions(+), 75 deletions(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index fe3e6b70..69a51099 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -22,7 +22,6 @@ "../backend/sync_lib/pkg": { "name": "sync_lib", "version": "0.0.30", - "dev": true, "license": "MIT" }, "node_modules/@ampproject/remapping": { @@ -39,7 +38,6 @@ }, "node_modules/@babel/code-frame": { "version": "7.26.2", - "dev": true, "license": "MIT", "dependencies": { "@babel/helper-validator-identifier": "^7.25.9", @@ -179,7 +177,6 @@ }, "node_modules/@babel/helper-validator-identifier": { "version": "7.25.9", - "dev": true, "license": "MIT", "engines": { "node": ">=6.9.0" @@ -1235,7 +1232,6 @@ }, "node_modules/@redocly/ajv": { "version": "8.11.2", - "dev": true, "license": "MIT", "dependencies": { "fast-deep-equal": "^3.1.1", @@ -1250,17 +1246,14 @@ }, "node_modules/@redocly/ajv/node_modules/json-schema-traverse": { "version": "1.0.0", - "dev": true, "license": "MIT" }, "node_modules/@redocly/config": { "version": "0.20.3", - "dev": true, "license": "MIT" }, "node_modules/@redocly/openapi-core": { "version": "1.29.0", - "dev": true, "license": "MIT", "dependencies": { "@redocly/ajv": "^8.11.2", @@ -1280,7 +1273,6 @@ }, "node_modules/@redocly/openapi-core/node_modules/brace-expansion": { "version": "2.0.1", - "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" @@ -1288,7 +1280,6 @@ }, "node_modules/@redocly/openapi-core/node_modules/minimatch": { "version": "5.1.6", - "dev": true, "license": "ISC", "dependencies": { "brace-expansion": "^2.0.1" @@ -1909,7 +1900,6 @@ }, "node_modules/agent-base": { "version": "7.1.3", - "dev": true, "license": "MIT", "engines": { "node": ">= 14" @@ -1976,7 +1966,6 @@ }, "node_modules/ansi-colors": { "version": "4.1.3", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -2039,7 +2028,6 @@ }, "node_modules/argparse": { "version": "2.0.1", - "dev": true, "license": "Python-2.0" }, "node_modules/async": { @@ -2161,7 +2149,6 @@ }, "node_modules/balanced-match": { "version": "1.0.2", - "dev": true, "license": "MIT" }, "node_modules/big.js": { @@ -2249,7 +2236,6 @@ }, "node_modules/byte-base64": { "version": "1.1.0", - "dev": true, "license": "MIT" }, "node_modules/call-bind-apply-helpers": { @@ -2335,7 +2321,6 @@ }, "node_modules/change-case": { "version": "5.4.4", - "dev": true, "license": "MIT" }, "node_modules/char-regex": { @@ -2445,7 +2430,6 @@ }, "node_modules/colorette": { "version": "1.4.0", - "dev": true, "license": "MIT" }, "node_modules/commander": { @@ -2597,7 +2581,6 @@ }, "node_modules/debug": { "version": "4.4.0", - "dev": true, "license": "MIT", "dependencies": { "ms": "^2.1.3" @@ -2991,7 +2974,6 @@ }, "node_modules/eventemitter3": { "version": "5.0.1", - "dev": true, "license": "MIT" }, "node_modules/events": { @@ -3048,7 +3030,6 @@ }, "node_modules/fast-deep-equal": { "version": "3.1.3", - "dev": true, "license": "MIT" }, "node_modules/fast-glob": { @@ -3128,7 +3109,6 @@ }, "node_modules/fetch-retry": { "version": "6.0.0", - "dev": true, "license": "MIT" }, "node_modules/file-entry-cache": { @@ -3449,7 +3429,6 @@ }, "node_modules/https-proxy-agent": { "version": "7.0.6", - "dev": true, "license": "MIT", "dependencies": { "agent-base": "^7.1.2", @@ -3536,7 +3515,6 @@ }, "node_modules/index-to-position": { "version": "0.1.2", - "dev": true, "license": "MIT", "engines": { "node": ">=18" @@ -4271,7 +4249,6 @@ }, "node_modules/js-levenshtein": { "version": "1.1.6", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -4279,12 +4256,10 @@ }, "node_modules/js-tokens": { "version": "4.0.0", - "dev": true, "license": "MIT" }, "node_modules/js-yaml": { "version": "4.1.0", - "dev": true, "license": "MIT", "dependencies": { "argparse": "^2.0.1" @@ -4631,7 +4606,6 @@ }, "node_modules/ms": { "version": "2.1.3", - "dev": true, "license": "MIT" }, "node_modules/nanoid": { @@ -4759,7 +4733,6 @@ }, "node_modules/openapi-fetch": { "version": "0.13.4", - "dev": true, "license": "MIT", "dependencies": { "openapi-typescript-helpers": "^0.0.15" @@ -4767,7 +4740,6 @@ }, "node_modules/openapi-typescript": { "version": "7.6.1", - "dev": true, "license": "MIT", "dependencies": { "@redocly/openapi-core": "^1.28.0", @@ -4786,12 +4758,10 @@ }, "node_modules/openapi-typescript-helpers": { "version": "0.0.15", - "dev": true, "license": "MIT" }, "node_modules/openapi-typescript/node_modules/parse-json": { "version": "8.1.0", - "dev": true, "license": "MIT", "dependencies": { "@babel/code-frame": "^7.22.13", @@ -4807,7 +4777,6 @@ }, "node_modules/openapi-typescript/node_modules/supports-color": { "version": "9.4.0", - "dev": true, "license": "MIT", "engines": { "node": ">=12" @@ -4818,7 +4787,6 @@ }, "node_modules/openapi-typescript/node_modules/type-fest": { "version": "4.35.0", - "dev": true, "license": "(MIT OR CC0-1.0)", "engines": { "node": ">=16" @@ -4873,7 +4841,6 @@ }, "node_modules/p-queue": { "version": "8.1.0", - "dev": true, "license": "MIT", "dependencies": { "eventemitter3": "^5.0.1", @@ -4888,7 +4855,6 @@ }, "node_modules/p-timeout": { "version": "6.1.4", - "dev": true, "license": "MIT", "engines": { "node": ">=14.16" @@ -4966,7 +4932,6 @@ }, "node_modules/picocolors": { "version": "1.1.1", - "dev": true, "license": "ISC" }, "node_modules/picomatch": { @@ -5049,7 +5014,6 @@ }, "node_modules/pluralize": { "version": "8.0.0", - "dev": true, "license": "MIT", "engines": { "node": ">=4" @@ -5328,7 +5292,6 @@ }, "node_modules/require-from-string": { "version": "2.0.2", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -6197,7 +6160,6 @@ }, "node_modules/typescript": { "version": "5.7.3", - "dev": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", @@ -6280,7 +6242,6 @@ }, "node_modules/uri-js-replace": { "version": "1.0.1", - "dev": true, "license": "MIT" }, "node_modules/url": { @@ -6311,7 +6272,6 @@ }, "node_modules/uuid": { "version": "11.1.0", - "dev": true, "funding": [ "https://github.com/sponsors/broofa", "https://github.com/sponsors/ctavan" @@ -6480,6 +6440,8 @@ }, "node_modules/webpack-merge": { "version": "6.0.1", + "resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-6.0.1.tgz", + "integrity": "sha512-hXXvrjtx2PLYx4qruKl+kyRSLc52V+cCvMxRjmKwoA+CBbbF5GfIBtR6kCvl0fYGqTUPKB+1ktVmTHqMOzgCBg==", "dev": true, "license": "MIT", "dependencies": { @@ -6643,7 +6605,6 @@ }, "node_modules/yaml-ast-parser": { "version": "0.0.43", - "dev": true, "license": "Apache-2.0" }, "node_modules/yargs": { @@ -6665,7 +6626,6 @@ }, "node_modules/yargs-parser": { "version": "21.1.1", - "dev": true, "license": "ISC", "engines": { "node": ">=12" @@ -6723,23 +6683,26 @@ }, "sync-client": { "version": "0.0.0", - "devDependencies": { - "@types/jest": "^29.5.14", - "@types/node": "^22.13.5", + "dependencies": { "byte-base64": "^1.1.0", "fetch-retry": "^6.0.0", - "jest": "^29.7.0", "openapi-fetch": "0.13.4", "openapi-typescript": "7.6.1", "p-queue": "^8.1.0", "sync_lib": "file:../../backend/sync_lib/pkg", + "uuid": "^11.1.0" + }, + "devDependencies": { + "@types/jest": "^29.5.14", + "@types/node": "^22.13.5", + "jest": "^29.7.0", "ts-jest": "^29.2.6", "ts-loader": "^9.5.2", "tslib": "2.8.1", "typescript": "5.7.3", - "uuid": "^11.1.0", "webpack": "^5.98.0", - "webpack-cli": "^6.0.1" + "webpack-cli": "^6.0.1", + "webpack-merge": "^6.0.1" } }, "test-client": { diff --git a/frontend/sync-client/package.json b/frontend/sync-client/package.json index 030d5c3d..d4a2850e 100644 --- a/frontend/sync-client/package.json +++ b/frontend/sync-client/package.json @@ -1,31 +1,36 @@ { "name": "sync-client", "version": "0.0.0", - "private": true, - "main": "dist/index.js", + "main": "dist/sync-client.node.js", + "browser": "dist/sync-client.web.js", "types": "dist/types/index.d.ts", - "module": "src/index.js", + "files": [ + "dist/**/*" + ], "scripts": { "dev": "webpack watch --mode development", "build": "webpack --mode production", "test": "NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules\" jest" }, - "devDependencies": { - "@types/jest": "^29.5.14", - "@types/node": "^22.13.5", + "dependencies": { "byte-base64": "^1.1.0", "fetch-retry": "^6.0.0", - "jest": "^29.7.0", "openapi-fetch": "0.13.4", "openapi-typescript": "7.6.1", "p-queue": "^8.1.0", - "sync_lib": "file:../../backend/sync_lib/pkg", + "uuid": "^11.1.0" + }, + "devDependencies": { + "@types/jest": "^29.5.14", + "@types/node": "^22.13.5", + "jest": "^29.7.0", "ts-jest": "^29.2.6", "ts-loader": "^9.5.2", "tslib": "2.8.1", "typescript": "5.7.3", - "uuid": "^11.1.0", "webpack": "^5.98.0", - "webpack-cli": "^6.0.1" + "webpack-cli": "^6.0.1", + "webpack-merge": "^6.0.1", + "sync_lib": "file:../../backend/sync_lib/pkg" } } \ No newline at end of file diff --git a/frontend/sync-client/webpack.config.js b/frontend/sync-client/webpack.config.js index 6d03f510..609d2533 100644 --- a/frontend/sync-client/webpack.config.js +++ b/frontend/sync-client/webpack.config.js @@ -1,6 +1,7 @@ const path = require("path"); +const { merge } = require("webpack-merge"); -module.exports = (_env, _argv) => ({ +const common = { entry: "./src/index.ts", module: { rules: [ @@ -27,15 +28,29 @@ module.exports = (_env, _argv) => ({ }, performance: { hints: false // it's a library, no need to warn about its size - }, - output: { - clean: true, - filename: "index.js", - globalObject: "this", - library: { - name: "SyncClient", - type: "umd" - }, - path: path.resolve(__dirname, "dist") } -}); +}; + +module.exports = [ + merge(common, { + target: "web", + output: { + path: path.resolve(__dirname, "dist"), + filename: "sync-client.web.js", + library: { + name: "SyncClient", + type: "umd", + export: "default" + }, + globalObject: "this" + } + }), + merge(common, { + target: "node", + output: { + path: path.resolve(__dirname, "dist"), + filename: "sync-client.node.js", + libraryTarget: "commonjs2" + } + }) +]; diff --git a/frontend/test-client/run.sh b/frontend/test-client/run.sh index 5bb39e94..04bbba67 100755 --- a/frontend/test-client/run.sh +++ b/frontend/test-client/run.sh @@ -14,9 +14,11 @@ process_count=$1 npm run build +mkdir -p logs + pids=() for i in $(seq 1 $process_count); do - node dist/cli.js 2>&1 > "log_${i}.log" & + node dist/cli.js 2>&1 > "logs/log_${i}.log" & pids+=($!) done diff --git a/frontend/test-client/src/cli.ts b/frontend/test-client/src/cli.ts index f714dadb..6db7b042 100644 --- a/frontend/test-client/src/cli.ts +++ b/frontend/test-client/src/cli.ts @@ -93,7 +93,7 @@ async function runTests(): Promise { const agentCounts = [2, 8]; const jitterScaleInSeconds = [0.5, 0, 2]; const concurrencies = [16, 1]; - const iterations = [50, 200]; + const iterations = [200]; const doDeletes = [true, false]; for (const agentCount of agentCounts) { @@ -101,7 +101,7 @@ async function runTests(): Promise { for (const jitter of jitterScaleInSeconds) { for (const iteration of iterations) { for (const deleteFiles of doDeletes) { - for (let i = 0; i < 20; i++) { + for (let i = 0; i < 3; i++) { await runTest({ agentCount, concurrency, diff --git a/frontend/test-client/webpack.config.js b/frontend/test-client/webpack.config.js index aa42a7df..b2324b9b 100644 --- a/frontend/test-client/webpack.config.js +++ b/frontend/test-client/webpack.config.js @@ -12,8 +12,7 @@ module.exports = { rules: [ { test: /\.ts$/, - use: "ts-loader", - exclude: /node_modules/ + use: "ts-loader" } ] }, From 826e391f87109f3326f9d1c48a67c9e7fa999d8b Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 15 Mar 2025 15:17:21 +0000 Subject: [PATCH 25/58] lint --- .../obsidian-plugin/src/views/history-view.ts | 1 - frontend/sync-client/package.json | 2 +- .../src/file-operations/file-operations.ts | 28 +- .../sync-client/src/persistence/database.ts | 19 +- .../src/services/connected-state.ts | 20 +- frontend/sync-client/src/services/types.ts | 1172 +++++++++-------- .../sync-client/src/sync-operations/syncer.ts | 14 +- .../sync-operations/unrestricted-syncer.ts | 27 +- frontend/sync-client/tsconfig.json | 8 +- frontend/test-client/src/agent/mock-client.ts | 15 +- 10 files changed, 652 insertions(+), 654 deletions(-) diff --git a/frontend/obsidian-plugin/src/views/history-view.ts b/frontend/obsidian-plugin/src/views/history-view.ts index ba2e18bc..d253b27f 100644 --- a/frontend/obsidian-plugin/src/views/history-view.ts +++ b/frontend/obsidian-plugin/src/views/history-view.ts @@ -60,7 +60,6 @@ export class HistoryView extends ItemView { } element.createEl("span", { - text: entry.relativePath }); diff --git a/frontend/sync-client/package.json b/frontend/sync-client/package.json index d4a2850e..778c5d35 100644 --- a/frontend/sync-client/package.json +++ b/frontend/sync-client/package.json @@ -33,4 +33,4 @@ "webpack-merge": "^6.0.1", "sync_lib": "file:../../backend/sync_lib/pkg" } -} \ No newline at end of file +} diff --git a/frontend/sync-client/src/file-operations/file-operations.ts b/frontend/sync-client/src/file-operations/file-operations.ts index 74448efa..de99bcb9 100644 --- a/frontend/sync-client/src/file-operations/file-operations.ts +++ b/frontend/sync-client/src/file-operations/file-operations.ts @@ -53,10 +53,14 @@ export class FileOperations { // All parent directories are created if they don't exist. public async create( path: RelativePath, - newContent: Uint8Array, - whatevs?: any + newContent: Uint8Array ): Promise { this.logger.debug(`Creating file: ${path}`); + + await this.fs.write(path, newContent); + } + + public async ensureClearPath(path: RelativePath): Promise { if (await this.fs.exists(path)) { const deconflictedPath = await this.deconflictPath(path); this.logger.debug( @@ -68,10 +72,6 @@ export class FileOperations { } else { await this.createParentDirectories(path); } - - whatevs?.(); - - await this.fs.write(path, newContent); } // Update the file at the given path. @@ -143,19 +143,7 @@ export class FileOperations { return; } - if (await this.fs.exists(newPath)) { - const deconflictedPath = await this.deconflictPath(newPath); - this.logger.debug( - `Conflict when moving '${oldPath}' to '${newPath}', the latter already exists, deconflicting by moving it to '${deconflictedPath}'` - ); - - // this.database.move(newPath, deconflictedPath); - // this.database.move(oldPath, newPath); - await this.fs.rename(newPath, deconflictedPath); - } else { - // this.database.move(oldPath, newPath); - await this.createParentDirectories(newPath); - } + await this.ensureClearPath(newPath); await this.fs.rename(oldPath, newPath); } @@ -198,7 +186,7 @@ export class FileOperations { ); stem = stem.replace(FileOperations.PARENTHESES_REGEX, ""); - let newName; + let newName = path; do { currentCount++; newName = `${directory}${stem} (${currentCount})${extension}`; diff --git a/frontend/sync-client/src/persistence/database.ts b/frontend/sync-client/src/persistence/database.ts index 42b8be86..73456dc3 100644 --- a/frontend/sync-client/src/persistence/database.ts +++ b/frontend/sync-client/src/persistence/database.ts @@ -130,12 +130,11 @@ export class Database { }, identity?: symbol ): void { - let entry: DocumentRecord | undefined; if (identity !== undefined) { const entry = this.getDocumentByIdentity(identity); this.documents = this.documents.filter( - ({ identity }) => identity !== entry.identity + (doc) => doc.identity !== entry.identity ); if (entry.relativePath !== relativePath) { @@ -161,7 +160,7 @@ export class Database { // We find a match based on relative path and we find one with a different document id // meaning that two documents occupy the same path in terms of in-flight requests so we // need to create a new parallel version. - entry = this.getLatestDocumentByRelativePath(relativePath); + const entry = this.getLatestDocumentByRelativePath(relativePath); if (entry && entry.documentId !== documentId) { this.documents.push({ // `entry` might be undefined if the document is new @@ -238,7 +237,8 @@ export class Database { relativePath: RelativePath, promise: Promise ): void { - const previousEntry = this.getLatestDocumentByRelativePath(relativePath); + const previousEntry = + this.getLatestDocumentByRelativePath(relativePath); const entry = { relativePath, @@ -300,7 +300,8 @@ export class Database { ({ identity }) => identity !== oldDocument.identity ); - const newDocument = this.getLatestDocumentByRelativePath(newRelativePath); + const newDocument = + this.getLatestDocumentByRelativePath(newRelativePath); if (newDocument !== undefined && !newDocument.isDeleted) { throw new Error( `Document already exists at new location: ${newRelativePath}` @@ -338,11 +339,13 @@ export class Database { this.ensureConsistency(); void this.saveData({ documents: this.resolvedDocuments.map( - ({ relativePath, metadata }) => ({ + ({ relativePath, documentId, metadata }) => ({ + documentId, relativePath, - ...metadata + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + ...metadata! // resolvedDocuments only returns docs with metadata set }) - ) as StoredDocumentMetadata[], + ), lastSeenUpdateId: this.lastSeenUpdateId }); } diff --git a/frontend/sync-client/src/services/connected-state.ts b/frontend/sync-client/src/services/connected-state.ts index 7e3a28e7..4b62b792 100644 --- a/frontend/sync-client/src/services/connected-state.ts +++ b/frontend/sync-client/src/services/connected-state.ts @@ -23,16 +23,6 @@ export class ConnectedState { }); } - private handleComingOnline() { - this.logger.debug("Sync is enabled"); - this.resolveIsSyncEnabled?.(); - } - - private handleGoingOffline() { - this.logger.debug("Sync is disabled"); - [this.syncIsEnabled, this.resolveIsSyncEnabled] = createPromise(); - } - public getFetchImplementation( fetch: typeof globalThis.fetch, { doRetries = true }: { doRetries: boolean } = { doRetries: true } @@ -48,4 +38,14 @@ export class ConnectedState { return retriedFetch(input); }; } + + private handleComingOnline(): void { + this.logger.debug("Sync is enabled"); + this.resolveIsSyncEnabled?.(); + } + + private handleGoingOffline(): void { + this.logger.debug("Sync is disabled"); + [this.syncIsEnabled, this.resolveIsSyncEnabled] = createPromise(); + } } diff --git a/frontend/sync-client/src/services/types.ts b/frontend/sync-client/src/services/types.ts index 642fd6c2..5c464075 100644 --- a/frontend/sync-client/src/services/types.ts +++ b/frontend/sync-client/src/services/types.ts @@ -4,594 +4,596 @@ */ export interface paths { - "/ping": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - get: { - parameters: { - query?: never; - header?: { - authorization?: string; - }; - path?: never; - cookie?: never; - }; - requestBody?: never; - responses: { - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["PingResponse"]; - }; - }; - default: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["SerializedError"]; - }; - }; - }; - }; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/vaults/{vault_id}/documents": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - get: { - parameters: { - query?: { - since_update_id?: number | null; - }; - header: { - authorization: string; - }; - path: { - vault_id: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["FetchLatestDocumentsResponse"]; - }; - }; - default: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["SerializedError"]; - }; - }; - }; - }; - put?: never; - post: { - parameters: { - query?: never; - header: { - authorization: string; - }; - path: { - vault_id: string; - }; - cookie?: never; - }; - requestBody: { - content: { - "multipart/form-data": components["schemas"]["CreateDocumentVersionMultipart"]; - }; - }; - responses: { - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["DocumentVersionWithoutContent"]; - }; - }; - default: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["SerializedError"]; - }; - }; - }; - }; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/vaults/{vault_id}/documents/json": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - get?: never; - put?: never; - post: { - parameters: { - query?: never; - header: { - authorization: string; - }; - path: { - vault_id: string; - }; - cookie?: never; - }; - requestBody: { - content: { - "application/json": components["schemas"]["CreateDocumentVersion"]; - }; - }; - responses: { - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["DocumentVersionWithoutContent"]; - }; - }; - default: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["SerializedError"]; - }; - }; - }; - }; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/vaults/{vault_id}/documents/{document_id}": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - get: { - parameters: { - query?: never; - header: { - authorization: string; - }; - path: { - document_id: string; - vault_id: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["DocumentVersion"]; - }; - }; - default: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["SerializedError"]; - }; - }; - }; - }; - put: { - parameters: { - query?: never; - header: { - authorization: string; - }; - path: { - document_id: string; - vault_id: string; - }; - cookie?: never; - }; - requestBody: { - content: { - "multipart/form-data": components["schemas"]["UpdateDocumentVersionMultipart"]; - }; - }; - responses: { - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["DocumentUpdateResponse"]; - }; - }; - default: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["SerializedError"]; - }; - }; - }; - }; - post?: never; - delete: { - parameters: { - query?: never; - header: { - authorization: string; - }; - path: { - document_id: string; - vault_id: string; - }; - cookie?: never; - }; - requestBody: { - content: { - "application/json": components["schemas"]["DeleteDocumentVersion"]; - }; - }; - responses: { - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["DocumentVersionWithoutContent"]; - }; - }; - default: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["SerializedError"]; - }; - }; - }; - }; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/vaults/{vault_id}/documents/{document_id}/json": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - get?: never; - put: { - parameters: { - query?: never; - header: { - authorization: string; - }; - path: { - document_id: string; - vault_id: string; - }; - cookie?: never; - }; - requestBody: { - content: { - "application/json": components["schemas"]["UpdateDocumentVersion"]; - }; - }; - responses: { - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["DocumentUpdateResponse"]; - }; - }; - default: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["SerializedError"]; - }; - }; - }; - }; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/vaults/{vault_id}/documents/{document_id}/versions/{version_id}": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - get?: never; - put: { - parameters: { - query?: never; - header: { - authorization: string; - }; - path: { - document_id: string; - vault_id: string; - vault_update_id: number; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["DocumentVersion"]; - }; - }; - default: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["SerializedError"]; - }; - }; - }; - }; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/vaults/{vault_id}/documents/{document_id}/versions/{version_id}/content": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - get?: never; - put: { - parameters: { - query?: never; - header: { - authorization: string; - }; - path: { - document_id: string; - vault_id: string; - vault_update_id: number; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description byte stream */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/octet-stream": unknown; - }; - }; - default: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["SerializedError"]; - }; - }; - }; - }; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; + "/ping": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: { + parameters: { + query?: never; + header?: { + authorization?: string; + }; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["PingResponse"]; + }; + }; + default: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SerializedError"]; + }; + }; + }; + }; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/vaults/{vault_id}/documents": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: { + parameters: { + query?: { + since_update_id?: number | null; + }; + header: { + authorization: string; + }; + path: { + vault_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["FetchLatestDocumentsResponse"]; + }; + }; + default: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SerializedError"]; + }; + }; + }; + }; + put?: never; + post: { + parameters: { + query?: never; + header: { + authorization: string; + }; + path: { + vault_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "multipart/form-data": components["schemas"]["CreateDocumentVersionMultipart"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DocumentVersionWithoutContent"]; + }; + }; + default: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SerializedError"]; + }; + }; + }; + }; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/vaults/{vault_id}/documents/json": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: { + parameters: { + query?: never; + header: { + authorization: string; + }; + path: { + vault_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["CreateDocumentVersion"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DocumentVersionWithoutContent"]; + }; + }; + default: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SerializedError"]; + }; + }; + }; + }; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/vaults/{vault_id}/documents/{document_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: { + parameters: { + query?: never; + header: { + authorization: string; + }; + path: { + document_id: string; + vault_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DocumentVersion"]; + }; + }; + default: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SerializedError"]; + }; + }; + }; + }; + put: { + parameters: { + query?: never; + header: { + authorization: string; + }; + path: { + document_id: string; + vault_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "multipart/form-data": components["schemas"]["UpdateDocumentVersionMultipart"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DocumentUpdateResponse"]; + }; + }; + default: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SerializedError"]; + }; + }; + }; + }; + post?: never; + delete: { + parameters: { + query?: never; + header: { + authorization: string; + }; + path: { + document_id: string; + vault_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["DeleteDocumentVersion"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DocumentVersionWithoutContent"]; + }; + }; + default: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SerializedError"]; + }; + }; + }; + }; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/vaults/{vault_id}/documents/{document_id}/json": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put: { + parameters: { + query?: never; + header: { + authorization: string; + }; + path: { + document_id: string; + vault_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["UpdateDocumentVersion"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DocumentUpdateResponse"]; + }; + }; + default: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SerializedError"]; + }; + }; + }; + }; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/vaults/{vault_id}/documents/{document_id}/versions/{version_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put: { + parameters: { + query?: never; + header: { + authorization: string; + }; + path: { + document_id: string; + vault_id: string; + vault_update_id: number; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DocumentVersion"]; + }; + }; + default: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SerializedError"]; + }; + }; + }; + }; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/vaults/{vault_id}/documents/{document_id}/versions/{version_id}/content": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put: { + parameters: { + query?: never; + header: { + authorization: string; + }; + path: { + document_id: string; + vault_id: string; + vault_update_id: number; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description byte stream */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/octet-stream": unknown; + }; + }; + default: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SerializedError"]; + }; + }; + }; + }; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; } export type webhooks = Record; export interface components { - schemas: { - Array_of_uint8: number[]; - CreateDocumentVersion: { - contentBase64: string; - /** Format: uuid */ - documentId?: string | null; - relativePath: string; - }; - CreateDocumentVersionMultipart: { - content: components["schemas"]["Array_of_uint8"]; - /** Format: uuid */ - document_id?: string | null; - relative_path: string; - }; - DeleteDocumentVersion: { - relativePath: string; - }; - /** @description Response to an update document request. */ - DocumentUpdateResponse: { - /** Format: uuid */ - documentId: string; - isDeleted: boolean; - relativePath: string; - /** @enum {string} */ - type: "FastForwardUpdate"; - /** Format: date-time */ - updatedDate: string; - vaultId: string; - /** Format: int64 */ - vaultUpdateId: number; - } | { - contentBase64: string; - /** Format: uuid */ - documentId: string; - isDeleted: boolean; - relativePath: string; - /** @enum {string} */ - type: "MergingUpdate"; - /** Format: date-time */ - updatedDate: string; - vaultId: string; - /** Format: int64 */ - vaultUpdateId: number; - }; - DocumentVersion: { - contentBase64: string; - /** Format: uuid */ - documentId: string; - isDeleted: boolean; - relativePath: string; - /** Format: date-time */ - updatedDate: string; - vaultId: string; - /** Format: int64 */ - vaultUpdateId: number; - }; - DocumentVersionWithoutContent: { - /** Format: uuid */ - documentId: string; - isDeleted: boolean; - relativePath: string; - /** Format: date-time */ - updatedDate: string; - vaultId: string; - /** Format: int64 */ - vaultUpdateId: number; - }; - /** @description Response to a fetch latest documents request. */ - FetchLatestDocumentsResponse: { - /** - * Format: int64 - * @description The update ID of the latest document in the response. - */ - lastUpdateId: number; - latestDocuments: components["schemas"]["DocumentVersionWithoutContent"][]; - }; - PathParams: { - vault_id: string; - }; - PathParams2: { - vault_id: string; - }; - PathParams3: { - /** Format: uuid */ - document_id: string; - vault_id: string; - }; - PathParams4: { - /** Format: uuid */ - document_id: string; - vault_id: string; - }; - PathParams5: { - /** Format: uuid */ - document_id: string; - vault_id: string; - /** Format: int64 */ - vault_update_id: number; - }; - PathParams6: { - /** Format: uuid */ - document_id: string; - vault_id: string; - /** Format: int64 */ - vault_update_id: number; - }; - PathParams7: { - /** Format: uuid */ - document_id: string; - vault_id: string; - }; - /** @description Response to a ping request. */ - PingResponse: { - /** @description Whether the client is authenticated based on the sent Authorization header. */ - isAuthenticated: boolean; - /** @description Semantic version of the server. */ - serverVersion: string; - }; - QueryParams: { - /** Format: int64 */ - since_update_id?: number | null; - }; - SerializedError: { - causes: string[]; - message: string; - }; - UpdateDocumentVersion: { - contentBase64: string; - /** Format: int64 */ - parentVersionId: number; - relativePath: string; - }; - UpdateDocumentVersionMultipart: { - content: components["schemas"]["Array_of_uint8"]; - /** Format: int64 */ - parentVersionId: number; - relativePath: string; - }; - }; - responses: never; - parameters: never; - requestBodies: never; - headers: never; - pathItems: never; + schemas: { + Array_of_uint8: number[]; + CreateDocumentVersion: { + contentBase64: string; + /** Format: uuid */ + documentId?: string | null; + relativePath: string; + }; + CreateDocumentVersionMultipart: { + content: components["schemas"]["Array_of_uint8"]; + /** Format: uuid */ + document_id?: string | null; + relative_path: string; + }; + DeleteDocumentVersion: { + relativePath: string; + }; + /** @description Response to an update document request. */ + DocumentUpdateResponse: + | { + /** Format: uuid */ + documentId: string; + isDeleted: boolean; + relativePath: string; + /** @enum {string} */ + type: "FastForwardUpdate"; + /** Format: date-time */ + updatedDate: string; + vaultId: string; + /** Format: int64 */ + vaultUpdateId: number; + } + | { + contentBase64: string; + /** Format: uuid */ + documentId: string; + isDeleted: boolean; + relativePath: string; + /** @enum {string} */ + type: "MergingUpdate"; + /** Format: date-time */ + updatedDate: string; + vaultId: string; + /** Format: int64 */ + vaultUpdateId: number; + }; + DocumentVersion: { + contentBase64: string; + /** Format: uuid */ + documentId: string; + isDeleted: boolean; + relativePath: string; + /** Format: date-time */ + updatedDate: string; + vaultId: string; + /** Format: int64 */ + vaultUpdateId: number; + }; + DocumentVersionWithoutContent: { + /** Format: uuid */ + documentId: string; + isDeleted: boolean; + relativePath: string; + /** Format: date-time */ + updatedDate: string; + vaultId: string; + /** Format: int64 */ + vaultUpdateId: number; + }; + /** @description Response to a fetch latest documents request. */ + FetchLatestDocumentsResponse: { + /** + * Format: int64 + * @description The update ID of the latest document in the response. + */ + lastUpdateId: number; + latestDocuments: components["schemas"]["DocumentVersionWithoutContent"][]; + }; + PathParams: { + vault_id: string; + }; + PathParams2: { + vault_id: string; + }; + PathParams3: { + /** Format: uuid */ + document_id: string; + vault_id: string; + }; + PathParams4: { + /** Format: uuid */ + document_id: string; + vault_id: string; + }; + PathParams5: { + /** Format: uuid */ + document_id: string; + vault_id: string; + /** Format: int64 */ + vault_update_id: number; + }; + PathParams6: { + /** Format: uuid */ + document_id: string; + vault_id: string; + /** Format: int64 */ + vault_update_id: number; + }; + PathParams7: { + /** Format: uuid */ + document_id: string; + vault_id: string; + }; + /** @description Response to a ping request. */ + PingResponse: { + /** @description Whether the client is authenticated based on the sent Authorization header. */ + isAuthenticated: boolean; + /** @description Semantic version of the server. */ + serverVersion: string; + }; + QueryParams: { + /** Format: int64 */ + since_update_id?: number | null; + }; + SerializedError: { + causes: string[]; + message: string; + }; + UpdateDocumentVersion: { + contentBase64: string; + /** Format: int64 */ + parentVersionId: number; + relativePath: string; + }; + UpdateDocumentVersionMultipart: { + content: components["schemas"]["Array_of_uint8"]; + /** Format: int64 */ + parentVersionId: number; + relativePath: string; + }; + }; + responses: never; + parameters: never; + requestBodies: never; + headers: never; + pathItems: never; } export type $defs = Record; export type operations = Record; diff --git a/frontend/sync-client/src/sync-operations/syncer.ts b/frontend/sync-client/src/sync-operations/syncer.ts index d30f8457..5e2327fd 100644 --- a/frontend/sync-client/src/sync-operations/syncer.ts +++ b/frontend/sync-client/src/sync-operations/syncer.ts @@ -44,11 +44,11 @@ export class Syncer { this.syncQueue.concurrency = newSettings.syncConcurrency; }); - this.syncQueue.on("active", () => - { this.remainingOperationsListeners.forEach((listener) => - { listener(this.syncQueue.size); } - ); } - ); + this.syncQueue.on("active", () => { + this.remainingOperationsListeners.forEach((listener) => { + listener(this.syncQueue.size); + }); + }); this.internalSyncer = new UnrestrictedSyncer( logger, @@ -261,7 +261,9 @@ export class Syncer { public async reset(): Promise { this.syncQueue.clear(); await this.syncQueue.onEmpty(); - this.remainingOperationsListeners.forEach((listener) => { listener(0); }); + this.remainingOperationsListeners.forEach((listener) => { + listener(0); + }); this.internalSyncer.reset(); } diff --git a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts index 1734e25b..ce3f4e60 100644 --- a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts +++ b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts @@ -288,9 +288,7 @@ export class UnrestrictedSyncer { async () => { let localMetadata = getLatestDocument(); - if ( - localMetadata?.metadata !== undefined - ) { + if (localMetadata?.metadata !== undefined) { // If the file exists locally, let's pretend the user has updated it // and deal with remote update/deletion within `unrestrictedSyncLocallyUpdatedFile` if ( @@ -306,6 +304,7 @@ export class UnrestrictedSyncer { return this.unrestrictedSyncLocallyUpdatedFile({ getLatestDocument: () => this.database.getDocumentByIdentity( + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion localMetadata!.identity ) }); @@ -334,8 +333,8 @@ export class UnrestrictedSyncer { } if ( - localMetadata?.metadata?.parentVersionId ?? - -1 >= remoteVersion.vaultUpdateId + (localMetadata?.metadata?.parentVersionId ?? -1) >= + remoteVersion.vaultUpdateId ) { this.logger.debug( `Document ${remoteVersion.relativePath} is already more up to date than the fetched version` @@ -347,15 +346,19 @@ export class UnrestrictedSyncer { const [promise, resolve] = createPromise(); + await this.operations.ensureClearPath( + remoteVersion.relativePath + ); + + this.database.getNewResolvedDocumentByRelativePath( + remoteVersion.documentId, + remoteVersion.relativePath, + promise + ); + await this.operations.create( remoteVersion.relativePath, - contentBytes, - () => - { this.database.getNewResolvedDocumentByRelativePath( - remoteVersion.documentId, - remoteVersion.relativePath, - promise - ); } + contentBytes ); const document = diff --git a/frontend/sync-client/tsconfig.json b/frontend/sync-client/tsconfig.json index 40647521..ee31a31e 100644 --- a/frontend/sync-client/tsconfig.json +++ b/frontend/sync-client/tsconfig.json @@ -6,12 +6,10 @@ "allowSyntheticDefaultImports": true, "moduleResolution": "bundler", "lib": [ - "DOM", // to get "fetch" + "DOM" // to get "fetch" ], "declaration": true, "declarationDir": "./dist/types" }, - "exclude": [ - "./dist" - ] -} \ No newline at end of file + "exclude": ["./dist"] +} diff --git a/frontend/test-client/src/agent/mock-client.ts b/frontend/test-client/src/agent/mock-client.ts index 261d4323..941b87d1 100644 --- a/frontend/test-client/src/agent/mock-client.ts +++ b/frontend/test-client/src/agent/mock-client.ts @@ -23,9 +23,10 @@ export class MockClient implements FileSystemOperations { await Promise.all( Object.keys(this.initialSettings).map(async (key) => { + const settingKey = key as keyof SyncSettings; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion return this.client.settings.setSetting( - key as keyof SyncSettings, - this.initialSettings[key as keyof SyncSettings]! + settingKey, + this.initialSettings[settingKey]! // eslint-disable-line @typescript-eslint/no-non-null-assertion ); }) ); @@ -88,10 +89,12 @@ export class MockClient implements FileSystemOperations { const newParts = newContent.split(" ").map((part) => part.trim()); existingParts.forEach((part) => // all changes should be additive - { assert( - newParts.includes(part), - `Part ${part} not found in new content` - ); } + { + assert( + newParts.includes(part), + `Part ${part} not found in new content` + ); + } ); this.client.logger.info( From aad93fef84b7359b39b43db92f7afd68b5017e3f Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 15 Mar 2025 17:13:11 +0000 Subject: [PATCH 26/58] Fix CI --- backend/rust-toolchain.toml | 4 ++++ backend/sync_lib/src/lib.rs | 2 +- frontend/test-client/run.sh | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 backend/rust-toolchain.toml diff --git a/backend/rust-toolchain.toml b/backend/rust-toolchain.toml new file mode 100644 index 00000000..46870171 --- /dev/null +++ b/backend/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "nightly-2025-03-14" +targets = [ "x86_64-unknown-linux-gnu" ] +profile = "default" diff --git a/backend/sync_lib/src/lib.rs b/backend/sync_lib/src/lib.rs index 16e01f24..e0fa5eb7 100644 --- a/backend/sync_lib/src/lib.rs +++ b/backend/sync_lib/src/lib.rs @@ -66,7 +66,7 @@ pub fn merge_text(parent: &str, left: &str, right: &str) -> String { pub fn is_binary(data: &[u8]) -> bool { set_panic_hook(); - if data.iter().any(|&b| b == 0) { + if data.contains(&0) { // Even though the NUL character is valid in UTF-8, it's highly suspicious in // human-readable text. return true; diff --git a/frontend/test-client/run.sh b/frontend/test-client/run.sh index 04bbba67..f559db2f 100755 --- a/frontend/test-client/run.sh +++ b/frontend/test-client/run.sh @@ -31,7 +31,7 @@ print_failed_log() { # Only consider non-zero exit codes as failures if [ $exit_code -ne 0 ]; then - echo "Process ${pids[$i-1]} failed with exit code $exit_code. Log file: $(pwd)/log_${i}.log" + echo "Process ${pids[$i-1]} failed with exit code $exit_code. Log file: $(pwd)/logs/log_${i}.log" return 0 else echo "Process ${pids[$i-1]} completed successfully with exit code 0" From 2987afb20a2e86aacfc1ff2744c4430673c00aea Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 15 Mar 2025 17:13:27 +0000 Subject: [PATCH 27/58] use toolchain --- .github/workflows/check.yml | 14 ++++++++++---- .github/workflows/publish-plugin.yml | 2 -- backend/Dockerfile | 2 -- backend/rust-toolchain.toml | 2 +- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 35e18428..ab601843 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -19,9 +19,6 @@ jobs: - name: Setup run: | - rustup install nightly - rustup default nightly - rustup component add clippy rustfmt cargo install sqlx-cli cd backend sqlx database create --database-url sqlite://db.sqlite3 @@ -44,7 +41,7 @@ jobs: cd backend cargo test --verbose cd sync_lib - # wasm-pack test --node # todo: fix this in CI + wasm-pack test --node - name: Lint frontend run: | @@ -62,3 +59,12 @@ jobs: run: | cd frontend npm run test + + - name: E2E tests + run: | + cd ../backend + UST_BACKTRACE=1 cargo run -p sync_server & + cd - + + npm run build + test-client/run.sh 32 diff --git a/.github/workflows/publish-plugin.yml b/.github/workflows/publish-plugin.yml index f1c816ff..c21d986e 100644 --- a/.github/workflows/publish-plugin.yml +++ b/.github/workflows/publish-plugin.yml @@ -23,8 +23,6 @@ jobs: - name: Build wasm run: | cd backend - rustup install nightly - rustup default nightly cargo install wasm-pack wasm-pack build --target web sync_lib diff --git a/backend/Dockerfile b/backend/Dockerfile index 8d2fdc46..c26c125d 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -3,8 +3,6 @@ FROM rust:1.83 AS builder WORKDIR /usr/src/backend RUN apt update && apt install -y musl-tools -RUN rustup install nightly && rustup default nightly -RUN rustup target add x86_64-unknown-linux-musl RUN cargo install sqlx-cli COPY . . diff --git a/backend/rust-toolchain.toml b/backend/rust-toolchain.toml index 46870171..8e466642 100644 --- a/backend/rust-toolchain.toml +++ b/backend/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] channel = "nightly-2025-03-14" -targets = [ "x86_64-unknown-linux-gnu" ] +targets = [ "x86_64-unknown-linux-gnu", "x86_64-unknown-linux-musl" ] profile = "default" From d5112a7d0fdd9cba991cd8b8d52b4dd14db2c33f Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 15 Mar 2025 17:15:44 +0000 Subject: [PATCH 28/58] clean up --- README.md | 4 - .../src/file-operations/file-operations.ts | 9 +- .../sync-client/src/persistence/database.ts | 132 ++++-------------- .../sync-client/src/sync-operations/syncer.ts | 68 ++++----- .../sync-operations/unrestricted-syncer.ts | 132 ++++++------------ frontend/test-client/src/agent/mock-client.ts | 44 ++++-- frontend/test-client/src/cli.ts | 30 ++-- 7 files changed, 147 insertions(+), 272 deletions(-) diff --git a/README.md b/README.md index 51440c31..62842bc3 100644 --- a/README.md +++ b/README.md @@ -17,10 +17,6 @@ - Install [`rustup`](https://rustup.rs): `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh` - `sudo apt install llvm -y` -- `rustup self update` -- `rustup update` -- `rustup install nightly` -- `rustup default nightly` - `rustup component add llvm-tools-preview` - `cargo install cargo-generate cargo-fuzz cargo-insta rustfilt cargo-binutils` - Install [`wasm-pack`](https://rustwasm.github.io/wasm-pack/installer): `curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh` diff --git a/frontend/sync-client/src/file-operations/file-operations.ts b/frontend/sync-client/src/file-operations/file-operations.ts index de99bcb9..01084647 100644 --- a/frontend/sync-client/src/file-operations/file-operations.ts +++ b/frontend/sync-client/src/file-operations/file-operations.ts @@ -2,10 +2,7 @@ import type { Logger } from "../tracing/logger"; import type { FileSystemOperations } from "./filesystem-operations"; import type { Database, RelativePath } from "../persistence/database"; import { isBinary, isFileTypeMergable, mergeText } from "sync_lib"; -import { - FileNotFoundError, - SafeFileSystemOperations -} from "./safe-filesystem-operations"; +import { SafeFileSystemOperations } from "./safe-filesystem-operations"; export class FileOperations { private static readonly PARENTHESES_REGEX = / \((\d+)\)$/; @@ -67,7 +64,7 @@ export class FileOperations { `Didn't expect ${path} to exist, deconflicting by moving it to '${deconflictedPath}'` ); - // this.database.move(path, deconflictedPath); + this.database.move(path, deconflictedPath); await this.fs.rename(path, deconflictedPath); } else { await this.createParentDirectories(path); @@ -142,9 +139,9 @@ export class FileOperations { if (oldPath === newPath) { return; } - await this.ensureClearPath(newPath); + this.database.move(oldPath, newPath); await this.fs.rename(oldPath, newPath); } diff --git a/frontend/sync-client/src/persistence/database.ts b/frontend/sync-client/src/persistence/database.ts index 73456dc3..c4ff6d82 100644 --- a/frontend/sync-client/src/persistence/database.ts +++ b/frontend/sync-client/src/persistence/database.ts @@ -22,7 +22,6 @@ export interface StoredDatabase { } export interface DocumentRecord { - identity: symbol; relativePath: RelativePath; documentId: DocumentId; metadata: DocumentMetadata | undefined; @@ -47,7 +46,6 @@ export class Database { ({ relativePath, documentId, ...metadata }) => ({ relativePath, documentId, - identity: Symbol(), metadata, isDeleted: false, updates: [], @@ -118,85 +116,33 @@ export class Database { public setDocument( { - documentId, - relativePath, parentVersionId, hash }: { - documentId: DocumentId; - relativePath: RelativePath; parentVersionId: VaultUpdateId; hash: string; }, - identity?: symbol + toUpdate: DocumentRecord ): void { - if (identity !== undefined) { - const entry = this.getDocumentByIdentity(identity); - - this.documents = this.documents.filter( - (doc) => doc.identity !== entry.identity - ); - - if (entry.relativePath !== relativePath) { - throw new Error( - "Document identity does not match the relative path" - ); - } - - this.documents.push({ - ...entry, - relativePath, - documentId, - metadata: { - parentVersionId, - hash - } - }); - - this.save(); - return; + if (!this.documents.includes(toUpdate)) { + throw new Error("Document not found in database"); } - // We find a match based on relative path and we find one with a different document id - // meaning that two documents occupy the same path in terms of in-flight requests so we - // need to create a new parallel version. - const entry = this.getLatestDocumentByRelativePath(relativePath); - if (entry && entry.documentId !== documentId) { - this.documents.push({ - // `entry` might be undefined if the document is new - identity: Symbol(), - relativePath, - documentId, - metadata: { - parentVersionId, - hash - }, - isDeleted: false, - updates: [], - parallelVersion: entry.parallelVersion + 1 - }); - this.save(); - return; - } - - this.documents.push({ - identity: Symbol(), - relativePath, - documentId, - metadata: { - parentVersionId, - hash - }, - isDeleted: false, - updates: [], - parallelVersion: 0 - }); + toUpdate.metadata = { parentVersionId, hash }; this.save(); + return; } public removeDocumentPromise(promise: Promise): void { - const entry = this.getDocumentByUpdatePromise(promise); + const entry = this.documents.find(({ updates }) => + updates.includes(promise) + ); + + if (entry === undefined) { + throw new Error("Document not found by update promise"); + } + entry.updates = entry.updates.filter((update) => update !== promise); // No need to save as Promises don't get serialized } @@ -214,7 +160,7 @@ export class Database { public async getResolvedDocumentByRelativePath( relativePath: RelativePath, promise: Promise - ): Promise { + ): Promise { const entry = this.getLatestDocumentByRelativePath(relativePath); if (entry === undefined) { @@ -230,20 +176,21 @@ export class Database { const currentPromises = entry.updates; entry.updates = [...currentPromises, promise]; await Promise.all(currentPromises); + + return entry; } - public getNewResolvedDocumentByRelativePath( + public createNewPendingDocument( documentId: DocumentId, relativePath: RelativePath, promise: Promise - ): void { + ): DocumentRecord { const previousEntry = this.getLatestDocumentByRelativePath(relativePath); const entry = { relativePath, documentId, - identity: Symbol(), metadata: undefined, isDeleted: false, updates: [promise], @@ -255,18 +202,8 @@ export class Database { this.documents.push(entry); this.save(); - } - public getDocumentByUpdatePromise(promise: Promise): DocumentRecord { - const result = this.documents.find(({ updates }) => - updates.includes(promise) - ); - - if (result === undefined) { - throw new Error("Document not found by update promise"); - } - - return result; + return entry; } public getDocumentByDocumentId( @@ -275,16 +212,6 @@ export class Database { return this.documents.find(({ documentId }) => documentId === find); } - public getDocumentByIdentity(find: symbol): DocumentRecord { - const result = this.documents.find(({ identity }) => identity === find); - - if (result === undefined) { - throw new Error("Document not found by identity symbol"); - } - - return result; - } - public move( oldRelativePath: RelativePath, newRelativePath: RelativePath @@ -296,10 +223,6 @@ export class Database { return; } - this.documents = this.documents.filter( - ({ identity }) => identity !== oldDocument.identity - ); - const newDocument = this.getLatestDocumentByRelativePath(newRelativePath); if (newDocument !== undefined && !newDocument.isDeleted) { @@ -308,17 +231,12 @@ export class Database { ); } - // It's either an invalid state of newDocument is pending deletion and we have - // to wait for it to complete. - this.documents.push({ - ...oldDocument, - relativePath: newRelativePath, - // We're in a strange state where the target of the move has just got deleted, - // however, its metadata might already have a bunch of updates queued up for - // the document at the new location. We need to keep these updates. - parallelVersion: - newDocument !== undefined ? newDocument.parallelVersion + 1 : 0 - }); + oldDocument.relativePath = newRelativePath; + // We're in a strange state where the target of the move has just got deleted, + // however, its metadata might already have a bunch of updates queued up for + // the document at the new location. We need to keep these updates. + oldDocument.parallelVersion = + newDocument !== undefined ? newDocument.parallelVersion + 1 : 0; this.save(); } diff --git a/frontend/sync-client/src/sync-operations/syncer.ts b/frontend/sync-client/src/sync-operations/syncer.ts index 5e2327fd..e882004f 100644 --- a/frontend/sync-client/src/sync-operations/syncer.ts +++ b/frontend/sync-client/src/sync-operations/syncer.ts @@ -3,8 +3,8 @@ import type { SyncService } from "../services/sync-service"; import type { Logger } from "../tracing/logger"; import type { SyncHistory } from "../tracing/sync-history"; import PQueue from "p-queue"; -import { v4 as uuidv4 } from "uuid"; import { hash } from "../utils/hash"; +import { v4 as uuidv4 } from "uuid"; import type { components } from "../services/types"; import type { Settings } from "../persistence/settings"; import type { FileOperations } from "../file-operations/file-operations"; @@ -98,20 +98,16 @@ export class Syncer { } const [promise, resolve, reject] = createPromise(); - const proposedDocumentId = uuidv4(); - this.database.getNewResolvedDocumentByRelativePath( - proposedDocumentId, + const document = this.database.createNewPendingDocument( + uuidv4(), relativePath, promise ); try { await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncLocallyCreatedFile( - proposedDocumentId, - () => this.database.getDocumentByUpdatePromise(promise) - ) + this.internalSyncer.unrestrictedSyncLocallyCreatedFile(document) ); resolve(); @@ -131,16 +127,14 @@ export class Syncer { const [promise, resolve, reject] = createPromise(); - await this.database.getResolvedDocumentByRelativePath( + const document = await this.database.getResolvedDocumentByRelativePath( relativePath, promise ); try { await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncLocallyDeletedFile(() => - this.database.getDocumentByUpdatePromise(promise) - ) + this.internalSyncer.unrestrictedSyncLocallyDeletedFile(document) ); resolve(); @@ -158,17 +152,13 @@ export class Syncer { oldPath?: RelativePath; relativePath: RelativePath; }): Promise { - if (oldPath !== undefined) { - if ( - this.database.getLatestDocumentByRelativePath(oldPath) - ?.isDeleted === true - ) { - this.logger.debug( - `Document ${oldPath} has been deleted locally, skipping` - ); - return; - } - + if ( + oldPath !== undefined && + (this.database.getLatestDocumentByRelativePath(relativePath) === + undefined || + this.database.getLatestDocumentByRelativePath(relativePath) + ?.isDeleted === true) + ) { if (oldPath === relativePath) { throw new Error( `Old path and new path are the same: ${oldPath}` @@ -178,10 +168,17 @@ export class Syncer { this.database.move(oldPath, relativePath); } - if ( - this.database.getLatestDocumentByRelativePath(relativePath) - ?.isDeleted === true - ) { + let document = + this.database.getLatestDocumentByRelativePath(relativePath); + + if (document === undefined) { + this.logger.debug( + `Cannot find document ${relativePath} in the database, skipping` + ); + return; + } + + if (document.isDeleted) { this.logger.debug( `Document ${relativePath} has been deleted locally, skipping` ); @@ -190,7 +187,7 @@ export class Syncer { const [promise, resolve, reject] = createPromise(); - await this.database.getResolvedDocumentByRelativePath( + document = await this.database.getResolvedDocumentByRelativePath( relativePath, promise ); @@ -199,8 +196,7 @@ export class Syncer { await this.syncQueue.add(async () => this.internalSyncer.unrestrictedSyncLocallyUpdatedFile({ oldPath, - getLatestDocument: () => - this.database.getDocumentByUpdatePromise(promise) + document }) ); @@ -299,7 +295,7 @@ export class Syncer { private async syncRemotelyUpdatedFile( remoteVersion: components["schemas"]["DocumentVersionWithoutContent"] ): Promise { - const document = this.database.getDocumentByDocumentId( + let document = this.database.getDocumentByDocumentId( remoteVersion.documentId ); @@ -308,15 +304,11 @@ export class Syncer { if (document === undefined) { await this.syncQueue.add(async () => this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile( - remoteVersion, - () => - this.database.getDocumentByDocumentId( - remoteVersion.documentId - ) + remoteVersion ) ); } else { - await this.database.getResolvedDocumentByRelativePath( + document = await this.database.getResolvedDocumentByRelativePath( document.relativePath, promise ); @@ -325,7 +317,7 @@ export class Syncer { await this.syncQueue.add(async () => this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile( remoteVersion, - () => this.database.getDocumentByUpdatePromise(promise) + document ) ); diff --git a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts index ce3f4e60..01cb665e 100644 --- a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts +++ b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts @@ -1,6 +1,5 @@ import type { Database, - DocumentId, DocumentRecord, RelativePath } from "../persistence/database"; @@ -33,31 +32,24 @@ export class UnrestrictedSyncer { } public async unrestrictedSyncLocallyCreatedFile( - proposedDocumentId: DocumentId, - getLatestDocument: () => DocumentRecord + document: DocumentRecord ): Promise { - let document = getLatestDocument(); - return this.executeSync( - [document.relativePath], + document.relativePath, SyncType.CREATE, SyncSource.PUSH, async () => { - document = getLatestDocument(); - const contentBytes = await this.operations.read( document.relativePath ); // this can throw FileNotFoundError const contentHash = hash(contentBytes); const response = await this.syncService.create({ - documentId: proposedDocumentId, + documentId: document.documentId, relativePath: document.relativePath, contentBytes }); - document = getLatestDocument(); - this.history.addHistoryEntry({ status: SyncStatus.SUCCESS, source: SyncSource.PUSH, @@ -68,12 +60,10 @@ export class UnrestrictedSyncer { this.database.setDocument( { - relativePath: document.relativePath, - documentId: response.documentId, parentVersionId: response.vaultUpdateId, hash: contentHash }, - document.identity + document ); this.tryIncrementVaultUpdateId(response.vaultUpdateId); @@ -82,16 +72,13 @@ export class UnrestrictedSyncer { } public async unrestrictedSyncLocallyDeletedFile( - getLatestDocument: () => DocumentRecord + document: DocumentRecord ): Promise { - let document = getLatestDocument(); await this.executeSync( - [document.relativePath], + document.relativePath, SyncType.DELETE, SyncSource.PUSH, async () => { - document = getLatestDocument(); - const response = await this.syncService.delete({ documentId: document.documentId, relativePath: document.relativePath @@ -105,16 +92,12 @@ export class UnrestrictedSyncer { type: SyncType.DELETE }); - document = getLatestDocument(); - this.database.setDocument( { - relativePath: document.relativePath, - documentId: response.documentId, parentVersionId: response.vaultUpdateId, hash: EMPTY_HASH }, - document.identity + document ); } ); @@ -122,21 +105,16 @@ export class UnrestrictedSyncer { public async unrestrictedSyncLocallyUpdatedFile({ oldPath, - getLatestDocument + document }: { oldPath?: RelativePath; - getLatestDocument: () => DocumentRecord; + document: DocumentRecord; }): Promise { - let document = getLatestDocument(); - await this.executeSync( - [oldPath, document.relativePath].filter( - (path) => path !== undefined - ), + document.relativePath, SyncType.UPDATE, SyncSource.PUSH, async () => { - document = getLatestDocument(); const originalRelativePath = document.relativePath; if (document.metadata === undefined || document.isDeleted) { @@ -168,8 +146,8 @@ export class UnrestrictedSyncer { contentBytes }); - document = getLatestDocument(); - + // `document` is mutable and reflects the latest state in the local database + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition if (document.isDeleted) { this.logger.info( `Document ${document.relativePath} has been deleted before we could finish updating it` @@ -177,7 +155,8 @@ export class UnrestrictedSyncer { return; } - if (!document.metadata) { + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + if (document.metadata === undefined) { throw new Error( `Document ${document.relativePath} no longer has metadata after updating it` ); @@ -213,12 +192,10 @@ export class UnrestrictedSyncer { this.database.delete(document.relativePath); this.database.setDocument( { - documentId: response.documentId, - relativePath: document.relativePath, parentVersionId: response.vaultUpdateId, hash: EMPTY_HASH }, - document.identity + document ); await this.operations.delete(document.relativePath); @@ -231,11 +208,6 @@ export class UnrestrictedSyncer { let actualPath = document.relativePath; if (response.relativePath != originalRelativePath) { - // this.database.getNewResolvedDocumentByRelativePath( - // response.relativePath, - // promise - // ); - actualPath = response.relativePath; await this.operations.move( document.relativePath, @@ -243,6 +215,14 @@ export class UnrestrictedSyncer { ); // this can throw FileNotFoundError } + this.database.setDocument( + { + parentVersionId: response.vaultUpdateId, + hash: contentHash + }, + document + ); + if (response.type === "MergingUpdate") { const responseBytes = deserialize(response.contentBase64); contentHash = hash(responseBytes); @@ -262,16 +242,6 @@ export class UnrestrictedSyncer { }); } - this.database.setDocument( - { - documentId: response.documentId, - relativePath: actualPath, - parentVersionId: response.vaultUpdateId, - hash: contentHash - }, - document.identity - ); - this.tryIncrementVaultUpdateId(response.vaultUpdateId); } ); @@ -279,20 +249,18 @@ export class UnrestrictedSyncer { public async unrestrictedSyncRemotelyUpdatedFile( remoteVersion: components["schemas"]["DocumentVersionWithoutContent"], - getLatestDocument: () => DocumentRecord | undefined + document?: DocumentRecord ): Promise { await this.executeSync( - [remoteVersion.relativePath], + remoteVersion.relativePath, SyncType.UPDATE, SyncSource.PULL, async () => { - let localMetadata = getLatestDocument(); - - if (localMetadata?.metadata !== undefined) { + if (document?.metadata !== undefined) { // If the file exists locally, let's pretend the user has updated it // and deal with remote update/deletion within `unrestrictedSyncLocallyUpdatedFile` if ( - localMetadata.metadata.parentVersionId >= + document.metadata.parentVersionId >= remoteVersion.vaultUpdateId ) { this.logger.debug( @@ -302,11 +270,7 @@ export class UnrestrictedSyncer { } return this.unrestrictedSyncLocallyUpdatedFile({ - getLatestDocument: () => - this.database.getDocumentByIdentity( - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - localMetadata!.identity - ) + document }); } else if (remoteVersion.isDeleted) { // Either the doc hasn't made it to us before and therefore we don't need to delete it, @@ -323,9 +287,11 @@ export class UnrestrictedSyncer { }) ).contentBase64; - localMetadata = getLatestDocument(); + document = this.database.getDocumentByDocumentId( + remoteVersion.documentId + ); - if (localMetadata?.isDeleted === true) { + if (document?.isDeleted === true) { this.logger.info( `Document ${remoteVersion.relativePath} has been deleted locally before we could finish updating it` ); @@ -333,7 +299,7 @@ export class UnrestrictedSyncer { } if ( - (localMetadata?.metadata?.parentVersionId ?? -1) >= + (document?.metadata?.parentVersionId ?? -1) >= remoteVersion.vaultUpdateId ) { this.logger.debug( @@ -344,16 +310,21 @@ export class UnrestrictedSyncer { const contentBytes = deserialize(content); - const [promise, resolve] = createPromise(); - await this.operations.ensureClearPath( remoteVersion.relativePath ); - this.database.getNewResolvedDocumentByRelativePath( - remoteVersion.documentId, - remoteVersion.relativePath, - promise + const [promise, resolve] = createPromise(); + this.database.setDocument( + { + parentVersionId: remoteVersion.vaultUpdateId, + hash: hash(contentBytes) + }, + this.database.createNewPendingDocument( + remoteVersion.documentId, + remoteVersion.relativePath, + promise + ) ); await this.operations.create( @@ -361,17 +332,6 @@ export class UnrestrictedSyncer { contentBytes ); - const document = - this.database.getDocumentByUpdatePromise(promise); - this.database.setDocument( - { - documentId: remoteVersion.documentId, - relativePath: remoteVersion.relativePath, - parentVersionId: remoteVersion.vaultUpdateId, - hash: hash(contentBytes) - }, - document.identity - ); resolve(); this.database.removeDocumentPromise(promise); @@ -387,13 +347,11 @@ export class UnrestrictedSyncer { } public async executeSync( - paths: RelativePath[], + relativePath: RelativePath, syncType: SyncType, syncSource: SyncSource, fn: () => Promise ): Promise { - const relativePath = paths[paths.length - 1]; - if (!this.operations.isFileEligibleForSync(relativePath)) { this.history.addHistoryEntry({ status: SyncStatus.ERROR, diff --git a/frontend/test-client/src/agent/mock-client.ts b/frontend/test-client/src/agent/mock-client.ts index 941b87d1..8aa0141d 100644 --- a/frontend/test-client/src/agent/mock-client.ts +++ b/frontend/test-client/src/agent/mock-client.ts @@ -63,7 +63,11 @@ export class MockClient implements FileSystemOperations { `Creating file ${path} with content ${new TextDecoder().decode(newContent)}` ); this.localFiles.set(path, newContent); - void this.client.syncer.syncLocallyCreatedFile(path); + + // we aren't the best client and it takes some time to notice changes + setImmediate(() => { + void this.client.syncer.syncLocallyCreatedFile(path); + }); } public async createDirectory(_path: RelativePath): Promise { @@ -101,8 +105,11 @@ export class MockClient implements FileSystemOperations { `Updated file ${path} with:\n current content: ${currentContent}\n new content: ${newContent}` ); - void this.client.syncer.syncLocallyUpdatedFile({ - relativePath: path + // we aren't the best client and it takes some time to notice changes + setImmediate(() => { + void this.client.syncer.syncLocallyUpdatedFile({ + relativePath: path + }); }); return newContent; @@ -116,13 +123,16 @@ export class MockClient implements FileSystemOperations { `Updated file ${path} with:\n new content: ${new TextDecoder().decode(content)}` ); - if (hasExisted) { - void this.client.syncer.syncLocallyUpdatedFile({ - relativePath: path - }); - } else { - void this.client.syncer.syncLocallyCreatedFile(path); - } + // we aren't the best client and it takes some time to notice changes + setImmediate(() => { + if (hasExisted) { + void this.client.syncer.syncLocallyUpdatedFile({ + relativePath: path + }); + } else { + void this.client.syncer.syncLocallyCreatedFile(path); + } + }); } public async delete(path: RelativePath): Promise { @@ -130,7 +140,10 @@ export class MockClient implements FileSystemOperations { `Deleting file: ${path} with:\n content ${new TextDecoder().decode(this.localFiles.get(path))}` ); this.localFiles.delete(path); - void this.client.syncer.syncLocallyDeletedFile(path); + // we aren't the best client and it takes some time to notice changes + setImmediate(() => { + void this.client.syncer.syncLocallyDeletedFile(path); + }); } public async rename( @@ -150,9 +163,12 @@ export class MockClient implements FileSystemOperations { `Renamed file: ${oldPath} -> ${newPath} with:\n content ${new TextDecoder().decode(file)}` ); - void this.client.syncer.syncLocallyUpdatedFile({ - oldPath, - relativePath: newPath + // we aren't the best client and it takes some time to notice changes + setImmediate(() => { + void this.client.syncer.syncLocallyUpdatedFile({ + oldPath, + relativePath: newPath + }); }); } } diff --git a/frontend/test-client/src/cli.ts b/frontend/test-client/src/cli.ts index 6db7b042..50543d94 100644 --- a/frontend/test-client/src/cli.ts +++ b/frontend/test-client/src/cli.ts @@ -91,26 +91,24 @@ async function runTest({ async function runTests(): Promise { const agentCounts = [2, 8]; - const jitterScaleInSeconds = [0.5, 0, 2]; - const concurrencies = [16, 1]; - const iterations = [200]; + const networkJitterScaleInSeconds = [0.5, 2]; + const concurrencies = [ + 16, + 1 // test with concurrency 1 to check for deadlocks + ]; const doDeletes = [true, false]; for (const agentCount of agentCounts) { for (const concurrency of concurrencies) { - for (const jitter of jitterScaleInSeconds) { - for (const iteration of iterations) { - for (const deleteFiles of doDeletes) { - for (let i = 0; i < 3; i++) { - await runTest({ - agentCount, - concurrency, - iterations: iteration, - doDeletes: deleteFiles, - jitterScaleInSeconds: jitter - }); - } - } + for (const jitter of networkJitterScaleInSeconds) { + for (const deleteFiles of doDeletes) { + await runTest({ + agentCount, + concurrency, + iterations: 200, + doDeletes: deleteFiles, + jitterScaleInSeconds: jitter + }); } } } From 78e1372483d8a341efd96000d11152fee70239c0 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 15 Mar 2025 18:01:33 +0000 Subject: [PATCH 29/58] Add useSlowFileEvents --- .../sync-operations/unrestricted-syncer.ts | 2 +- frontend/test-client/src/agent/mock-agent.ts | 19 +++++-- frontend/test-client/src/agent/mock-client.ts | 56 +++++++++++-------- frontend/test-client/src/cli.ts | 51 +++++++++++++---- 4 files changed, 87 insertions(+), 41 deletions(-) diff --git a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts index 01cb665e..8300f10b 100644 --- a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts +++ b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts @@ -158,7 +158,7 @@ export class UnrestrictedSyncer { // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition if (document.metadata === undefined) { throw new Error( - `Document ${document.relativePath} no longer has metadata after updating it` + `Document ${document.relativePath} no longer has metadata after updating it, this cannot happen` ); } diff --git a/frontend/test-client/src/agent/mock-agent.ts b/frontend/test-client/src/agent/mock-agent.ts index 94c106da..bbed0ef2 100644 --- a/frontend/test-client/src/agent/mock-agent.ts +++ b/frontend/test-client/src/agent/mock-agent.ts @@ -18,9 +18,10 @@ export class MockAgent extends MockClient { initialSettings: Partial, public readonly name: string, private readonly doDeletes: boolean, + useSlowFileEvents: boolean, private readonly jitterScaleInSeconds: number ) { - super(initialSettings); + super(initialSettings, useSlowFileEvents); } public async init(): Promise { @@ -62,9 +63,11 @@ export class MockAgent extends MockClient { case LogLevel.ERROR: console.error(formatted); - // Let's not ignore errors - // eslint-disable-next-line @typescript-eslint/no-floating-promises - sleep(100).then(() => process.exit(1)); + if (!this.useSlowFileEvents) { + // Let's not ignore errors + // eslint-disable-next-line @typescript-eslint/no-floating-promises + sleep(100).then(() => process.exit(1)); + } break; case LogLevel.WARNING: @@ -189,6 +192,14 @@ export class MockAgent extends MockClient { } public assertAllContentIsPresentOnce(): void { + if (this.useSlowFileEvents) { + this.client.logger.info( + // We can't ensure that we have seen every single update + `Skipping content check for ${this.name} because slow file events are enabled` + ); + return; + } + for (const content of this.writtenContents) { const found = Array.from(this.localFiles.keys()).filter((key) => { return new TextDecoder() diff --git a/frontend/test-client/src/agent/mock-client.ts b/frontend/test-client/src/agent/mock-client.ts index 8aa0141d..7e4e14c3 100644 --- a/frontend/test-client/src/agent/mock-client.ts +++ b/frontend/test-client/src/agent/mock-client.ts @@ -12,7 +12,8 @@ export class MockClient implements FileSystemOperations { protected data: object | undefined = undefined; public constructor( - private readonly initialSettings: Partial + private readonly initialSettings: Partial, + protected readonly useSlowFileEvents: boolean ) {} public async init(): Promise { @@ -64,8 +65,7 @@ export class MockClient implements FileSystemOperations { ); this.localFiles.set(path, newContent); - // we aren't the best client and it takes some time to notice changes - setImmediate(() => { + this.runCallback(() => { void this.client.syncer.syncLocallyCreatedFile(path); }); } @@ -87,26 +87,27 @@ export class MockClient implements FileSystemOperations { const newContentUint8Array = new TextEncoder().encode(newContent); this.localFiles.set(path, newContentUint8Array); - const existingParts = currentContent - .split(" ") - .map((part) => part.trim()); - const newParts = newContent.split(" ").map((part) => part.trim()); - existingParts.forEach((part) => - // all changes should be additive - { - assert( - newParts.includes(part), - `Part ${part} not found in new content` - ); - } - ); + if (!this.useSlowFileEvents) { + const existingParts = currentContent + .split(" ") + .map((part) => part.trim()); + const newParts = newContent.split(" ").map((part) => part.trim()); + existingParts.forEach((part) => + // all changes should be additive + { + assert( + newParts.includes(part), + `Part ${part} not found in new content` + ); + } + ); + } this.client.logger.info( `Updated file ${path} with:\n current content: ${currentContent}\n new content: ${newContent}` ); - // we aren't the best client and it takes some time to notice changes - setImmediate(() => { + this.runCallback(() => { void this.client.syncer.syncLocallyUpdatedFile({ relativePath: path }); @@ -123,8 +124,7 @@ export class MockClient implements FileSystemOperations { `Updated file ${path} with:\n new content: ${new TextDecoder().decode(content)}` ); - // we aren't the best client and it takes some time to notice changes - setImmediate(() => { + this.runCallback(() => { if (hasExisted) { void this.client.syncer.syncLocallyUpdatedFile({ relativePath: path @@ -140,8 +140,8 @@ export class MockClient implements FileSystemOperations { `Deleting file: ${path} with:\n content ${new TextDecoder().decode(this.localFiles.get(path))}` ); this.localFiles.delete(path); - // we aren't the best client and it takes some time to notice changes - setImmediate(() => { + + this.runCallback(() => { void this.client.syncer.syncLocallyDeletedFile(path); }); } @@ -163,12 +163,20 @@ export class MockClient implements FileSystemOperations { `Renamed file: ${oldPath} -> ${newPath} with:\n content ${new TextDecoder().decode(file)}` ); - // we aren't the best client and it takes some time to notice changes - setImmediate(() => { + this.runCallback(() => { void this.client.syncer.syncLocallyUpdatedFile({ oldPath, relativePath: newPath }); }); } + + private runCallback(callback: () => void): void { + if (this.useSlowFileEvents) { + // we aren't the best client and it takes some time to notice changes + setTimeout(callback, 100); + } else { + callback(); + } + } } diff --git a/frontend/test-client/src/cli.ts b/frontend/test-client/src/cli.ts index 50543d94..02e86237 100644 --- a/frontend/test-client/src/cli.ts +++ b/frontend/test-client/src/cli.ts @@ -3,20 +3,26 @@ import { MockAgent } from "./agent/mock-agent"; import { sleep } from "./utils/sleep"; import { v4 as uuidv4 } from "uuid"; +let slowFileEvents = false; + async function runTest({ agentCount, concurrency, iterations, doDeletes, + useSlowFileEvents, jitterScaleInSeconds }: { agentCount: number; concurrency: number; iterations: number; doDeletes: boolean; + useSlowFileEvents: boolean; jitterScaleInSeconds: number; }): Promise { - const settings = `with ${agentCount} agents, concurrency ${concurrency}, iterations ${iterations}, doDeletes ${doDeletes}, jitterScaleInSeconds ${jitterScaleInSeconds}`; + slowFileEvents = useSlowFileEvents; + + const settings = `with ${agentCount} agents, concurrency ${concurrency}, iterations ${iterations}, doDeletes ${doDeletes}, jitterScaleInSeconds ${jitterScaleInSeconds}, useSlowFileEvents ${useSlowFileEvents}`; console.info(`Running test ${settings}`); const initialSettings: Partial = { @@ -34,6 +40,7 @@ async function runTest({ initialSettings, `agent-${i}`, doDeletes, + useSlowFileEvents, jitterScaleInSeconds ) ); @@ -56,12 +63,24 @@ async function runTest({ // Each agent can have unpushed changes which might conflict with eachother so each has to resolve the conflicts & push, and for (const client of clients) { - await client.finish(); + try { + await client.finish(); + } catch (err) { + if (!slowFileEvents) { + throw err; + } + } } // then we need a second pass to ensure that all agents pull the same state. for (const client of clients) { - await client.finish(); + try { + await client.finish(); + } catch (err) { + if (!slowFileEvents) { + throw err; + } + } } console.info("Agents finished successfully"); @@ -96,19 +115,21 @@ async function runTests(): Promise { 16, 1 // test with concurrency 1 to check for deadlocks ]; - const doDeletes = [true, false]; for (const agentCount of agentCounts) { for (const concurrency of concurrencies) { for (const jitter of networkJitterScaleInSeconds) { - for (const deleteFiles of doDeletes) { - await runTest({ - agentCount, - concurrency, - iterations: 200, - doDeletes: deleteFiles, - jitterScaleInSeconds: jitter - }); + for (const doDeletes of [true, false]) { + for (const useSlowFileEvents of [true, false]) { + await runTest({ + agentCount, + concurrency, + iterations: 200, + doDeletes, + useSlowFileEvents, + jitterScaleInSeconds: jitter + }); + } } } } @@ -116,11 +137,17 @@ async function runTests(): Promise { } process.on("uncaughtException", (error) => { + if (slowFileEvents) { + return; + } console.error("Uncaught Exception:", error); process.exit(1); }); process.on("unhandledRejection", (reason, _promise) => { + if (slowFileEvents) { + return; + } console.error("Unhandled Rejection:", reason); process.exit(1); }); From 3649f335fe9d5cb9d25f96030ff65aee2fb89197 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 16 Mar 2025 10:32:29 +0000 Subject: [PATCH 30/58] Delete fuzz --- backend/Cargo.toml | 1 - backend/fuzz/.gitignore | 4 ---- backend/fuzz/Cargo.toml | 25 ------------------------- backend/fuzz/fuzz_targets/reconcile.rs | 8 -------- 4 files changed, 38 deletions(-) delete mode 100644 backend/fuzz/.gitignore delete mode 100644 backend/fuzz/Cargo.toml delete mode 100644 backend/fuzz/fuzz_targets/reconcile.rs diff --git a/backend/Cargo.toml b/backend/Cargo.toml index cfb865a0..c405cf52 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -2,7 +2,6 @@ resolver = "2" members = [ "reconcile", - "fuzz", "sync_server", "sync_lib" ] diff --git a/backend/fuzz/.gitignore b/backend/fuzz/.gitignore deleted file mode 100644 index 1a45eee7..00000000 --- a/backend/fuzz/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -target -corpus -artifacts -coverage diff --git a/backend/fuzz/Cargo.toml b/backend/fuzz/Cargo.toml deleted file mode 100644 index d764ba40..00000000 --- a/backend/fuzz/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -[package] -name = "reconcile-fuzz" -version.workspace = true -edition.workspace = true -authors.workspace = true -license.workspace = true -repository.workspace = true -publish = false - -[package.metadata] -cargo-fuzz = true - -[dependencies] -libfuzzer-sys = "0.4" -reconcile = { path = "../reconcile" } - -[[bin]] -name = "reconcile" -path = "fuzz_targets/reconcile.rs" -test = false -doc = false -bench = false - -[lints] -workspace = true diff --git a/backend/fuzz/fuzz_targets/reconcile.rs b/backend/fuzz/fuzz_targets/reconcile.rs deleted file mode 100644 index b30d9f57..00000000 --- a/backend/fuzz/fuzz_targets/reconcile.rs +++ /dev/null @@ -1,8 +0,0 @@ -#![no_main] - -use libfuzzer_sys::fuzz_target; - -fuzz_target!(|texts: (String, String, String)| { - let (original, left, right) = texts; - let _ = reconcile::reconcile(&original, &left, &right); -}); From 031628cff9d6a6459c20c43d05a8f8679f4a7888 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 16 Mar 2025 10:32:43 +0000 Subject: [PATCH 31/58] Fix CI --- .github/workflows/check.yml | 9 -------- .github/workflows/e2e.yml | 39 +++++++++++++++++++++++++++++++++++ backend/Cargo.lock | 35 ------------------------------- backend/sync_lib/tests/web.rs | 5 +++-- 4 files changed, 42 insertions(+), 46 deletions(-) create mode 100644 .github/workflows/e2e.yml diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index ab601843..f6c010bf 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -59,12 +59,3 @@ jobs: run: | cd frontend npm run test - - - name: E2E tests - run: | - cd ../backend - UST_BACKTRACE=1 cargo run -p sync_server & - cd - - - npm run build - test-client/run.sh 32 diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 00000000..4c9f5585 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,39 @@ +name: E2E tests + +on: + push: + branches: ["master"] + pull_request: + branches: ["master"] + +env: + CARGO_TERM_COLOR: always + RUSTFLAGS: "-Dwarnings" + +jobs: + build: + runs-on: self-hosted + + steps: + - uses: actions/checkout@v4 + + - name: Setup + run: | + cargo install sqlx-cli + cd backend + sqlx database create --database-url sqlite://db.sqlite3 + sqlx migrate run --source sync_server/src/database/migrations --database-url sqlite://db.sqlite3 + + - name: Build wasm + run: | + cd backend + cargo install wasm-pack + wasm-pack build --target web sync_lib + + - name: E2E tests + run: | + UST_BACKTRACE=1 cargo run -p sync_server & + cd ../frontend + + npm run build + test-client/run.sh 32 diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 3cce7cfc..8eddadf5 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -105,12 +105,6 @@ dependencies = [ "backtrace", ] -[[package]] -name = "arbitrary" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" - [[package]] name = "async-trait" version = "0.1.85" @@ -381,8 +375,6 @@ version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc" dependencies = [ - "jobserver", - "libc", "shlex", ] @@ -1228,15 +1220,6 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" -[[package]] -name = "jobserver" -version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" -dependencies = [ - "libc", -] - [[package]] name = "js-sys" version = "0.3.76" @@ -1290,16 +1273,6 @@ version = "0.2.167" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc" -[[package]] -name = "libfuzzer-sys" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b9569d2f74e257076d8c6bfa73fb505b46b851e51ddaecc825944aa3bed17fa" -dependencies = [ - "arbitrary", - "cc", -] - [[package]] name = "libm" version = "0.2.11" @@ -1781,14 +1754,6 @@ dependencies = [ "test-case", ] -[[package]] -name = "reconcile-fuzz" -version = "0.0.30" -dependencies = [ - "libfuzzer-sys", - "reconcile", -] - [[package]] name = "redox_syscall" version = "0.5.7" diff --git a/backend/sync_lib/tests/web.rs b/backend/sync_lib/tests/web.rs index ffea18d9..eca585b6 100644 --- a/backend/sync_lib/tests/web.rs +++ b/backend/sync_lib/tests/web.rs @@ -25,8 +25,9 @@ fn test_base64_to_bytes_error() { #[wasm_bindgen_test(unsupported = test)] fn merge_text() { let left = b"hello "; - let right = b"world"; - assert_eq!(merge(b"", left, right), b"hello world".to_vec()); + let right = b"world "; + let result = merge(b"", left, right); + assert!(result == b"hello world ".to_vec() || result == b"world hello ".to_vec()); } #[wasm_bindgen_test(unsupported = test)] From fb15e4839196b1760f91aa3238be08513d3be14b Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 16 Mar 2025 10:48:21 +0000 Subject: [PATCH 32/58] use docker --- .github/workflows/publish-docker.yml | 3 ++- .github/workflows/publish-plugin.yml | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index 993c03db..5d22ee89 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -18,7 +18,8 @@ env: jobs: build-docker: - runs-on: ubuntu-latest + runs-on: self-hosted + permissions: contents: read packages: write diff --git a/.github/workflows/publish-plugin.yml b/.github/workflows/publish-plugin.yml index c21d986e..4896fa8f 100644 --- a/.github/workflows/publish-plugin.yml +++ b/.github/workflows/publish-plugin.yml @@ -2,14 +2,13 @@ name: Publish Obsidian plugin on: push: - tags: - - "*" + tags: ["*"] env: CARGO_TERM_COLOR: always jobs: - build-plugin: + publish-plugin: runs-on: self-hosted steps: From 111134d7e69303303480f8efdb5086c2e1b351d2 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 16 Mar 2025 10:48:26 +0000 Subject: [PATCH 33/58] fix script --- frontend/test-client/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/test-client/run.sh b/frontend/test-client/run.sh index f559db2f..4a32c002 100755 --- a/frontend/test-client/run.sh +++ b/frontend/test-client/run.sh @@ -18,7 +18,7 @@ mkdir -p logs pids=() for i in $(seq 1 $process_count); do - node dist/cli.js 2>&1 > "logs/log_${i}.log" & + node dist/cli.js > "logs/log_${i}.log" 2>&1 & pids+=($!) done From bb2ff23a4a57b634848f113c8d2f52f7ddaa7f71 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 16 Mar 2025 10:48:34 +0000 Subject: [PATCH 34/58] clean up --- frontend/test-client/src/agent/mock-agent.ts | 2 +- frontend/test-client/src/cli.ts | 32 ++++++++------------ 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/frontend/test-client/src/agent/mock-agent.ts b/frontend/test-client/src/agent/mock-agent.ts index bbed0ef2..2dc4ec99 100644 --- a/frontend/test-client/src/agent/mock-agent.ts +++ b/frontend/test-client/src/agent/mock-agent.ts @@ -229,7 +229,7 @@ export class MockAgent extends MockClient { ); assert( fileContent.split(content).length == 2, - `Content ${content} (of ${this.name}) found more than once in file ${file}. File content:\n${fileContent}` + `Content ${content} (of ${this.name}) found more than once in '${file}'. File content:\n${fileContent}` ); } } diff --git a/frontend/test-client/src/cli.ts b/frontend/test-client/src/cli.ts index 02e86237..85fb3c43 100644 --- a/frontend/test-client/src/cli.ts +++ b/frontend/test-client/src/cli.ts @@ -109,28 +109,20 @@ async function runTest({ } async function runTests(): Promise { - const agentCounts = [2, 8]; - const networkJitterScaleInSeconds = [0.5, 2]; - const concurrencies = [ + for (const concurrency of [ 16, 1 // test with concurrency 1 to check for deadlocks - ]; - - for (const agentCount of agentCounts) { - for (const concurrency of concurrencies) { - for (const jitter of networkJitterScaleInSeconds) { - for (const doDeletes of [true, false]) { - for (const useSlowFileEvents of [true, false]) { - await runTest({ - agentCount, - concurrency, - iterations: 200, - doDeletes, - useSlowFileEvents, - jitterScaleInSeconds: jitter - }); - } - } + ]) { + for (const doDeletes of [true, false]) { + for (const useSlowFileEvents of [true, false]) { + await runTest({ + agentCount: 4, + concurrency, + iterations: 200, + doDeletes, + useSlowFileEvents, + jitterScaleInSeconds: 0.75 + }); } } } From 61e4f431316e5526ed35788a7d69eddb0d81ffcd Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 16 Mar 2025 11:07:39 +0000 Subject: [PATCH 35/58] Clean up --- .github/workflows/publish-docker.yml | 2 +- .../sync-client/src/persistence/database.ts | 18 ++++++++++-------- .../src/sync-operations/unrestricted-syncer.ts | 10 +++++----- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index 5d22ee89..638e3f36 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -17,7 +17,7 @@ env: IMAGE_NAME: ${{ github.repository }} jobs: - build-docker: + publish-docker: runs-on: self-hosted permissions: diff --git a/frontend/sync-client/src/persistence/database.ts b/frontend/sync-client/src/persistence/database.ts index c4ff6d82..5011fc85 100644 --- a/frontend/sync-client/src/persistence/database.ts +++ b/frontend/sync-client/src/persistence/database.ts @@ -21,6 +21,12 @@ export interface StoredDatabase { lastSeenUpdateId: VaultUpdateId | undefined; } +/** + * Represents a document in the database. + * + * It is mutable and its content should always represent the latest + * state of the document on disk based on the update events we have seen. + */ export interface DocumentRecord { relativePath: RelativePath; documentId: DocumentId; @@ -114,11 +120,8 @@ export class Database { this.save(); } - public setDocument( - { - parentVersionId, - hash - }: { + public updateDocumentMetadata( + metadata: { parentVersionId: VaultUpdateId; hash: string; }, @@ -128,10 +131,9 @@ export class Database { throw new Error("Document not found in database"); } - toUpdate.metadata = { parentVersionId, hash }; + toUpdate.metadata = metadata; this.save(); - return; } public removeDocumentPromise(promise: Promise): void { @@ -225,7 +227,7 @@ export class Database { const newDocument = this.getLatestDocumentByRelativePath(newRelativePath); - if (newDocument !== undefined && !newDocument.isDeleted) { + if (newDocument?.isDeleted === false) { throw new Error( `Document already exists at new location: ${newRelativePath}` ); diff --git a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts index 8300f10b..e38af28c 100644 --- a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts +++ b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts @@ -58,7 +58,7 @@ export class UnrestrictedSyncer { type: SyncType.CREATE }); - this.database.setDocument( + this.database.updateDocumentMetadata( { parentVersionId: response.vaultUpdateId, hash: contentHash @@ -92,7 +92,7 @@ export class UnrestrictedSyncer { type: SyncType.DELETE }); - this.database.setDocument( + this.database.updateDocumentMetadata( { parentVersionId: response.vaultUpdateId, hash: EMPTY_HASH @@ -190,7 +190,7 @@ export class UnrestrictedSyncer { }); this.database.delete(document.relativePath); - this.database.setDocument( + this.database.updateDocumentMetadata( { parentVersionId: response.vaultUpdateId, hash: EMPTY_HASH @@ -215,7 +215,7 @@ export class UnrestrictedSyncer { ); // this can throw FileNotFoundError } - this.database.setDocument( + this.database.updateDocumentMetadata( { parentVersionId: response.vaultUpdateId, hash: contentHash @@ -315,7 +315,7 @@ export class UnrestrictedSyncer { ); const [promise, resolve] = createPromise(); - this.database.setDocument( + this.database.updateDocumentMetadata( { parentVersionId: remoteVersion.vaultUpdateId, hash: hash(contentBytes) From 172776fdbda72d9396d0ee1977541810530376b8 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 16 Mar 2025 11:32:38 +0000 Subject: [PATCH 36/58] change node version --- .github/workflows/check.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index f6c010bf..2c9cb4af 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -41,6 +41,8 @@ jobs: cd backend cargo test --verbose cd sync_lib + nvm install 22 + nvm use 22 wasm-pack test --node - name: Lint frontend From 3eaf52549d714018d3540aebacd6bbe1f238d00b Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 16 Mar 2025 11:35:24 +0000 Subject: [PATCH 37/58] Build docker image on every commit --- .github/workflows/publish-docker.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index 638e3f36..14516a66 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -7,8 +7,9 @@ name: Publish server Docker image on: push: - tags: - - "*" + branches: ["master"] + pull_request: + branches: ["master"] env: # Use docker.io for Docker Hub if empty @@ -34,7 +35,7 @@ jobs: # Install the cosign tool except on PR # https://github.com/sigstore/cosign-installer - name: Install cosign - if: github.event_name != 'pull_request' + if: ${{ github.ref_type == 'tag' }} uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 #v3.5.0 with: cosign-release: "v2.2.4" @@ -48,7 +49,7 @@ jobs: # Login against a Docker registry except on PR # https://github.com/docker/login-action - name: Log into registry ${{ env.REGISTRY }} - if: github.event_name != 'pull_request' + if: ${{ github.ref_type == 'tag' }} uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 with: registry: ${{ env.REGISTRY }} @@ -70,7 +71,7 @@ jobs: uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 with: context: backend - push: ${{ github.event_name != 'pull_request' }} + push: ${{ github.ref_type == 'tag' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha @@ -82,7 +83,7 @@ jobs: # transparency data even for private images, pass --force to cosign below. # https://github.com/sigstore/cosign - name: Sign the published Docker image - if: ${{ github.event_name != 'pull_request' }} + if: ${{ github.ref_type == 'tag' }} env: # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable TAGS: ${{ steps.meta.outputs.tags }} From c62957087f000e379049cd39dc9bbedfd671fd76 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 16 Mar 2025 12:23:18 +0000 Subject: [PATCH 38/58] fix ci --- .github/workflows/check.yml | 6 ++++++ .github/workflows/e2e.yml | 1 - .github/workflows/publish-plugin.yml | 7 ++++--- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 2c9cb4af..dd5841ed 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -36,6 +36,12 @@ jobs: cargo clippy --all-targets --all-features cargo fmt --all -- --check + - name: Setup Node.js environment + uses: actions/setup-node@v4.2.0 + with: + node-version: "22.x" + check-latest: true + - name: Test backend run: | cd backend diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 4c9f5585..f6561501 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -26,7 +26,6 @@ jobs: - name: Build wasm run: | - cd backend cargo install wasm-pack wasm-pack build --target web sync_lib diff --git a/.github/workflows/publish-plugin.yml b/.github/workflows/publish-plugin.yml index 4896fa8f..d8c4b468 100644 --- a/.github/workflows/publish-plugin.yml +++ b/.github/workflows/publish-plugin.yml @@ -14,10 +14,11 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Use Node.js - uses: actions/setup-node@v3 + - name: Setup Node.js environment + uses: actions/setup-node@v4.2.0 with: - node-version: "18.x" + node-version: "22.x" + check-latest: true - name: Build wasm run: | From c49ee759ac3f1609528c5761f35bcc72e7f01f8f Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 16 Mar 2025 14:51:58 +0000 Subject: [PATCH 39/58] 1 db per vault --- .gitignore | 1 + backend/Dockerfile | 2 +- .../sync_server/src/config/database_config.rs | 18 +- .../sync_server/src/config/server_config.rs | 9 +- backend/sync_server/src/consts.rs | 2 +- backend/sync_server/src/database.rs | 161 ++++++++++++------ .../migrations/20241207143519_bootstrap.sql | 15 +- backend/sync_server/src/database/models.rs | 9 +- .../sync_server/src/server/create_document.rs | 7 +- .../sync_server/src/server/delete_document.rs | 7 +- .../src/server/fetch_document_version.rs | 2 +- .../server/fetch_document_version_content.rs | 2 +- .../server/fetch_latest_document_version.rs | 2 +- .../src/server/fetch_latest_documents.rs | 2 +- .../sync_server/src/server/update_document.rs | 7 +- clean-up.sh | 4 + 16 files changed, 151 insertions(+), 99 deletions(-) create mode 100644 clean-up.sh diff --git a/.gitignore b/.gitignore index 41188af7..d2a83679 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ backend/target frontend/*/dist backend/db.sqlite3* +backend/databases backend/config.yml *.log diff --git a/backend/Dockerfile b/backend/Dockerfile index c26c125d..24388c7f 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -21,7 +21,7 @@ RUN apk add --no-cache curl COPY --from=builder /usr/src/backend/target/x86_64-unknown-linux-musl/release/sync_server /app/sync_server -VOLUME /data +VOLUME /data/databases EXPOSE 3000/tcp WORKDIR /data diff --git a/backend/sync_server/src/config/database_config.rs b/backend/sync_server/src/config/database_config.rs index effcfde6..b3d2fad7 100644 --- a/backend/sync_server/src/config/database_config.rs +++ b/backend/sync_server/src/config/database_config.rs @@ -1,31 +1,33 @@ +use std::path::PathBuf; + use log::debug; use serde::{Deserialize, Serialize}; -use crate::consts::{DEFAULT_MAX_CONNECTIONS, DEFAULT_SQLITE_URL}; +use crate::consts::{DEFAULT_DATABASES_DIRECTORY_PATH, DEFAULT_MAX_CONNECTIONS}; #[derive(Debug, Deserialize, Serialize, Clone)] pub struct DatabaseConfig { - #[serde(default = "default_sqlite_url")] - pub sqlite_url: String, + #[serde(default = "default_databases_directory_path")] + pub databases_directory_path: PathBuf, #[serde(default = "default_max_connections")] pub max_connections: u32, } -fn default_sqlite_url() -> String { - debug!("Using default sqlite url: {}", DEFAULT_SQLITE_URL); - DEFAULT_SQLITE_URL.to_owned() +fn default_databases_directory_path() -> PathBuf { + debug!("Using default databases directory path: {DEFAULT_DATABASES_DIRECTORY_PATH:?}"); + PathBuf::from(DEFAULT_DATABASES_DIRECTORY_PATH) } fn default_max_connections() -> u32 { - debug!("Using default max connections: {}", DEFAULT_MAX_CONNECTIONS); + debug!("Using default max connections: {DEFAULT_MAX_CONNECTIONS}"); DEFAULT_MAX_CONNECTIONS } impl Default for DatabaseConfig { fn default() -> Self { Self { - sqlite_url: default_sqlite_url(), + databases_directory_path: default_databases_directory_path(), max_connections: default_max_connections(), } } diff --git a/backend/sync_server/src/config/server_config.rs b/backend/sync_server/src/config/server_config.rs index 88b1f480..8d7c63ea 100644 --- a/backend/sync_server/src/config/server_config.rs +++ b/backend/sync_server/src/config/server_config.rs @@ -15,20 +15,17 @@ pub struct ServerConfig { } fn default_host() -> String { - debug!("Using default server host: {}", DEFAULT_HOST); + debug!("Using default server host: {DEFAULT_HOST}"); DEFAULT_HOST.to_owned() } fn default_port() -> u16 { - debug!("Using default server port: {}", DEFAULT_PORT); + debug!("Using default server port: {DEFAULT_PORT}"); DEFAULT_PORT } fn default_max_body_size_mb() -> usize { - debug!( - "Using default max body size (MB): {}", - DEFAULT_MAX_BODY_SIZE_MB - ); + debug!("Using default max body size (MB): {DEFAULT_MAX_BODY_SIZE_MB}"); DEFAULT_MAX_BODY_SIZE_MB } diff --git a/backend/sync_server/src/consts.rs b/backend/sync_server/src/consts.rs index 2b727f5a..f38012de 100644 --- a/backend/sync_server/src/consts.rs +++ b/backend/sync_server/src/consts.rs @@ -1,5 +1,5 @@ pub const CONFIG_PATH: &str = "config.yml"; -pub const DEFAULT_SQLITE_URL: &str = "db.sqlite3"; +pub const DEFAULT_DATABASES_DIRECTORY_PATH: &str = "databases"; pub const DEFAULT_HOST: &str = "127.0.0.1"; pub const DEFAULT_PORT: u16 = 3000; pub const DEFAULT_MAX_CONNECTIONS: u32 = 12; diff --git a/backend/sync_server/src/database.rs b/backend/sync_server/src/database.rs index e31fc02d..a2a7499e 100644 --- a/backend/sync_server/src/database.rs +++ b/backend/sync_server/src/database.rs @@ -1,4 +1,5 @@ -use core::{str::FromStr as _, time::Duration}; +use core::time::Duration; +use std::{collections::HashMap, sync::Arc}; use anyhow::{Context as _, Result}; use models::{ @@ -7,20 +8,68 @@ use models::{ use sqlx::{sqlite::SqliteConnectOptions, types::chrono::Utc}; pub mod models; use sqlx::{Pool, Sqlite, sqlite::SqlitePoolOptions}; +use tokio::sync::Mutex; use uuid::fmt::Hyphenated; use crate::config::database_config::DatabaseConfig; #[derive(Clone, Debug)] pub struct Database { - connection_pool: Pool, + config: DatabaseConfig, + connection_pools: Arc>>>, } pub type Transaction<'a> = sqlx::Transaction<'a, Sqlite>; impl Database { pub async fn try_new(config: &DatabaseConfig) -> Result { - let connection_options = SqliteConnectOptions::from_str(&config.sqlite_url)? + // Create the databases directory if it doesn't exist + tokio::fs::create_dir_all(&config.databases_directory_path) + .await + .with_context(|| { + format!( + "Failed to create databases directory: {}", + config.databases_directory_path.to_string_lossy() + ) + })?; + + let mut connection_pools = std::collections::HashMap::new(); + + let mut entries = tokio::fs::read_dir(&config.databases_directory_path).await?; + while let Some(entry) = entries.next_entry().await? { + if !entry.file_name().to_string_lossy().ends_with(".sqlite") { + continue; + } + + let vault: VaultId = entry + .file_name() + .to_string_lossy() + .trim_end_matches(".sqlite") + .to_owned(); + + connection_pools.insert( + vault.clone(), + Self::create_vault_database(config, &vault).await?, + ); + } + + Ok(Self { + config: config.clone(), + connection_pools: Arc::new(Mutex::new(connection_pools)), + }) + } + + async fn create_vault_database( + config: &DatabaseConfig, + vault: &VaultId, + ) -> Result> { + let file_name = config + .databases_directory_path + .join(format!("{vault}.sqlite")); + + // Continue with database connection setup + let connection_options = SqliteConnectOptions::new() + .filename(file_name.clone()) .create_if_missing(true) .busy_timeout(Duration::from_secs(3600)) .journal_mode(sqlx::sqlite::SqliteJournalMode::Wal); @@ -30,18 +79,11 @@ impl Database { .test_before_acquire(true) .connect_with(connection_options) .await - .with_context(|| { - format!( - "Cannot connect to database with url: {}", - &config.sqlite_url - ) - })?; + .with_context(|| format!("Cannot open database at '{file_name:?}'"))?; Self::run_migrations(&pool).await?; - Ok(Self { - connection_pool: pool, - }) + Ok(pool) } async fn run_migrations(pool: &Pool) -> Result<()> { @@ -51,17 +93,38 @@ impl Database { .context("Cannot check for pending migrations") } + async fn get_connection_pool(&mut self, vault: &VaultId) -> Result> { + let mut pools = self.connection_pools.lock().await; + if !pools.contains_key(vault) { + let pool = Self::create_vault_database(&self.config, vault).await?; + pools.insert(vault.clone(), pool); + } + + let pool = pools + .get(vault) + .expect("Pool was just inserted or already exists"); + + Ok(pool.clone()) + } + /// Attempting to write from this transaction might result in a /// database locked error. Use this transaction for read-only operations. - pub async fn create_readonly_transaction(&self) -> Result> { - self.connection_pool + pub async fn create_readonly_transaction( + &mut self, + vault: &VaultId, + ) -> Result> { + self.get_connection_pool(vault) + .await? .begin() .await .context("Cannot create transaction") } - pub async fn create_write_transaction(&self) -> Result> { - let mut transaction = self.create_readonly_transaction().await?; + pub async fn create_write_transaction( + &mut self, + vault: &VaultId, + ) -> Result> { + let mut transaction = self.create_readonly_transaction(vault).await?; // sqlx doesn't support immediate transactions for sqlite: https://github.com/launchbadge/sqlx/issues/481 sqlx::query!("END; BEGIN IMMEDIATE;") @@ -73,7 +136,7 @@ impl Database { /// Return the latest state of all documents in the vault pub async fn get_latest_documents( - &self, + &mut self, vault: &VaultId, transaction: Option<&mut Transaction<'_>>, ) -> Result> { @@ -81,23 +144,22 @@ impl Database { DocumentVersionWithoutContent, r#" select - vault_id, vault_update_id, document_id as "document_id: Hyphenated", relative_path, updated_date as "updated_date: chrono::DateTime", is_deleted from latest_document_versions - where vault_id = ? order by vault_update_id desc "#, - vault, ); if let Some(transaction) = transaction { query.fetch_all(&mut **transaction).await } else { - query.fetch_all(&self.connection_pool).await + query + .fetch_all(&self.get_connection_pool(vault).await?) + .await } .context("Cannot fetch latest documents") } @@ -105,7 +167,7 @@ impl Database { /// Return the latest state of all documents (including deleted) in the /// vault which have changed since the given update id pub async fn get_latest_documents_since( - &self, + &mut self, vault: &VaultId, vault_update_id: VaultUpdateId, transaction: Option<&mut Transaction<'_>>, @@ -114,24 +176,24 @@ impl Database { DocumentVersionWithoutContent, r#" select - vault_id, vault_update_id, document_id as "document_id: Hyphenated", relative_path, updated_date as "updated_date: chrono::DateTime", is_deleted from latest_document_versions - where vault_id = ? and vault_update_id > ? + where vault_update_id > ? order by vault_update_id desc "#, - vault, vault_update_id ); if let Some(transaction) = transaction { query.fetch_all(&mut **transaction).await } else { - query.fetch_all(&self.connection_pool).await + query + .fetch_all(&self.get_connection_pool(vault).await?) + .await } .with_context(|| { format!("Cannot fetch latest documents since vault_update_id {vault_update_id}") @@ -139,7 +201,7 @@ impl Database { } pub async fn get_max_update_id_in_vault( - &self, + &mut self, vault: &VaultId, transaction: Option<&mut Transaction<'_>>, ) -> Result { @@ -147,22 +209,22 @@ impl Database { r#" select coalesce(max(vault_update_id), 0) as max_vault_update_id from documents - where vault_id = ? "#, - vault ); if let Some(transaction) = transaction { query.fetch_one(&mut **transaction).await } else { - query.fetch_one(&self.connection_pool).await + query + .fetch_one(&self.get_connection_pool(vault).await?) + .await } .map(|row| row.max_vault_update_id) .context("Cannot fetch max update id in vault") } pub async fn get_latest_document_by_path( - &self, + &mut self, vault: &VaultId, relative_path: &str, transaction: Option<&mut Transaction<'_>>, @@ -171,7 +233,6 @@ impl Database { StoredDocumentVersion, r#" select - vault_id, vault_update_id, document_id as "document_id: Hyphenated", relative_path, @@ -179,26 +240,27 @@ impl Database { content, is_deleted from latest_document_versions - where vault_id = ? and relative_path = ? + where relative_path = ? order by vault_update_id desc -- `latest_document_versions` only contains a single latest version of each document, however, -- multiple documents can have the same `relative_path`, if they have been deleted. That's -- why we only care about the latest version of the document with the given relative path. limit 1 "#, - vault, relative_path ); if let Some(transaction) = transaction { query.fetch_optional(&mut **transaction).await } else { - query.fetch_optional(&self.connection_pool).await + query + .fetch_optional(&self.get_connection_pool(vault).await?) + .await } .context("Cannot fetch latest document version") } pub async fn get_latest_document( - &self, + &mut self, vault: &VaultId, document_id: &DocumentId, transaction: Option<&mut Transaction<'_>>, @@ -208,7 +270,6 @@ impl Database { StoredDocumentVersion, r#" select - vault_id, vault_update_id, document_id as "document_id: Hyphenated", relative_path, @@ -216,22 +277,23 @@ impl Database { content, is_deleted from latest_document_versions - where vault_id = ? and document_id = ? + where document_id = ? "#, - vault, document_id ); if let Some(transaction) = transaction { query.fetch_optional(&mut **transaction).await } else { - query.fetch_optional(&self.connection_pool).await + query + .fetch_optional(&self.get_connection_pool(vault).await?) + .await } .context("Cannot fetch latest document version") } pub async fn get_document_version( - &self, + &mut self, vault: &VaultId, vault_update_id: VaultUpdateId, transaction: Option<&mut Transaction<'_>>, @@ -240,7 +302,6 @@ impl Database { StoredDocumentVersion, r#" select - vault_id, vault_update_id, document_id as "document_id: Hyphenated", relative_path, @@ -248,21 +309,23 @@ impl Database { content, is_deleted from documents - where vault_id = ? and vault_update_id = ?"#, - vault, + where vault_update_id = ?"#, vault_update_id ); if let Some(transaction) = transaction { query.fetch_optional(&mut **transaction).await } else { - query.fetch_optional(&self.connection_pool).await + query + .fetch_optional(&self.get_connection_pool(vault).await?) + .await } .context("Cannot fetch document version") } pub async fn insert_document_version( - &self, + &mut self, + vault: &VaultId, version: &StoredDocumentVersion, transaction: Option<&mut Transaction<'_>>, ) -> Result<()> { @@ -270,7 +333,6 @@ impl Database { let query = sqlx::query!( r#" insert into documents ( - vault_id, vault_update_id, document_id, relative_path, @@ -278,9 +340,8 @@ impl Database { content, is_deleted ) - values (?, ?, ?, ?, ?, ?, ?) + values (?, ?, ?, ?, ?, ?) "#, - version.vault_id, version.vault_update_id, document_id, version.relative_path, @@ -292,7 +353,7 @@ impl Database { if let Some(transaction) = transaction { query.execute(&mut **transaction).await } else { - query.execute(&self.connection_pool).await + query.execute(&self.get_connection_pool(vault).await?).await } .context("Cannot insert document version")?; diff --git a/backend/sync_server/src/database/migrations/20241207143519_bootstrap.sql b/backend/sync_server/src/database/migrations/20241207143519_bootstrap.sql index 62002b58..4a9f31ba 100644 --- a/backend/sync_server/src/database/migrations/20241207143519_bootstrap.sql +++ b/backend/sync_server/src/database/migrations/20241207143519_bootstrap.sql @@ -1,24 +1,21 @@ CREATE TABLE IF NOT EXISTS documents ( - vault_id TEXT NOT NULL, - vault_update_id INTEGER NOT NULL, + vault_update_id INTEGER NOT NULL PRIMARY KEY, document_id TEXT NOT NULL, relative_path TEXT NOT NULL, updated_date TIMESTAMP NOT NULL, content BLOB NOT NULL, - is_deleted BOOLEAN NOT NULL, - PRIMARY KEY (vault_id, vault_update_id) + is_deleted BOOLEAN NOT NULL ); CREATE VIEW IF NOT EXISTS latest_document_versions AS SELECT d.* FROM documents d INNER JOIN ( - SELECT vault_id, MAX(vault_update_id) AS max_version_id + SELECT MAX(vault_update_id) AS max_version_id FROM documents - GROUP BY vault_id, document_id + GROUP BY document_id ) max_versions -ON d.vault_id = max_versions.vault_id -AND d.vault_update_id = max_versions.max_version_id; +ON d.vault_update_id = max_versions.max_version_id; CREATE INDEX IF NOT EXISTS idx_documents_vault_id_relative_path -ON documents (vault_id, relative_path); +ON documents (relative_path); diff --git a/backend/sync_server/src/database/models.rs b/backend/sync_server/src/database/models.rs index 9ba1832b..a837e93c 100644 --- a/backend/sync_server/src/database/models.rs +++ b/backend/sync_server/src/database/models.rs @@ -9,7 +9,6 @@ pub type DocumentId = uuid::Uuid; #[derive(Debug, Clone)] pub struct StoredDocumentVersion { - pub vault_id: VaultId, pub vault_update_id: VaultUpdateId, pub document_id: DocumentId, pub relative_path: String, @@ -19,15 +18,12 @@ pub struct StoredDocumentVersion { } impl PartialEq for StoredDocumentVersion { - fn eq(&self, other: &Self) -> bool { - self.vault_id == other.vault_id && self.vault_update_id == other.vault_update_id - } + fn eq(&self, other: &Self) -> bool { self.vault_update_id == other.vault_update_id } } #[derive(Debug, Clone, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct DocumentVersionWithoutContent { - pub vault_id: VaultId, pub vault_update_id: VaultUpdateId, pub document_id: DocumentId, pub relative_path: String, @@ -38,7 +34,6 @@ pub struct DocumentVersionWithoutContent { impl From for DocumentVersionWithoutContent { fn from(value: StoredDocumentVersion) -> Self { Self { - vault_id: value.vault_id, vault_update_id: value.vault_update_id, document_id: value.document_id, relative_path: value.relative_path, @@ -51,7 +46,6 @@ impl From for DocumentVersionWithoutContent { #[derive(Debug, Clone, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct DocumentVersion { - pub vault_id: VaultId, pub vault_update_id: VaultUpdateId, pub document_id: DocumentId, pub relative_path: String, @@ -63,7 +57,6 @@ pub struct DocumentVersion { impl From for DocumentVersion { fn from(value: StoredDocumentVersion) -> Self { Self { - vault_id: value.vault_id, vault_update_id: value.vault_update_id, document_id: value.document_id, relative_path: value.relative_path, diff --git a/backend/sync_server/src/server/create_document.rs b/backend/sync_server/src/server/create_document.rs index 4d17effc..89f54783 100644 --- a/backend/sync_server/src/server/create_document.rs +++ b/backend/sync_server/src/server/create_document.rs @@ -77,7 +77,7 @@ pub async fn create_document_json( async fn internal_create_document( auth_header: Authorization, - state: AppState, + mut state: AppState, vault_id: VaultId, document_id: Option, relative_path: String, @@ -87,7 +87,7 @@ async fn internal_create_document( let mut transaction = state .database - .create_write_transaction() + .create_write_transaction(&vault_id) .await .map_err(server_error)?; @@ -119,7 +119,6 @@ async fn internal_create_document( let sanitized_relative_path = sanitize_path(&relative_path); let new_version = StoredDocumentVersion { - vault_id, vault_update_id: last_update_id + 1, document_id, relative_path: sanitized_relative_path, @@ -130,7 +129,7 @@ async fn internal_create_document( state .database - .insert_document_version(&new_version, Some(&mut transaction)) + .insert_document_version(&vault_id, &new_version, Some(&mut transaction)) .await .map_err(server_error)?; diff --git a/backend/sync_server/src/server/delete_document.rs b/backend/sync_server/src/server/delete_document.rs index 25901e84..75f90d23 100644 --- a/backend/sync_server/src/server/delete_document.rs +++ b/backend/sync_server/src/server/delete_document.rs @@ -29,14 +29,14 @@ pub async fn delete_document( vault_id, document_id, }): Path, - State(state): State, + State(mut state): State, Json(request): Json, ) -> Result, SyncServerError> { auth(&state, auth_header.token())?; let mut transaction = state .database - .create_write_transaction() + .create_write_transaction(&vault_id) .await .map_err(server_error)?; @@ -47,7 +47,6 @@ pub async fn delete_document( .map_err(server_error)?; let new_version = StoredDocumentVersion { - vault_id, vault_update_id: last_update_id + 1, document_id, relative_path: sanitize_path(&request.relative_path), @@ -58,7 +57,7 @@ pub async fn delete_document( state .database - .insert_document_version(&new_version, Some(&mut transaction)) + .insert_document_version(&vault_id, &new_version, Some(&mut transaction)) .await .map_err(server_error)?; diff --git a/backend/sync_server/src/server/fetch_document_version.rs b/backend/sync_server/src/server/fetch_document_version.rs index c6431601..13978b8f 100644 --- a/backend/sync_server/src/server/fetch_document_version.rs +++ b/backend/sync_server/src/server/fetch_document_version.rs @@ -30,7 +30,7 @@ pub async fn fetch_document_version( document_id, vault_update_id, }): Path, - State(state): State, + State(mut state): State, ) -> Result, SyncServerError> { auth(&state, auth_header.token())?; diff --git a/backend/sync_server/src/server/fetch_document_version_content.rs b/backend/sync_server/src/server/fetch_document_version_content.rs index 68e38254..2889d435 100644 --- a/backend/sync_server/src/server/fetch_document_version_content.rs +++ b/backend/sync_server/src/server/fetch_document_version_content.rs @@ -32,7 +32,7 @@ pub async fn fetch_document_version_content( document_id, vault_update_id, }): Path, - State(state): State, + State(mut state): State, ) -> Result { auth(&state, auth_header.token())?; diff --git a/backend/sync_server/src/server/fetch_latest_document_version.rs b/backend/sync_server/src/server/fetch_latest_document_version.rs index a53f2703..89e35882 100644 --- a/backend/sync_server/src/server/fetch_latest_document_version.rs +++ b/backend/sync_server/src/server/fetch_latest_document_version.rs @@ -28,7 +28,7 @@ pub async fn fetch_latest_document_version( vault_id, document_id, }): Path, - State(state): State, + State(mut state): State, ) -> Result, SyncServerError> { auth(&state, auth_header.token())?; diff --git a/backend/sync_server/src/server/fetch_latest_documents.rs b/backend/sync_server/src/server/fetch_latest_documents.rs index b19c3dec..b7ff09b7 100644 --- a/backend/sync_server/src/server/fetch_latest_documents.rs +++ b/backend/sync_server/src/server/fetch_latest_documents.rs @@ -30,7 +30,7 @@ pub async fn fetch_latest_documents( TypedHeader(auth_header): TypedHeader>, Path(PathParams { vault_id }): Path, Query(QueryParams { since_update_id }): Query, - State(state): State, + State(mut state): State, ) -> Result, SyncServerError> { auth(&state, auth_header.token())?; diff --git a/backend/sync_server/src/server/update_document.rs b/backend/sync_server/src/server/update_document.rs index 316b06f4..93ed5417 100644 --- a/backend/sync_server/src/server/update_document.rs +++ b/backend/sync_server/src/server/update_document.rs @@ -83,7 +83,7 @@ pub async fn update_document_json( #[allow(clippy::too_many_arguments)] async fn internal_update_document( auth_header: Authorization, - state: AppState, + mut state: AppState, vault_id: VaultId, document_id: DocumentId, parent_version_id: VaultUpdateId, @@ -110,7 +110,7 @@ async fn internal_update_document( let mut transaction = state .database - .create_write_transaction() + .create_write_transaction(&vault_id) .await .map_err(server_error)?; @@ -196,7 +196,6 @@ async fn internal_update_document( }; let new_version = StoredDocumentVersion { - vault_id, document_id, vault_update_id: last_update_id + 1, relative_path: new_relative_path, @@ -207,7 +206,7 @@ async fn internal_update_document( state .database - .insert_document_version(&new_version, Some(&mut transaction)) + .insert_document_version(&vault_id, &new_version, Some(&mut transaction)) .await .map_err(server_error)?; diff --git a/clean-up.sh b/clean-up.sh new file mode 100644 index 00000000..2e7f7c3e --- /dev/null +++ b/clean-up.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +rm -rf backend/databases +rm -rf frontend/test-client/logs From 993515ea1243b67cd2ccdff1e048a92c649cb724 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 16 Mar 2025 14:59:39 +0000 Subject: [PATCH 40/58] Add scritps folder --- .github/workflows/e2e.yml | 6 ++---- README.md | 6 +----- bump-version.sh => scripts/bump-version.sh | 0 clean-up.sh => scripts/clean-up.sh | 2 +- frontend/test-client/run.sh => scripts/e2e.sh | 6 +++++- scripts/update-api-types.sh | 4 ++++ 6 files changed, 13 insertions(+), 11 deletions(-) rename bump-version.sh => scripts/bump-version.sh (100%) rename clean-up.sh => scripts/clean-up.sh (83%) mode change 100644 => 100755 rename frontend/test-client/run.sh => scripts/e2e.sh (95%) create mode 100755 scripts/update-api-types.sh diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index f6561501..4cbcf72c 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -32,7 +32,5 @@ jobs: - name: E2E tests run: | UST_BACKTRACE=1 cargo run -p sync_server & - cd ../frontend - - npm run build - test-client/run.sh 32 + cd .. + scripts/e2e.sh 32 diff --git a/README.md b/README.md index 62842bc3..280e65c0 100644 --- a/README.md +++ b/README.md @@ -16,9 +16,6 @@ ## Set up Rust - Install [`rustup`](https://rustup.rs): `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh` -- `sudo apt install llvm -y` -- `rustup component add llvm-tools-preview` -- `cargo install cargo-generate cargo-fuzz cargo-insta rustfilt cargo-binutils` - Install [`wasm-pack`](https://rustwasm.github.io/wasm-pack/installer): `curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh` - `cargo install cargo-insta sqlx-cli cargo-edit` @@ -33,8 +30,7 @@ ## Update HTTP API TS bindings ```sh -npm install -g openapi-typescript -openapi-typescript http://localhost:3030/api.json --output frontend/sync-client/src/services/types.ts +./update-api-types.sh ``` ``` diff --git a/bump-version.sh b/scripts/bump-version.sh similarity index 100% rename from bump-version.sh rename to scripts/bump-version.sh diff --git a/clean-up.sh b/scripts/clean-up.sh old mode 100644 new mode 100755 similarity index 83% rename from clean-up.sh rename to scripts/clean-up.sh index 2e7f7c3e..6602e0c7 --- a/clean-up.sh +++ b/scripts/clean-up.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash rm -rf backend/databases rm -rf frontend/test-client/logs diff --git a/frontend/test-client/run.sh b/scripts/e2e.sh similarity index 95% rename from frontend/test-client/run.sh rename to scripts/e2e.sh index 4a32c002..8b61885d 100755 --- a/frontend/test-client/run.sh +++ b/scripts/e2e.sh @@ -12,16 +12,19 @@ fi # Get the number of processes from the first argument process_count=$1 +cd frontend npm run build mkdir -p logs pids=() for i in $(seq 1 $process_count); do - node dist/cli.js > "logs/log_${i}.log" 2>&1 & + node test-client/dist/cli.js > "logs/log_${i}.log" 2>&1 & pids+=($!) done +cd - + print_failed_log() { for i in $(seq 1 $process_count); do if [ -n "${pids[$i-1]}" ] && ! kill -0 ${pids[$i-1]} 2>/dev/null; then @@ -73,3 +76,4 @@ while true; do sleep 0.2 done + diff --git a/scripts/update-api-types.sh b/scripts/update-api-types.sh new file mode 100755 index 00000000..d0aa2357 --- /dev/null +++ b/scripts/update-api-types.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +npm install -g openapi-typescript +openapi-typescript http://localhost:3030/api.json --output frontend/sync-client/src/services/types.ts From 7eb740cd4c2113885f34e7c09a5d55f82e1765ad Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 16 Mar 2025 14:59:47 +0000 Subject: [PATCH 41/58] Bump versions --- frontend/obsidian-plugin/package.json | 8 +- frontend/package-lock.json | 220 +++++++++++++++++--------- frontend/package.json | 8 +- frontend/sync-client/package.json | 6 +- frontend/test-client/package.json | 4 +- 5 files changed, 158 insertions(+), 88 deletions(-) diff --git a/frontend/obsidian-plugin/package.json b/frontend/obsidian-plugin/package.json index 30fad71d..7fcd9c02 100644 --- a/frontend/obsidian-plugin/package.json +++ b/frontend/obsidian-plugin/package.json @@ -14,7 +14,7 @@ "license": "MIT", "devDependencies": { "@types/jest": "^29.5.14", - "@types/node": "^22.13.5", + "@types/node": "^22.13.10", "css-loader": "^7.1.2", "date-fns": "^4.1.0", "file-loader": "^6.2.0", @@ -23,14 +23,14 @@ "mini-css-extract-plugin": "^2.9.2", "obsidian": "1.8.7", "resolve-url-loader": "^5.0.0", - "sass": "^1.85.0", + "sass": "^1.85.1", "sass-loader": "^16.0.5", "sync-client": "file:../sync-client", - "terser-webpack-plugin": "^5.3.11", + "terser-webpack-plugin": "^5.3.14", "ts-jest": "^29.2.6", "ts-loader": "^9.5.2", "tslib": "2.8.1", - "typescript": "5.7.3", + "typescript": "5.8.2", "url": "^0.11.4", "virtual-scroller": "^1.13.1", "webpack": "^5.98.0", diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 69a51099..fea89cd9 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -12,16 +12,17 @@ ], "devDependencies": { "concurrently": "^9.1.2", - "eslint": "9.21.0", + "eslint": "9.22.0", "eslint-plugin-unused-imports": "^4.1.4", - "npm-check-updates": "^17.1.14", - "prettier": "^3.5.2", - "typescript-eslint": "8.24.1" + "npm-check-updates": "^17.1.15", + "prettier": "^3.5.3", + "typescript-eslint": "8.26.1" } }, "../backend/sync_lib/pkg": { "name": "sync_lib", "version": "0.0.30", + "dev": true, "license": "MIT" }, "node_modules/@ampproject/remapping": { @@ -191,12 +192,14 @@ } }, "node_modules/@babel/helpers": { - "version": "7.26.9", + "version": "7.26.10", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.26.10.tgz", + "integrity": "sha512-UPYc3SauzZ3JGgj87GgZ89JVdC5dj0AoetR5Bw6wj4niittNyFh6+eOGonYvJ1ao6B8lEa3Q3klS7ADZ53bc5g==", "dev": true, "license": "MIT", "dependencies": { "@babel/template": "^7.26.9", - "@babel/types": "^7.26.9" + "@babel/types": "^7.26.10" }, "engines": { "node": ">=6.9.0" @@ -460,7 +463,9 @@ } }, "node_modules/@babel/types": { - "version": "7.26.9", + "version": "7.26.10", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.10.tgz", + "integrity": "sha512-emqcG3vHrpxUKTrxcblR36dcrcoRDvKmnL/dCL6ZsHaShW80qxCAcNhzQZrpeM765VzEos+xOi4s+r4IXzTwdQ==", "dev": true, "license": "MIT", "dependencies": { @@ -577,6 +582,16 @@ "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, + "node_modules/@eslint/config-helpers": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.1.0.tgz", + "integrity": "sha512-kLrdPDJE1ckPo94kmPPf9Hfd0DU0Jw6oKYrhe+pwSC0iTUInmTa+w6fw8sGgcfkFJGNdWOUeOaDM4quW4a7OkA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, "node_modules/@eslint/core": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.12.0.tgz", @@ -615,9 +630,9 @@ } }, "node_modules/@eslint/js": { - "version": "9.21.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.21.0.tgz", - "integrity": "sha512-BqStZ3HX8Yz6LvsF5ByXYrtigrV5AXADWLAGc7PH/1SxOb7/FIYYMszZZWiUou/GB9P2lXWk2SV4d+Z8h0nknw==", + "version": "9.22.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.22.0.tgz", + "integrity": "sha512-vLFajx9o8d1/oL2ZkpMYbkLv8nDB6yaIwFNt7nI4+I80U/z03SxmfOMsLbvWr3p7C+Wnoh//aOu2pQW8cS0HCQ==", "dev": true, "license": "MIT", "engines": { @@ -1127,6 +1142,8 @@ }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", "dev": true, "license": "MIT", "dependencies": { @@ -1139,6 +1156,8 @@ }, "node_modules/@nodelib/fs.stat": { "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", "dev": true, "license": "MIT", "engines": { @@ -1147,6 +1166,8 @@ }, "node_modules/@nodelib/fs.walk": { "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", "dev": true, "license": "MIT", "dependencies": { @@ -1449,9 +1470,9 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "22.13.5", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.5.tgz", - "integrity": "sha512-+lTU0PxZXn0Dr1NBtC7Y8cR21AJr87dLLU953CWA6pMxxv/UDc7jYAY90upcrie1nRcD6XNG5HOYEDtgW5TxAg==", + "version": "22.13.10", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.10.tgz", + "integrity": "sha512-I6LPUvlRH+O6VRUqYOcMudhaIdUVWfsjnZavnsraHvpBwaEyMN29ry+0UVJhImYL16xsscu0aske3yA+uPOWfw==", "dev": true, "license": "MIT", "dependencies": { @@ -1485,15 +1506,17 @@ "license": "MIT" }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "8.24.1", + "version": "8.26.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.26.1.tgz", + "integrity": "sha512-2X3mwqsj9Bd3Ciz508ZUtoQQYpOhU/kWoUqIf49H8Z0+Vbh6UF/y0OEYp0Q0axOGzaBGs7QxRwq0knSQ8khQNA==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/regexpp": "^4.10.0", - "@typescript-eslint/scope-manager": "8.24.1", - "@typescript-eslint/type-utils": "8.24.1", - "@typescript-eslint/utils": "8.24.1", - "@typescript-eslint/visitor-keys": "8.24.1", + "@typescript-eslint/scope-manager": "8.26.1", + "@typescript-eslint/type-utils": "8.26.1", + "@typescript-eslint/utils": "8.26.1", + "@typescript-eslint/visitor-keys": "8.26.1", "graphemer": "^1.4.0", "ignore": "^5.3.1", "natural-compare": "^1.4.0", @@ -1509,18 +1532,20 @@ "peerDependencies": { "@typescript-eslint/parser": "^8.0.0 || ^8.0.0-alpha.0", "eslint": "^8.57.0 || ^9.0.0", - "typescript": ">=4.8.4 <5.8.0" + "typescript": ">=4.8.4 <5.9.0" } }, "node_modules/@typescript-eslint/parser": { - "version": "8.24.1", + "version": "8.26.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.26.1.tgz", + "integrity": "sha512-w6HZUV4NWxqd8BdeFf81t07d7/YV9s7TCWrQQbG5uhuvGUAW+fq1usZ1Hmz9UPNLniFnD8GLSsDpjP0hm1S4lQ==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/scope-manager": "8.24.1", - "@typescript-eslint/types": "8.24.1", - "@typescript-eslint/typescript-estree": "8.24.1", - "@typescript-eslint/visitor-keys": "8.24.1", + "@typescript-eslint/scope-manager": "8.26.1", + "@typescript-eslint/types": "8.26.1", + "@typescript-eslint/typescript-estree": "8.26.1", + "@typescript-eslint/visitor-keys": "8.26.1", "debug": "^4.3.4" }, "engines": { @@ -1532,16 +1557,18 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0", - "typescript": ">=4.8.4 <5.8.0" + "typescript": ">=4.8.4 <5.9.0" } }, "node_modules/@typescript-eslint/scope-manager": { - "version": "8.24.1", + "version": "8.26.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.26.1.tgz", + "integrity": "sha512-6EIvbE5cNER8sqBu6V7+KeMZIC1664d2Yjt+B9EWUXrsyWpxx4lEZrmvxgSKRC6gX+efDL/UY9OpPZ267io3mg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.24.1", - "@typescript-eslint/visitor-keys": "8.24.1" + "@typescript-eslint/types": "8.26.1", + "@typescript-eslint/visitor-keys": "8.26.1" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -1552,12 +1579,14 @@ } }, "node_modules/@typescript-eslint/type-utils": { - "version": "8.24.1", + "version": "8.26.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.26.1.tgz", + "integrity": "sha512-Kcj/TagJLwoY/5w9JGEFV0dclQdyqw9+VMndxOJKtoFSjfZhLXhYjzsQEeyza03rwHx2vFEGvrJWJBXKleRvZg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/typescript-estree": "8.24.1", - "@typescript-eslint/utils": "8.24.1", + "@typescript-eslint/typescript-estree": "8.26.1", + "@typescript-eslint/utils": "8.26.1", "debug": "^4.3.4", "ts-api-utils": "^2.0.1" }, @@ -1570,11 +1599,13 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0", - "typescript": ">=4.8.4 <5.8.0" + "typescript": ">=4.8.4 <5.9.0" } }, "node_modules/@typescript-eslint/types": { - "version": "8.24.1", + "version": "8.26.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.26.1.tgz", + "integrity": "sha512-n4THUQW27VmQMx+3P+B0Yptl7ydfceUj4ON/AQILAASwgYdZ/2dhfymRMh5egRUrvK5lSmaOm77Ry+lmXPOgBQ==", "dev": true, "license": "MIT", "engines": { @@ -1586,12 +1617,14 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "8.24.1", + "version": "8.26.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.26.1.tgz", + "integrity": "sha512-yUwPpUHDgdrv1QJ7YQal3cMVBGWfnuCdKbXw1yyjArax3353rEJP1ZA+4F8nOlQ3RfS2hUN/wze3nlY+ZOhvoA==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.24.1", - "@typescript-eslint/visitor-keys": "8.24.1", + "@typescript-eslint/types": "8.26.1", + "@typescript-eslint/visitor-keys": "8.26.1", "debug": "^4.3.4", "fast-glob": "^3.3.2", "is-glob": "^4.0.3", @@ -1607,11 +1640,13 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "typescript": ">=4.8.4 <5.8.0" + "typescript": ">=4.8.4 <5.9.0" } }, "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", "dev": true, "license": "MIT", "dependencies": { @@ -1620,6 +1655,8 @@ }, "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", "dev": true, "license": "ISC", "dependencies": { @@ -1633,14 +1670,16 @@ } }, "node_modules/@typescript-eslint/utils": { - "version": "8.24.1", + "version": "8.26.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.26.1.tgz", + "integrity": "sha512-V4Urxa/XtSUroUrnI7q6yUTD3hDtfJ2jzVfeT3VK0ciizfK2q/zGC0iDh1lFMUZR8cImRrep6/q0xd/1ZGPQpg==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.4.0", - "@typescript-eslint/scope-manager": "8.24.1", - "@typescript-eslint/types": "8.24.1", - "@typescript-eslint/typescript-estree": "8.24.1" + "@typescript-eslint/scope-manager": "8.26.1", + "@typescript-eslint/types": "8.26.1", + "@typescript-eslint/typescript-estree": "8.26.1" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -1651,15 +1690,17 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0", - "typescript": ">=4.8.4 <5.8.0" + "typescript": ">=4.8.4 <5.9.0" } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "8.24.1", + "version": "8.26.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.26.1.tgz", + "integrity": "sha512-AjOC3zfnxd6S4Eiy3jwktJPclqhFHNyd8L6Gycf9WUPoKZpgM5PjkxY1X7uSy61xVpiJDhhk7XT2NVsN3ALTWg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.24.1", + "@typescript-eslint/types": "8.26.1", "eslint-visitor-keys": "^4.2.0" }, "engines": { @@ -2805,18 +2846,19 @@ } }, "node_modules/eslint": { - "version": "9.21.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.21.0.tgz", - "integrity": "sha512-KjeihdFqTPhOMXTt7StsDxriV4n66ueuF/jfPNC3j/lduHwr/ijDwJMsF+wyMJethgiKi5wniIE243vi07d3pg==", + "version": "9.22.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.22.0.tgz", + "integrity": "sha512-9V/QURhsRN40xuHXWjV64yvrzMjcz7ZyNoF2jJFmy9j/SLk0u1OLSZgXi28MrXjymnjEGSR80WCdab3RGMDveQ==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.12.1", "@eslint/config-array": "^0.19.2", + "@eslint/config-helpers": "^0.1.0", "@eslint/core": "^0.12.0", "@eslint/eslintrc": "^3.3.0", - "@eslint/js": "9.21.0", + "@eslint/js": "9.22.0", "@eslint/plugin-kit": "^0.2.7", "@humanfs/node": "^0.16.6", "@humanwhocodes/module-importer": "^1.0.1", @@ -2828,7 +2870,7 @@ "cross-spawn": "^7.0.6", "debug": "^4.3.2", "escape-string-regexp": "^4.0.0", - "eslint-scope": "^8.2.0", + "eslint-scope": "^8.3.0", "eslint-visitor-keys": "^4.2.0", "espree": "^10.3.0", "esquery": "^1.5.0", @@ -2879,7 +2921,9 @@ } }, "node_modules/eslint-scope": { - "version": "8.2.0", + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.3.0.tgz", + "integrity": "sha512-pUNxi75F8MJ/GdeKtVLSbYg4ZI34J6C0C7sbL4YOp2exGwen7ZsuBqKzUhXd0qMQ362yET3z+uPwKeg/0C2XCQ==", "dev": true, "license": "BSD-2-Clause", "dependencies": { @@ -3034,6 +3078,8 @@ }, "node_modules/fast-glob": { "version": "3.3.3", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", + "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", "dev": true, "license": "MIT", "dependencies": { @@ -3049,6 +3095,8 @@ }, "node_modules/fast-glob/node_modules/glob-parent": { "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", "dev": true, "license": "ISC", "dependencies": { @@ -3092,7 +3140,9 @@ } }, "node_modules/fastq": { - "version": "1.19.0", + "version": "1.19.1", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.19.1.tgz", + "integrity": "sha512-GwLTyxkCXjXbxqIhTsMI2Nui8huMPtnxg7krajPJAjnEG/iiOS7i+zCtWGZR9G0NBKbXKh6X9m9UIsYX/N6vvQ==", "dev": true, "license": "ISC", "dependencies": { @@ -4472,6 +4522,8 @@ }, "node_modules/merge2": { "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", "dev": true, "license": "MIT", "engines": { @@ -4660,7 +4712,9 @@ } }, "node_modules/npm-check-updates": { - "version": "17.1.14", + "version": "17.1.15", + "resolved": "https://registry.npmjs.org/npm-check-updates/-/npm-check-updates-17.1.15.tgz", + "integrity": "sha512-miATvKu5rjec/1wxc5TGDjpsucgtCHwRVZorZpDkS6NzdWXfnUWlN4abZddWb7XSijAuBNzzYglIdTm9SbgMVg==", "dev": true, "license": "Apache-2.0", "bin": { @@ -4732,7 +4786,9 @@ } }, "node_modules/openapi-fetch": { - "version": "0.13.4", + "version": "0.13.5", + "resolved": "https://registry.npmjs.org/openapi-fetch/-/openapi-fetch-0.13.5.tgz", + "integrity": "sha512-AQK8T9GSKFREFlN1DBXTYsLjs7YV2tZcJ7zUWxbjMoQmj8dDSFRrzhLCbHPZWA1TMV3vACqfCxLEZcwf2wxV6Q==", "license": "MIT", "dependencies": { "openapi-typescript-helpers": "^0.0.15" @@ -5127,9 +5183,9 @@ } }, "node_modules/prettier": { - "version": "3.5.2", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.5.2.tgz", - "integrity": "sha512-lc6npv5PH7hVqozBR7lkBNOGXV9vMwROAPlumdBkX0wTbbzPu/U1hk5yL8p2pt4Xoc+2mkT8t/sow2YrV/M5qg==", + "version": "3.5.3", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.5.3.tgz", + "integrity": "sha512-QQtaxnoDJeAkDvDKWCLiwIXkTgRhwYDEQCghU9Z6q03iyek/rxRh/2lC3HB7P8sWT2xC/y5JDctPLBIGzHKbhw==", "dev": true, "license": "MIT", "bin": { @@ -5219,6 +5275,8 @@ }, "node_modules/queue-microtask": { "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", "dev": true, "funding": [ { @@ -5374,7 +5432,9 @@ } }, "node_modules/reusify": { - "version": "1.0.4", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", + "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", "dev": true, "license": "MIT", "engines": { @@ -5384,6 +5444,8 @@ }, "node_modules/run-parallel": { "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", "dev": true, "funding": [ { @@ -5432,7 +5494,9 @@ "license": "MIT" }, "node_modules/sass": { - "version": "1.85.0", + "version": "1.85.1", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.85.1.tgz", + "integrity": "sha512-Uk8WpxM5v+0cMR0XjX9KfRIacmSG86RH4DCCZjLU2rFh5tyutt9siAXJ7G+YfxQ99Q6wrRMbMlVl6KqUms71ag==", "dev": true, "license": "MIT", "dependencies": { @@ -5834,7 +5898,9 @@ } }, "node_modules/terser-webpack-plugin": { - "version": "5.3.11", + "version": "5.3.14", + "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.14.tgz", + "integrity": "sha512-vkZjpUjb6OMS7dhV+tILUW6BhpDR7P2L/aQSAv+Uwk+m8KATX9EccViHTJR2qDtACKPIYndLGCyl3FMo+r2LMw==", "dev": true, "license": "MIT", "dependencies": { @@ -6159,7 +6225,9 @@ } }, "node_modules/typescript": { - "version": "5.7.3", + "version": "5.8.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.2.tgz", + "integrity": "sha512-aJn6wq13/afZp/jT9QZmwEjDqqvSGp1VT5GVg+f/t6/oVyrgXM6BY1h9BRh/O5p3PlUPAe+WuiEZOmb/49RqoQ==", "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", @@ -6170,13 +6238,15 @@ } }, "node_modules/typescript-eslint": { - "version": "8.24.1", + "version": "8.26.1", + "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.26.1.tgz", + "integrity": "sha512-t/oIs9mYyrwZGRpDv3g+3K6nZ5uhKEMt2oNmAPwaY4/ye0+EH4nXIPYNtkYFS6QHm+1DFg34DbglYBz5P9Xysg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/eslint-plugin": "8.24.1", - "@typescript-eslint/parser": "8.24.1", - "@typescript-eslint/utils": "8.24.1" + "@typescript-eslint/eslint-plugin": "8.26.1", + "@typescript-eslint/parser": "8.26.1", + "@typescript-eslint/utils": "8.26.1" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -6187,7 +6257,7 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0", - "typescript": ">=4.8.4 <5.8.0" + "typescript": ">=4.8.4 <5.9.0" } }, "node_modules/undici-types": { @@ -6658,7 +6728,7 @@ "license": "MIT", "devDependencies": { "@types/jest": "^29.5.14", - "@types/node": "^22.13.5", + "@types/node": "^22.13.10", "css-loader": "^7.1.2", "date-fns": "^4.1.0", "file-loader": "^6.2.0", @@ -6667,14 +6737,14 @@ "mini-css-extract-plugin": "^2.9.2", "obsidian": "1.8.7", "resolve-url-loader": "^5.0.0", - "sass": "^1.85.0", + "sass": "^1.85.1", "sass-loader": "^16.0.5", "sync-client": "file:../sync-client", - "terser-webpack-plugin": "^5.3.11", + "terser-webpack-plugin": "^5.3.14", "ts-jest": "^29.2.6", "ts-loader": "^9.5.2", "tslib": "2.8.1", - "typescript": "5.7.3", + "typescript": "5.8.2", "url": "^0.11.4", "virtual-scroller": "^1.13.1", "webpack": "^5.98.0", @@ -6686,20 +6756,20 @@ "dependencies": { "byte-base64": "^1.1.0", "fetch-retry": "^6.0.0", - "openapi-fetch": "0.13.4", + "openapi-fetch": "0.13.5", "openapi-typescript": "7.6.1", "p-queue": "^8.1.0", - "sync_lib": "file:../../backend/sync_lib/pkg", "uuid": "^11.1.0" }, "devDependencies": { "@types/jest": "^29.5.14", - "@types/node": "^22.13.5", + "@types/node": "^22.13.10", "jest": "^29.7.0", + "sync_lib": "file:../../backend/sync_lib/pkg", "ts-jest": "^29.2.6", "ts-loader": "^9.5.2", "tslib": "2.8.1", - "typescript": "5.7.3", + "typescript": "5.8.2", "webpack": "^5.98.0", "webpack-cli": "^6.0.1", "webpack-merge": "^6.0.1" @@ -6711,11 +6781,11 @@ "test-client": "dist/cli.js" }, "devDependencies": { - "@types/node": "^22.13.5", + "@types/node": "^22.13.10", "sync-client": "file:../sync-client", "ts-loader": "^9.5.2", "tslib": "2.8.1", - "typescript": "5.7.3", + "typescript": "5.8.2", "uuid": "^11.1.0", "webpack": "^5.98.0", "webpack-cli": "^6.0.1" diff --git a/frontend/package.json b/frontend/package.json index 301e28f7..24c46388 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -21,10 +21,10 @@ }, "devDependencies": { "concurrently": "^9.1.2", - "eslint": "9.21.0", + "eslint": "9.22.0", "eslint-plugin-unused-imports": "^4.1.4", - "npm-check-updates": "^17.1.14", - "prettier": "^3.5.2", - "typescript-eslint": "8.24.1" + "npm-check-updates": "^17.1.15", + "prettier": "^3.5.3", + "typescript-eslint": "8.26.1" } } diff --git a/frontend/sync-client/package.json b/frontend/sync-client/package.json index 778c5d35..37e63bc8 100644 --- a/frontend/sync-client/package.json +++ b/frontend/sync-client/package.json @@ -15,19 +15,19 @@ "dependencies": { "byte-base64": "^1.1.0", "fetch-retry": "^6.0.0", - "openapi-fetch": "0.13.4", + "openapi-fetch": "0.13.5", "openapi-typescript": "7.6.1", "p-queue": "^8.1.0", "uuid": "^11.1.0" }, "devDependencies": { "@types/jest": "^29.5.14", - "@types/node": "^22.13.5", + "@types/node": "^22.13.10", "jest": "^29.7.0", "ts-jest": "^29.2.6", "ts-loader": "^9.5.2", "tslib": "2.8.1", - "typescript": "5.7.3", + "typescript": "5.8.2", "webpack": "^5.98.0", "webpack-cli": "^6.0.1", "webpack-merge": "^6.0.1", diff --git a/frontend/test-client/package.json b/frontend/test-client/package.json index 676da96f..88877204 100644 --- a/frontend/test-client/package.json +++ b/frontend/test-client/package.json @@ -11,11 +11,11 @@ "test": "jest --passWithNoTests" }, "devDependencies": { - "@types/node": "^22.13.5", + "@types/node": "^22.13.10", "sync-client": "file:../sync-client", "ts-loader": "^9.5.2", "tslib": "2.8.1", - "typescript": "5.7.3", + "typescript": "5.8.2", "uuid": "^11.1.0", "webpack": "^5.98.0", "webpack-cli": "^6.0.1" From 9f09b07de9eef7b71b8306ff4bdeaa4264d58810 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 16 Mar 2025 15:38:13 +0000 Subject: [PATCH 42/58] Lint --- README.md | 71 +++++++++--------- backend/Cargo.toml | 16 ++++ backend/reconcile/src/diffs/myers.rs | 1 - backend/reconcile/src/diffs/raw_operation.rs | 6 +- .../operation_transformation/edited_text.rs | 1 + .../src/operation_transformation/operation.rs | 16 ++-- backend/reconcile/src/utils/merge_iters.rs | 3 +- backend/sync_lib/src/lib.rs | 44 +++++++++++ backend/sync_server/src/config.rs | 9 ++- backend/sync_server/src/database.rs | 2 +- backend/sync_server/src/errors.rs | 17 +++-- backend/sync_server/src/server.rs | 73 +++++++++++-------- .../src/server/fetch_document_version.rs | 14 ++-- .../server/fetch_document_version_content.rs | 14 ++-- .../server/fetch_latest_document_version.rs | 14 ++-- .../sync_server/src/server/update_document.rs | 4 +- 16 files changed, 191 insertions(+), 114 deletions(-) diff --git a/README.md b/README.md index 280e65c0..6ad857e0 100644 --- a/README.md +++ b/README.md @@ -1,57 +1,56 @@ -## VaultLink self-hosted Obsidian sync plugin +# VaultLink self-hosted Obsidian plugin for file syncing [![Check](https://github.com/schmelczer/vault-link/actions/workflows/check.yml/badge.svg)](https://github.com/schmelczer/vault-link/actions/workflows/check.yml) +[![E2E tests](https://github.com/schmelczer/vault-link/actions/workflows/e2e.yml/badge.svg)](https://github.com/schmelczer/vault-link/actions/workflows/e2e.yml) [![Publish server Docker image](https://github.com/schmelczer/vault-link/actions/workflows/publish-docker.yml/badge.svg)](https://github.com/schmelczer/vault-link/actions/workflows/publish-docker.yml) [![Publish Obsidian plugin](https://github.com/schmelczer/vault-link/actions/workflows/publish-plugin.yml/badge.svg)](https://github.com/schmelczer/vault-link/actions/workflows/publish-plugin.yml) -## Install [nvm](https://github.com/nvm-sh/nvm) +## Develop + +### Install [nvm](https://github.com/nvm-sh/nvm) - `curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash` -- `nvm install 20` -- `nvm use 20` -- Optionally set the system-wide default: `nvm alias default 20` +- `nvm install 22` +- `nvm use 22` +- Optionally set the system-wide default: `nvm alias default 22` -## Set up Rust +### Set up Rust - Install [`rustup`](https://rustup.rs): `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh` - Install [`wasm-pack`](https://rustwasm.github.io/wasm-pack/installer): `curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh` - `cargo install cargo-insta sqlx-cli cargo-edit` - -## Publish new version +### Install Obsidian on Linux ```sh -./bump-version.sh patch -``` - - -## Update HTTP API TS bindings - -```sh -./update-api-types.sh -``` - -``` - -todo: enable -[workspace.lints.clippy] -single_call_fn = { level = "allow", priority = 1 } -absolute_paths = { level = "allow", priority = 1 } -arithmetic_side_effects = { level = "allow", priority = 1 } -similar_names = { level = "allow", priority = 1 } -self_named_module_files = { level = "allow", priority = 1 } -single_char_lifetime_names = { level = "allow", priority = 1 } -missing_docs_in_private_items = { level = "allow", priority = 1 } -question_mark_used = { level = "allow", priority = 1 } -implicit_return = { level = "allow", priority = 1 } -pedantic = { level = "warn", priority = 0 } -cargo = { level = "warn", priority = 0 } - -``` - apt install flatpak flatpak remote-add --if-not-exists flathub https://dl.flathub.org/repo/flathub.flatpakrepo flatpak install flathub md.obsidian.Obsidian flatpak run md.obsidian.Obsidian +``` + +### Scripts + +#### Update HTTP API TS bindings + +```sh +scripts/update-api-types.sh +``` + +#### Publish new version + +```sh +scripts/bump-version.sh patch +``` + + +#### Run E2E tests + +```sh +scripts/e2e.sh +``` + +And to clean up the logs & database files, run `scripts/clean-up.sh` +``` diff --git a/backend/Cargo.toml b/backend/Cargo.toml index c405cf52..5c3768a5 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -56,3 +56,19 @@ uninlined_format_args = "warn" unnested_or_patterns = "warn" unused_self = "warn" verbose_file_reads = "warn" + +cast_possible_truncation = { level = "allow", priority = 1 } +doc_link_with_quotes = { level = "allow", priority = 1 } +cast_sign_loss = { level = "allow", priority = 1 } +cast_possible_wrap = { level = "allow", priority = 1 } +struct_field_names = { level = "allow", priority = 1 } +single_call_fn = { level = "allow", priority = 1 } +absolute_paths = { level = "allow", priority = 1 } +arithmetic_side_effects = { level = "allow", priority = 1 } +similar_names = { level = "allow", priority = 1 } +self_named_module_files = { level = "allow", priority = 1 } +single_char_lifetime_names = { level = "allow", priority = 1 } +missing_docs_in_private_items = { level = "allow", priority = 1 } +question_mark_used = { level = "allow", priority = 1 } +implicit_return = { level = "allow", priority = 1 } +pedantic = { level = "warn", priority = 0 } diff --git a/backend/reconcile/src/diffs/myers.rs b/backend/reconcile/src/diffs/myers.rs index cce51d54..9692c221 100644 --- a/backend/reconcile/src/diffs/myers.rs +++ b/backend/reconcile/src/diffs/myers.rs @@ -99,7 +99,6 @@ impl IndexMut for V { } } -#[inline(always)] fn split_at(range: Range, at: usize) -> (Range, Range) { (range.start..at, at..range.end) } diff --git a/backend/reconcile/src/diffs/raw_operation.rs b/backend/reconcile/src/diffs/raw_operation.rs index bf970062..0df48f5d 100644 --- a/backend/reconcile/src/diffs/raw_operation.rs +++ b/backend/reconcile/src/diffs/raw_operation.rs @@ -16,9 +16,9 @@ where { pub fn tokens(&self) -> &Vec> { match self { - RawOperation::Insert(tokens) => tokens, - RawOperation::Delete(tokens) => tokens, - RawOperation::Equal(tokens) => tokens, + RawOperation::Insert(tokens) + | RawOperation::Delete(tokens) + | RawOperation::Equal(tokens) => tokens, } } diff --git a/backend/reconcile/src/operation_transformation/edited_text.rs b/backend/reconcile/src/operation_transformation/edited_text.rs index 87a5df40..d3ae3832 100644 --- a/backend/reconcile/src/operation_transformation/edited_text.rs +++ b/backend/reconcile/src/operation_transformation/edited_text.rs @@ -234,6 +234,7 @@ where } /// Apply the operations to the text and return the resulting text. + #[must_use] pub fn apply(&self) -> String { let mut builder: StringBuilder<'_> = StringBuilder::new(self.text); diff --git a/backend/reconcile/src/operation_transformation/operation.rs b/backend/reconcile/src/operation_transformation/operation.rs index ffc4f7d6..73fa6140 100644 --- a/backend/reconcile/src/operation_transformation/operation.rs +++ b/backend/reconcile/src/operation_transformation/operation.rs @@ -1,8 +1,8 @@ -use core::{ - fmt::{Debug, Display}, +use core::fmt::{Debug, Display}; +use std::{ + hash::{DefaultHasher, Hash, Hasher}, ops::Range, }; -use std::hash::{DefaultHasher, Hash, Hasher}; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; @@ -57,7 +57,7 @@ where index.hash(state); deleted_character_count.hash(state); } - }; + } } } @@ -133,7 +133,7 @@ where builder.delete(self.range()); } - }; + } builder } @@ -141,8 +141,7 @@ where /// Returns the index of the first character that the operation affects. pub fn start_index(&self) -> usize { match self { - Operation::Insert { index, .. } => *index, - Operation::Delete { index, .. } => *index, + Operation::Insert { index, .. } | Operation::Delete { index, .. } => *index, } } @@ -156,6 +155,7 @@ where } /// Returns the range of indices of characters that the operation affects. + #[allow(clippy::range_plus_one)] pub fn range(&self) -> Range { self.start_index()..self.end_index() + 1 } /// Returns the number of affected characters. It is always greater than 0 @@ -382,7 +382,7 @@ mod tests { use super::*; #[test] - #[should_panic] + #[should_panic(expected = "Shifted index must be non-negative")] fn test_shifting_error() { insta::assert_debug_snapshot!( Operation::create_insert(1, vec!["hi".into()]) diff --git a/backend/reconcile/src/utils/merge_iters.rs b/backend/reconcile/src/utils/merge_iters.rs index c7b73345..2730c336 100644 --- a/backend/reconcile/src/utils/merge_iters.rs +++ b/backend/reconcile/src/utils/merge_iters.rs @@ -46,8 +46,7 @@ where }; match order { - Some(Ordering::Less) | None => self.left.next(), - Some(Ordering::Equal) => self.left.next(), + Some(Ordering::Less | Ordering::Equal) | None => self.left.next(), Some(Ordering::Greater) => self.right.next(), } } diff --git a/backend/sync_lib/src/lib.rs b/backend/sync_lib/src/lib.rs index e0fa5eb7..6f27e055 100644 --- a/backend/sync_lib/src/lib.rs +++ b/backend/sync_lib/src/lib.rs @@ -18,7 +18,20 @@ pub mod errors; /// Encode binary data for easy transport over HTTP. Inverse of /// `base64_to_bytes`. +/// +/// # Arguments +/// +/// - `input`: The binary data to encode. +/// +/// # Returns +/// +/// The base64-encoded string. +/// +/// # Panics +/// +/// If the input is not valid UTF-8. #[wasm_bindgen(js_name = bytesToBase64)] +#[must_use] pub fn bytes_to_base64(input: &[u8]) -> String { set_panic_hook(); @@ -26,6 +39,19 @@ pub fn bytes_to_base64(input: &[u8]) -> String { } /// Inverse of `bytes_to_base64`. +/// Decode base64-encoded data into binary data. +/// +/// # Arguments +/// +/// - `input`: The base64-encoded string. +/// +/// # Returns +/// +/// The decoded binary data. +/// +/// # Errors +/// +/// If the input is not valid base64. #[wasm_bindgen(js_name = base64ToBytes)] pub fn base64_to_bytes(input: &str) -> Result, SyncLibError> { set_panic_hook(); @@ -36,7 +62,22 @@ pub fn base64_to_bytes(input: &str) -> Result, SyncLibError> { /// Merge two documents with a common parent. Relies on `reconcile::reconcile` /// for texts and returns the right document as-is if either of the updated /// documents is binary. +/// +/// # Arguments +/// +/// - `parent`: The common parent document. +/// - `left`: The left document updated by one user. +/// - `right`: The right document updated by another user. +/// +/// # Returns +/// +/// The merged document. +/// +/// # Panics +/// +/// If any of the input documents are not valid UTF-8 strings. #[wasm_bindgen] +#[must_use] pub fn merge(parent: &[u8], left: &[u8], right: &[u8]) -> Vec { set_panic_hook(); @@ -54,6 +95,7 @@ pub fn merge(parent: &[u8], left: &[u8], right: &[u8]) -> Vec { /// WASM wrapper around `reconcile::reconcile` for text merging. #[wasm_bindgen(js_name = mergeText)] +#[must_use] pub fn merge_text(parent: &str, left: &str, right: &str) -> String { set_panic_hook(); @@ -63,6 +105,7 @@ pub fn merge_text(parent: &str, left: &str, right: &str) -> String { /// Heuristically determine if the given data is a binary or a text file's /// content. #[wasm_bindgen(js_name = isBinary)] +#[must_use] pub fn is_binary(data: &[u8]) -> bool { set_panic_hook(); @@ -77,6 +120,7 @@ pub fn is_binary(data: &[u8]) -> bool { /// We don't want to support merging structured data like JSON, YAML, etc. #[wasm_bindgen(js_name = isFileTypeMergable)] +#[must_use] pub fn is_file_type_mergable(path_or_file_name: &str) -> bool { set_panic_hook(); diff --git a/backend/sync_server/src/config.rs b/backend/sync_server/src/config.rs index 829375da..0de65f1a 100644 --- a/backend/sync_server/src/config.rs +++ b/backend/sync_server/src/config.rs @@ -42,9 +42,12 @@ impl Config { } pub async fn load_from_file(path: &Path) -> Result { - let contents = fs::read_to_string(path) - .await - .with_context(|| format!("Cannot load configuration from disk from ({path:?})"))?; + let contents = fs::read_to_string(path).await.with_context(|| { + format!( + "Cannot load configuration from disk from ({})", + path.display() + ) + })?; let config = serde_yaml::from_str(&contents).context("Failed to parse configuration")?; diff --git a/backend/sync_server/src/database.rs b/backend/sync_server/src/database.rs index a2a7499e..5492d911 100644 --- a/backend/sync_server/src/database.rs +++ b/backend/sync_server/src/database.rs @@ -79,7 +79,7 @@ impl Database { .test_before_acquire(true) .connect_with(connection_options) .await - .with_context(|| format!("Cannot open database at '{file_name:?}'"))?; + .with_context(|| format!("Cannot open database at '{}'", file_name.display()))?; Self::run_migrations(&pool).await?; diff --git a/backend/sync_server/src/errors.rs b/backend/sync_server/src/errors.rs index aa109c82..5aec9c32 100644 --- a/backend/sync_server/src/errors.rs +++ b/backend/sync_server/src/errors.rs @@ -33,12 +33,12 @@ pub enum SyncServerError { impl SyncServerError { pub fn serialize(&self) -> SerializedError { match self { - Self::InitError(error) => error.into(), - Self::ClientError(error) => error.into(), - Self::ServerError(error) => error.into(), - Self::NotFound(error) => error.into(), - Self::Unauthorized(error) => error.into(), - Self::PermissionDeniedError(error) => error.into(), + Self::InitError(error) + | Self::ClientError(error) + | Self::ServerError(error) + | Self::NotFound(error) + | Self::Unauthorized(error) + | Self::PermissionDeniedError(error) => error.into(), } } } @@ -48,9 +48,10 @@ impl IntoResponse for SyncServerError { let body = Json(self.serialize()); match self { - Self::InitError(_) => (StatusCode::INTERNAL_SERVER_ERROR, body).into_response(), + Self::InitError(_) | Self::ServerError(_) => { + (StatusCode::INTERNAL_SERVER_ERROR, body).into_response() + } Self::ClientError(_) => (StatusCode::BAD_REQUEST, body).into_response(), - Self::ServerError(_) => (StatusCode::INTERNAL_SERVER_ERROR, body).into_response(), Self::NotFound(_) => (StatusCode::NOT_FOUND, body).into_response(), Self::Unauthorized(_) => (StatusCode::UNAUTHORIZED, body).into_response(), Self::PermissionDeniedError(_) => (StatusCode::FORBIDDEN, body).into_response(), diff --git a/backend/sync_server/src/server.rs b/backend/sync_server/src/server.rs index bf62fec5..511187e0 100644 --- a/backend/sync_server/src/server.rs +++ b/backend/sync_server/src/server.rs @@ -16,6 +16,7 @@ use axum::{ extract::{DefaultBodyLimit, Request}, http::{self, HeaderValue, Method}, response::IntoResponse, + routing::IntoMakeService, }; use log::{error, info}; use tokio::signal; @@ -30,7 +31,10 @@ use tower_http::{ }; use tracing::{Level, info_span}; -use crate::errors::{SerializedError, not_found_error}; +use crate::{ + config::server_config::ServerConfig, + errors::{SerializedError, not_found_error}, +}; mod app_state; mod auth; mod create_document; @@ -52,24 +56,9 @@ pub async fn create_server() -> Result<()> { .await .context("Failed to initialise app state")?; - let address = format!( - "{}:{}", - &app_state.config.server.host, &app_state.config.server.port - ); - - let mut api = OpenApi { - info: Info { - title: "VaultLink sync server".to_owned(), - summary: Some( - "Simple API for syncing documents between concurrent clients.".to_owned(), - ), - description: Some(include_str!("../README.md").to_owned()), - version: env!("CARGO_PKG_VERSION").to_owned(), - ..Info::default() - }, - ..OpenApi::default() - }; + let server_config = app_state.config.server.clone(); + let mut api = create_open_api(); let app = ApiRouter::new() .api_route("/ping", get(ping::ping)) .api_route( @@ -140,11 +129,42 @@ pub async fn create_server() -> Result<()> { .allow_methods([Method::GET, Method::POST, Method::PUT, Method::DELETE]), ) .with_state(app_state) - .finish_api_with(&mut api, api_docs) + .finish_api_with(&mut api, add_api_docs_error_example) .layer(Extension(Arc::new(api))) // https://github.com/tamasfe/aide/blob/507f4a8822bc0c13cbda0f589da1e0f4cbcdb812/examples/example-axum/src/main.rs#L39 .fallback(handler_404) .into_make_service(); + start_server(app, &server_config).await +} + +async fn serve_api(Extension(api): Extension>) -> impl IntoResponse { Json(api) } + +fn create_open_api() -> OpenApi { + OpenApi { + info: Info { + title: "VaultLink sync server".to_owned(), + summary: Some( + "Simple API for syncing documents between concurrent clients.".to_owned(), + ), + description: Some(include_str!("../README.md").to_owned()), + version: env!("CARGO_PKG_VERSION").to_owned(), + ..Info::default() + }, + ..OpenApi::default() + } +} + +fn add_api_docs_error_example(api: TransformOpenApi<'_>) -> TransformOpenApi<'_> { + api.default_response_with::, _>(|res| { + res.example(SerializedError { + message: "An error has occurred".to_owned(), + causes: vec![], + }) + }) +} + +async fn start_server(app: IntoMakeService, config: &ServerConfig) -> Result<()> { + let address = format!("{}:{}", config.host, config.port); let listener = tokio::net::TcpListener::bind(address.clone()) .await .with_context(|| format!("Failed to bind to address: {address}"))?; @@ -163,17 +183,6 @@ pub async fn create_server() -> Result<()> { .context("Failed to start server") } -async fn serve_api(Extension(api): Extension>) -> impl IntoResponse { Json(api) } - -fn api_docs(api: TransformOpenApi<'_>) -> TransformOpenApi<'_> { - api.default_response_with::, _>(|res| { - res.example(SerializedError { - message: "An error has occurred".to_owned(), - causes: vec![], - }) - }) -} - async fn shutdown_signal() { let ctrl_c = async { signal::ctrl_c() @@ -193,8 +202,8 @@ async fn shutdown_signal() { let terminate = std::future::pending::<()>(); tokio::select! { - _ = ctrl_c => {}, - _ = terminate => {}, + () = ctrl_c => {}, + () = terminate => {}, } } diff --git a/backend/sync_server/src/server/fetch_document_version.rs b/backend/sync_server/src/server/fetch_document_version.rs index 13978b8f..a2b157e3 100644 --- a/backend/sync_server/src/server/fetch_document_version.rs +++ b/backend/sync_server/src/server/fetch_document_version.rs @@ -39,12 +39,14 @@ pub async fn fetch_document_version( .get_document_version(&vault_id, vault_update_id, None) .await .map_err(server_error)? - .map(Ok) - .unwrap_or_else(|| { - Err(not_found_error(anyhow!( - "Document with vault update id `{vault_update_id}` not found", - ))) - })?; + .map_or_else( + || { + Err(not_found_error(anyhow!( + "Document with vault update id `{vault_update_id}` not found", + ))) + }, + Ok, + )?; if result.document_id != document_id { return Err(not_found_error(anyhow!( diff --git a/backend/sync_server/src/server/fetch_document_version_content.rs b/backend/sync_server/src/server/fetch_document_version_content.rs index 2889d435..203f0afb 100644 --- a/backend/sync_server/src/server/fetch_document_version_content.rs +++ b/backend/sync_server/src/server/fetch_document_version_content.rs @@ -41,12 +41,14 @@ pub async fn fetch_document_version_content( .get_document_version(&vault_id, vault_update_id, None) .await .map_err(server_error)? - .map(Ok) - .unwrap_or_else(|| { - Err(not_found_error(anyhow!( - "Document with vault update id `{vault_update_id}` not found", - ))) - })?; + .map_or_else( + || { + Err(not_found_error(anyhow!( + "Document with vault update id `{vault_update_id}` not found", + ))) + }, + Ok, + )?; if result.document_id != document_id { return Err(not_found_error(anyhow!( diff --git a/backend/sync_server/src/server/fetch_latest_document_version.rs b/backend/sync_server/src/server/fetch_latest_document_version.rs index 89e35882..331730e0 100644 --- a/backend/sync_server/src/server/fetch_latest_document_version.rs +++ b/backend/sync_server/src/server/fetch_latest_document_version.rs @@ -37,12 +37,14 @@ pub async fn fetch_latest_document_version( .get_latest_document(&vault_id, &document_id, None) .await .map_err(server_error)? - .map(Ok) - .unwrap_or_else(|| { - Err(not_found_error(anyhow!( - "Document with id `{document_id}` not found", - ))) - })?; + .map_or_else( + || { + Err(not_found_error(anyhow!( + "Document with id `{document_id}` not found", + ))) + }, + Ok, + )?; Ok(Json(latest_version.into())) } diff --git a/backend/sync_server/src/server/update_document.rs b/backend/sync_server/src/server/update_document.rs index 93ed5417..a9b9c13e 100644 --- a/backend/sync_server/src/server/update_document.rs +++ b/backend/sync_server/src/server/update_document.rs @@ -80,7 +80,7 @@ pub async fn update_document_json( .await } -#[allow(clippy::too_many_arguments)] +#[allow(clippy::too_many_arguments, clippy::too_many_lines)] async fn internal_update_document( auth_header: Authorization, mut state: AppState, @@ -176,7 +176,7 @@ async fn internal_update_document( let new_relative_path = if parent_document.relative_path == latest_version.relative_path && latest_version.relative_path != sanitized_relative_path { - let mut new_relative_path = Default::default(); + let mut new_relative_path = String::default(); for candidate in deduped_file_paths(&sanitized_relative_path) { if state .database From 99608b55cd3529874436b19fbf52b77e41c0a321 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 16 Mar 2025 15:46:02 +0000 Subject: [PATCH 43/58] . --- .github/workflows/check.yml | 12 ++++++------ .github/workflows/e2e.yml | 12 ++++++------ .github/workflows/publish-plugin.yml | 2 +- scripts/clean-up.sh | 2 +- scripts/e2e.sh | 6 +++--- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index dd5841ed..3c777e0a 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -17,6 +17,12 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Setup Node.js environment + uses: actions/setup-node@v4.2.0 + with: + node-version: "22.x" + check-latest: true + - name: Setup run: | cargo install sqlx-cli @@ -36,12 +42,6 @@ jobs: cargo clippy --all-targets --all-features cargo fmt --all -- --check - - name: Setup Node.js environment - uses: actions/setup-node@v4.2.0 - with: - node-version: "22.x" - check-latest: true - - name: Test backend run: | cd backend diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 4cbcf72c..3e1a5ae8 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -17,16 +17,16 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Setup - run: | - cargo install sqlx-cli - cd backend - sqlx database create --database-url sqlite://db.sqlite3 - sqlx migrate run --source sync_server/src/database/migrations --database-url sqlite://db.sqlite3 + - name: Setup Node.js environment + uses: actions/setup-node@v4.2.0 + with: + node-version: "22.x" + check-latest: true - name: Build wasm run: | cargo install wasm-pack + cd sync_lib wasm-pack build --target web sync_lib - name: E2E tests diff --git a/.github/workflows/publish-plugin.yml b/.github/workflows/publish-plugin.yml index d8c4b468..19bcc788 100644 --- a/.github/workflows/publish-plugin.yml +++ b/.github/workflows/publish-plugin.yml @@ -12,7 +12,7 @@ jobs: runs-on: self-hosted steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Node.js environment uses: actions/setup-node@v4.2.0 diff --git a/scripts/clean-up.sh b/scripts/clean-up.sh index 6602e0c7..85c12d10 100755 --- a/scripts/clean-up.sh +++ b/scripts/clean-up.sh @@ -1,4 +1,4 @@ #!/bin/bash rm -rf backend/databases -rm -rf frontend/test-client/logs +rm -rf logs diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 8b61885d..fa06d82e 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -12,14 +12,14 @@ fi # Get the number of processes from the first argument process_count=$1 +mkdir -p logs + cd frontend npm run build -mkdir -p logs - pids=() for i in $(seq 1 $process_count); do - node test-client/dist/cli.js > "logs/log_${i}.log" 2>&1 & + node test-client/dist/cli.js > "../logs/log_${i}.log" 2>&1 & pids+=($!) done From a1a7b200c04ae25637e3f5e3ce72a90cf567f107 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 16 Mar 2025 17:51:59 +0000 Subject: [PATCH 44/58] Fix tests for real --- .github/workflows/check.yml | 2 -- .github/workflows/e2e.yml | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 3c777e0a..2bd76f29 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -47,8 +47,6 @@ jobs: cd backend cargo test --verbose cd sync_lib - nvm install 22 - nvm use 22 wasm-pack test --node - name: Lint frontend diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 3e1a5ae8..ada56b16 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -26,7 +26,7 @@ jobs: - name: Build wasm run: | cargo install wasm-pack - cd sync_lib + cd backend/sync_lib wasm-pack build --target web sync_lib - name: E2E tests From 7bf7790e1f7a03228274dd681ae722d28e179fc1 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 16 Mar 2025 17:52:40 +0000 Subject: [PATCH 45/58] Style --- backend/sync_server/src/config.rs | 2 +- backend/sync_server/src/database.rs | 2 +- frontend/sync-client/src/persistence/database.ts | 2 -- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/backend/sync_server/src/config.rs b/backend/sync_server/src/config.rs index 0de65f1a..862dd0e7 100644 --- a/backend/sync_server/src/config.rs +++ b/backend/sync_server/src/config.rs @@ -44,7 +44,7 @@ impl Config { pub async fn load_from_file(path: &Path) -> Result { let contents = fs::read_to_string(path).await.with_context(|| { format!( - "Cannot load configuration from disk from ({})", + "Cannot load configuration from disk from {}", path.display() ) })?; diff --git a/backend/sync_server/src/database.rs b/backend/sync_server/src/database.rs index 5492d911..123f1c8a 100644 --- a/backend/sync_server/src/database.rs +++ b/backend/sync_server/src/database.rs @@ -79,7 +79,7 @@ impl Database { .test_before_acquire(true) .connect_with(connection_options) .await - .with_context(|| format!("Cannot open database at '{}'", file_name.display()))?; + .with_context(|| format!("Cannot open database at {}", file_name.display()))?; Self::run_migrations(&pool).await?; diff --git a/frontend/sync-client/src/persistence/database.ts b/frontend/sync-client/src/persistence/database.ts index 5011fc85..9f003d4c 100644 --- a/frontend/sync-client/src/persistence/database.ts +++ b/frontend/sync-client/src/persistence/database.ts @@ -254,8 +254,6 @@ export class Database { } private save(): void { - this.logger.debug(JSON.stringify(this.documents, null, 2)); - this.ensureConsistency(); void this.saveData({ documents: this.resolvedDocuments.map( From 424342cfb4a294c67207c88408aed87d69883855 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 16 Mar 2025 18:00:31 +0000 Subject: [PATCH 46/58] . --- .github/workflows/e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index ada56b16..f4f3d66e 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -26,7 +26,7 @@ jobs: - name: Build wasm run: | cargo install wasm-pack - cd backend/sync_lib + cd backend wasm-pack build --target web sync_lib - name: E2E tests From 7be14544601bfc35cbb20e79faf50804cb593bba Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 16 Mar 2025 18:17:45 +0000 Subject: [PATCH 47/58] try --- .github/workflows/e2e.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index f4f3d66e..9ff8c248 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -31,6 +31,7 @@ jobs: - name: E2E tests run: | - UST_BACKTRACE=1 cargo run -p sync_server & + pwd + RUST_BACKTRACE=1 cargo run -p sync_server & cd .. scripts/e2e.sh 32 From 74c007be2537d3f9c80bd27f4c770b9fb5d5a862 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 16 Mar 2025 18:19:36 +0000 Subject: [PATCH 48/58] Consistent ordering --- .../reconcile/src/operation_transformation.rs | 2 +- .../operation_transformation/edited_text.rs | 13 +++++-- .../src/operation_transformation/operation.rs | 34 +------------------ backend/sync_lib/tests/web.rs | 4 +-- 4 files changed, 15 insertions(+), 38 deletions(-) diff --git a/backend/reconcile/src/operation_transformation.rs b/backend/reconcile/src/operation_transformation.rs index 30e32502..a71bc65a 100644 --- a/backend/reconcile/src/operation_transformation.rs +++ b/backend/reconcile/src/operation_transformation.rs @@ -160,7 +160,7 @@ mod test { "hi ", "hi there you ", "hi there my friend ", - "hi there you my friend ", + "hi there my friend you ", ); test_merge_both_ways("a", "a b c", "a b c d", "a b c d"); diff --git a/backend/reconcile/src/operation_transformation/edited_text.rs b/backend/reconcile/src/operation_transformation/edited_text.rs index d3ae3832..8a7013e4 100644 --- a/backend/reconcile/src/operation_transformation/edited_text.rs +++ b/backend/reconcile/src/operation_transformation/edited_text.rs @@ -211,7 +211,16 @@ where usize::from(matches!(operation.operation, Operation::Delete { .. })), // Make sure that the ordering is deterministic regardless which text // is left or right. - operation.operation.get_hash(), + match &operation.operation { + Operation::Insert { text, .. } => text + .iter() + .map(super::super::tokenizer::token::Token::original) + .collect::(), + Operation::Delete { + deleted_character_count, + .. + } => deleted_character_count.to_string(), + }, ) }, ) @@ -285,7 +294,7 @@ mod tests { let original = "hello world! ..."; let left = "Hello world! I'm Andras."; let right = "Hello world! How are you?"; - let expected = "Hello world! I'm Andras. How are you?"; + let expected = "Hello world! How are you? I'm Andras."; let operations_1 = EditedText::from_strings(original, left); let operations_2 = EditedText::from_strings(original, right); diff --git a/backend/reconcile/src/operation_transformation/operation.rs b/backend/reconcile/src/operation_transformation/operation.rs index 73fa6140..d0d285b0 100644 --- a/backend/reconcile/src/operation_transformation/operation.rs +++ b/backend/reconcile/src/operation_transformation/operation.rs @@ -1,8 +1,5 @@ use core::fmt::{Debug, Display}; -use std::{ - hash::{DefaultHasher, Hash, Hasher}, - ops::Range, -}; +use std::ops::Range; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; @@ -39,28 +36,6 @@ where }, } -impl Hash for Operation -where - T: PartialEq + Clone + std::fmt::Debug, -{ - fn hash(&self, state: &mut H) { - match self { - Operation::Insert { index, text } => { - index.hash(state); - text.iter().for_each(|token| token.original().hash(state)); - } - Operation::Delete { - index, - deleted_character_count, - .. - } => { - index.hash(state); - deleted_character_count.hash(state); - } - } - } -} - impl Operation where T: PartialEq + Clone + std::fmt::Debug, @@ -315,13 +290,6 @@ where } } } - - /// Gets the hash of the operation based on the indexes and original text. - pub fn get_hash(&self) -> u64 { - let mut hasher = DefaultHasher::new(); - self.hash(&mut hasher); - hasher.finish() - } } impl Display for Operation diff --git a/backend/sync_lib/tests/web.rs b/backend/sync_lib/tests/web.rs index eca585b6..e45cbea6 100644 --- a/backend/sync_lib/tests/web.rs +++ b/backend/sync_lib/tests/web.rs @@ -25,9 +25,9 @@ fn test_base64_to_bytes_error() { #[wasm_bindgen_test(unsupported = test)] fn merge_text() { let left = b"hello "; - let right = b"world "; + let right = b"world"; let result = merge(b"", left, right); - assert!(result == b"hello world ".to_vec() || result == b"world hello ".to_vec()); + assert_eq!(result, b"hello world"); } #[wasm_bindgen_test(unsupported = test)] From f0bdecf44714eb7b8878b57c212cb644edf8c776 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 16 Mar 2025 18:19:40 +0000 Subject: [PATCH 49/58] Fix tests --- .../src/file-operations/file-operations.test.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/frontend/sync-client/src/file-operations/file-operations.test.ts b/frontend/sync-client/src/file-operations/file-operations.test.ts index e0514ae8..43308f8c 100644 --- a/frontend/sync-client/src/file-operations/file-operations.test.ts +++ b/frontend/sync-client/src/file-operations/file-operations.test.ts @@ -9,12 +9,20 @@ import { assertSetContainsExactly } from "../utils/assert-set-contains-exactly"; import type { FileSystemOperations } from "./filesystem-operations"; describe("File operations", () => { - class MockDatabase { + class MockDatabase implements Partial { public getLatestDocumentByRelativePath( _find: RelativePath ): DocumentRecord | undefined { + // no-op return undefined; } + + public move( + _oldRelativePath: RelativePath, + _newRelativePath: RelativePath + ): void { + // no-op + } } class FakeFileSystemOperations implements FileSystemOperations { From 1bae310cba5f5cbc7ce79a88c87b06d363732eed Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 16 Mar 2025 18:35:57 +0000 Subject: [PATCH 50/58] hmm --- .github/workflows/e2e.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 9ff8c248..85b9feaf 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -32,6 +32,8 @@ jobs: - name: E2E tests run: | pwd + ls -lah + cd backend RUST_BACKTRACE=1 cargo run -p sync_server & cd .. scripts/e2e.sh 32 From 8d695999c639ab067f63e25a3802dc57cbf69d8d Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 16 Mar 2025 19:41:47 +0000 Subject: [PATCH 51/58] . --- .github/workflows/e2e.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 85b9feaf..403449c6 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -31,9 +31,8 @@ jobs: - name: E2E tests run: | - pwd - ls -lah cd backend RUST_BACKTRACE=1 cargo run -p sync_server & cd .. + npm ci scripts/e2e.sh 32 From 10568efebecbec703541378c0035ca31e045401a Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 16 Mar 2025 19:49:09 +0000 Subject: [PATCH 52/58] Clean up diff --- backend/sync_server/src/database.rs | 2 -- .../src/file-operations/file-operations.ts | 2 +- frontend/sync-client/src/services/types.ts | 9 +++-- .../sync-client/src/sync-operations/syncer.ts | 26 +------------- frontend/test-client/src/agent/mock-agent.ts | 7 ++-- frontend/test-client/src/cli.ts | 36 +++++++++---------- 6 files changed, 25 insertions(+), 57 deletions(-) diff --git a/backend/sync_server/src/database.rs b/backend/sync_server/src/database.rs index 123f1c8a..882bd0a2 100644 --- a/backend/sync_server/src/database.rs +++ b/backend/sync_server/src/database.rs @@ -23,7 +23,6 @@ pub type Transaction<'a> = sqlx::Transaction<'a, Sqlite>; impl Database { pub async fn try_new(config: &DatabaseConfig) -> Result { - // Create the databases directory if it doesn't exist tokio::fs::create_dir_all(&config.databases_directory_path) .await .with_context(|| { @@ -67,7 +66,6 @@ impl Database { .databases_directory_path .join(format!("{vault}.sqlite")); - // Continue with database connection setup let connection_options = SqliteConnectOptions::new() .filename(file_name.clone()) .create_if_missing(true) diff --git a/frontend/sync-client/src/file-operations/file-operations.ts b/frontend/sync-client/src/file-operations/file-operations.ts index 01084647..b198caa4 100644 --- a/frontend/sync-client/src/file-operations/file-operations.ts +++ b/frontend/sync-client/src/file-operations/file-operations.ts @@ -107,7 +107,7 @@ export class FileOperations { currentText = currentText.replace(/\r\n/g, "\n"); if (currentText !== expectedText) { this.logger.debug( - `Performing a 3-way merge for ${path} with the expected content:\n${expectedText}` + `Performing a 3-way merge for ${path} with the expected content` ); return mergeText(expectedText, currentText, newText); diff --git a/frontend/sync-client/src/services/types.ts b/frontend/sync-client/src/services/types.ts index 5c464075..e8a954f3 100644 --- a/frontend/sync-client/src/services/types.ts +++ b/frontend/sync-client/src/services/types.ts @@ -452,7 +452,10 @@ export interface components { Array_of_uint8: number[]; CreateDocumentVersion: { contentBase64: string; - /** Format: uuid */ + /** + * Format: uuid + * @description The client can decide the document id (if it wishes to) in order to help with syncing. If the client does not provide a document id, the server will generate one. If the client provides a document id it must not already exist in the database. + */ documentId?: string | null; relativePath: string; }; @@ -476,7 +479,6 @@ export interface components { type: "FastForwardUpdate"; /** Format: date-time */ updatedDate: string; - vaultId: string; /** Format: int64 */ vaultUpdateId: number; } @@ -490,7 +492,6 @@ export interface components { type: "MergingUpdate"; /** Format: date-time */ updatedDate: string; - vaultId: string; /** Format: int64 */ vaultUpdateId: number; }; @@ -502,7 +503,6 @@ export interface components { relativePath: string; /** Format: date-time */ updatedDate: string; - vaultId: string; /** Format: int64 */ vaultUpdateId: number; }; @@ -513,7 +513,6 @@ export interface components { relativePath: string; /** Format: date-time */ updatedDate: string; - vaultId: string; /** Format: int64 */ vaultUpdateId: number; }; diff --git a/frontend/sync-client/src/sync-operations/syncer.ts b/frontend/sync-client/src/sync-operations/syncer.ts index e882004f..70ba88d5 100644 --- a/frontend/sync-client/src/sync-operations/syncer.ts +++ b/frontend/sync-client/src/sync-operations/syncer.ts @@ -60,24 +60,6 @@ export class Syncer { ); } - private static async forgivingFileNotFoundWrapper( - fn: () => Promise, - logger: Logger - ): Promise { - try { - return await fn(); - } catch (e) { - if (e instanceof FileNotFoundError) { - logger.debug( - `File has been deleted or moved before we had a chance to inspect it, skipping` - ); - return undefined; - } - - throw e; - } - } - public addRemainingOperationsListener( listener: (remainingOperations: number) => void ): void { @@ -355,13 +337,7 @@ export class Syncer { // Perhaps the file has been moved; let's check by looking at the deleted files const contentHash = await this.syncQueue.add(async () => { const contentBytes = - await Syncer.forgivingFileNotFoundWrapper( - async () => this.operations.read(relativePath), - this.logger - ); - if (contentBytes === undefined) { - return; - } + await this.operations.read(relativePath); // this can throw FileNotFoundError return hash(contentBytes); }); diff --git a/frontend/test-client/src/agent/mock-agent.ts b/frontend/test-client/src/agent/mock-agent.ts index 2dc4ec99..7713b524 100644 --- a/frontend/test-client/src/agent/mock-agent.ts +++ b/frontend/test-client/src/agent/mock-agent.ts @@ -251,10 +251,7 @@ export class MockAgent extends MockClient { `Decided to create file ${file} with content ${content}` ); - return this.create( - file, - new TextEncoder().encode(` |${content}| `) - ); + return this.create(file, new TextEncoder().encode(` ${content} `)); } private async changeFetchChangesUpdateIntervalMsAction(): Promise { @@ -328,7 +325,7 @@ export class MockAgent extends MockClient { `Decided to update file ${file} with ${content}` ); this.doNotTouchWhileOffline.push(file); - await this.atomicUpdateText(file, (old) => old + ` |${content}| `); + await this.atomicUpdateText(file, (old) => old + ` ${content} `); } private async deleteFileAction(files: RelativePath[]): Promise { diff --git a/frontend/test-client/src/cli.ts b/frontend/test-client/src/cli.ts index 85fb3c43..326ebb99 100644 --- a/frontend/test-client/src/cli.ts +++ b/frontend/test-client/src/cli.ts @@ -46,10 +46,6 @@ async function runTest({ ); } - // for debugging - // eslint-disable-next-line - (globalThis as any).clients = clients; - try { await Promise.all(clients.map(async (client) => client.init())); @@ -109,20 +105,22 @@ async function runTest({ } async function runTests(): Promise { - for (const concurrency of [ - 16, - 1 // test with concurrency 1 to check for deadlocks - ]) { - for (const doDeletes of [true, false]) { - for (const useSlowFileEvents of [true, false]) { - await runTest({ - agentCount: 4, - concurrency, - iterations: 200, - doDeletes, - useSlowFileEvents, - jitterScaleInSeconds: 0.75 - }); + for (const useSlowFileEvents of [false, true]) { + for (const concurrency of [ + 16, + 1 // test with concurrency 1 to check for deadlocks + ]) { + for (const doDeletes of [true, false]) { + for (let i = 0; i < 4; i++) { + await runTest({ + agentCount: 2, + concurrency, + iterations: 200, + doDeletes, + useSlowFileEvents, + jitterScaleInSeconds: 0.75 + }); + } } } } @@ -148,7 +146,7 @@ runTests() .then(() => { process.exit(0); }) - .catch(async (err: unknown) => { + .catch((err: unknown) => { console.error(err); process.exit(1); }); From 7f6fe8a582e96da99616c5a6f41baebb20302d4a Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 16 Mar 2025 20:01:43 +0000 Subject: [PATCH 53/58] Fixes --- frontend/obsidian-plugin/webpack.config.js | 3 --- .../sync-client/src/services/sync-service.ts | 2 +- .../src/sync-operations/unrestricted-syncer.ts | 10 +++++++--- frontend/sync-client/webpack.config.js | 3 +-- frontend/test-client/src/cli.ts | 18 ++++++++---------- 5 files changed, 17 insertions(+), 19 deletions(-) diff --git a/frontend/obsidian-plugin/webpack.config.js b/frontend/obsidian-plugin/webpack.config.js index 66c35d63..75c87fb7 100644 --- a/frontend/obsidian-plugin/webpack.config.js +++ b/frontend/obsidian-plugin/webpack.config.js @@ -94,9 +94,6 @@ module.exports = (env, argv) => ({ alias: { root: __dirname, src: path.resolve(__dirname, "src") - }, - fallback: { - url: require.resolve("url") } }, output: { diff --git a/frontend/sync-client/src/services/sync-service.ts b/frontend/sync-client/src/services/sync-service.ts index 92455643..74954cf3 100644 --- a/frontend/sync-client/src/services/sync-service.ts +++ b/frontend/sync-client/src/services/sync-service.ts @@ -112,7 +112,7 @@ export class SyncService { contentBytes: Uint8Array; }): Promise { this.logger.debug( - `Updating document ${documentId} with parent version ${parentVersionId} & ${new TextDecoder().decode(contentBytes)} & ${relativePath}` + `Updating document ${documentId} with parent version ${parentVersionId} and relative path ${relativePath}` ); const formData = new FormData(); formData.append("parent_version_id", parentVersionId.toString()); diff --git a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts index e38af28c..fe268f4d 100644 --- a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts +++ b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts @@ -105,9 +105,11 @@ export class UnrestrictedSyncer { public async unrestrictedSyncLocallyUpdatedFile({ oldPath, - document + document, + force = false }: { oldPath?: RelativePath; + force?: boolean; document: DocumentRecord; }): Promise { await this.executeSync( @@ -131,7 +133,8 @@ export class UnrestrictedSyncer { if ( document.metadata.hash === contentHash && - oldPath === undefined + oldPath === undefined && + !force ) { this.logger.debug( `File hash of ${document.relativePath} matches with last synced version and the path hasn't changed; no need to sync` @@ -270,7 +273,8 @@ export class UnrestrictedSyncer { } return this.unrestrictedSyncLocallyUpdatedFile({ - document + document, + force: true }); } else if (remoteVersion.isDeleted) { // Either the doc hasn't made it to us before and therefore we don't need to delete it, diff --git a/frontend/sync-client/webpack.config.js b/frontend/sync-client/webpack.config.js index 609d2533..3f913041 100644 --- a/frontend/sync-client/webpack.config.js +++ b/frontend/sync-client/webpack.config.js @@ -39,8 +39,7 @@ module.exports = [ filename: "sync-client.web.js", library: { name: "SyncClient", - type: "umd", - export: "default" + type: "umd" }, globalObject: "this" } diff --git a/frontend/test-client/src/cli.ts b/frontend/test-client/src/cli.ts index 326ebb99..ea7ebbd0 100644 --- a/frontend/test-client/src/cli.ts +++ b/frontend/test-client/src/cli.ts @@ -111,16 +111,14 @@ async function runTests(): Promise { 1 // test with concurrency 1 to check for deadlocks ]) { for (const doDeletes of [true, false]) { - for (let i = 0; i < 4; i++) { - await runTest({ - agentCount: 2, - concurrency, - iterations: 200, - doDeletes, - useSlowFileEvents, - jitterScaleInSeconds: 0.75 - }); - } + await runTest({ + agentCount: 3, + concurrency, + iterations: 100, + doDeletes, + useSlowFileEvents, + jitterScaleInSeconds: 0.75 + }); } } } From 8a36be7d56d448eeab1d308557df1e021aca7869 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 16 Mar 2025 20:02:35 +0000 Subject: [PATCH 54/58] . --- frontend/manifest.json => manifest.json | 0 scripts/bump-version.sh | 14 +++++++------- 2 files changed, 7 insertions(+), 7 deletions(-) rename frontend/manifest.json => manifest.json (100%) diff --git a/frontend/manifest.json b/manifest.json similarity index 100% rename from frontend/manifest.json rename to manifest.json diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh index b17842be..b2e715e1 100755 --- a/scripts/bump-version.sh +++ b/scripts/bump-version.sh @@ -41,12 +41,12 @@ cd .. cp plugin/manifest.json manifest.json # for BRAT, otherwise it wouldn't update # Commit and tag -git add . -TAG=$(node -p "require('./plugin/package.json').version") -git commit -m "Bump versions to $TAG" +# git add . +# TAG=$(node -p "require('./plugin/package.json').version") +# git commit -m "Bump versions to $TAG" -git push -echo "Tagging $TAG" -git tag -a $TAG -m "Release $TAG" -git push origin $TAG +# git push +# echo "Tagging $TAG" +# git tag -a $TAG -m "Release $TAG" +# git push origin $TAG echo "Done" From 2bf5223275c89351128a3c7f4a059545229ca705 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 16 Mar 2025 20:08:02 +0000 Subject: [PATCH 55/58] Fix version bump --- frontend/sync-client/package.json | 4 ++-- frontend/test-client/package.json | 4 ++-- scripts/bump-version.sh | 24 ++++++++++++------------ 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/frontend/sync-client/package.json b/frontend/sync-client/package.json index 37e63bc8..f08b5f5a 100644 --- a/frontend/sync-client/package.json +++ b/frontend/sync-client/package.json @@ -1,6 +1,6 @@ { "name": "sync-client", - "version": "0.0.0", + "version": "0.0.30", "main": "dist/sync-client.node.js", "browser": "dist/sync-client.web.js", "types": "dist/types/index.d.ts", @@ -33,4 +33,4 @@ "webpack-merge": "^6.0.1", "sync_lib": "file:../../backend/sync_lib/pkg" } -} +} \ No newline at end of file diff --git a/frontend/test-client/package.json b/frontend/test-client/package.json index 88877204..b11a4c41 100644 --- a/frontend/test-client/package.json +++ b/frontend/test-client/package.json @@ -1,6 +1,6 @@ { "name": "test-client", - "version": "0.0.0", + "version": "0.0.30", "private": true, "bin": { "test-client": "./dist/cli.js" @@ -20,4 +20,4 @@ "webpack": "^5.98.0", "webpack-cli": "^6.0.1" } -} +} \ No newline at end of file diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh index b2e715e1..c8de4839 100755 --- a/scripts/bump-version.sh +++ b/scripts/bump-version.sh @@ -27,26 +27,26 @@ cd backend cargo set-version --bump patch echo "Bumping frontend versions" -cd ../plugin -npm version patch +cd ../frontend +npm version patch --workspaces echo "Updating frontend dependencies to match the new backend versions" cd ../backend/sync_lib wasm-pack build --target web --features console_error_panic_hook -cd ../../plugin +cd ../../frontend npm install cd .. -cp plugin/manifest.json manifest.json # for BRAT, otherwise it wouldn't update +cp frontend/obsidian-plugin/manifest.json manifest.json # for BRAT, otherwise it wouldn't update -# Commit and tag -# git add . -# TAG=$(node -p "require('./plugin/package.json').version") -# git commit -m "Bump versions to $TAG" +Commit and tag +git add . +TAG=$(node -p "require('./plugin/package.json').version") +git commit -m "Bump versions to $TAG" -# git push -# echo "Tagging $TAG" -# git tag -a $TAG -m "Release $TAG" -# git push origin $TAG +git push +echo "Tagging $TAG" +git tag -a $TAG -m "Release $TAG" +git push origin $TAG echo "Done" From 4c2d5e90d075fef08a8e1dcb6fa3b969d6c98419 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 16 Mar 2025 20:08:58 +0000 Subject: [PATCH 56/58] . --- .github/workflows/e2e.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 403449c6..aa4fa803 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -33,6 +33,7 @@ jobs: run: | cd backend RUST_BACKTRACE=1 cargo run -p sync_server & - cd .. + cd ../frontend npm ci + cd .. scripts/e2e.sh 32 From 4bad00fe54b908e54ba2b7b0fd09c967d8480a67 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 16 Mar 2025 20:11:52 +0000 Subject: [PATCH 57/58] . --- .github/workflows/e2e.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index aa4fa803..b9fb32ad 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -23,10 +23,15 @@ jobs: node-version: "22.x" check-latest: true + - name: Setup rust + run: | + cargo install sqlx-cli wasm-pack + cd backend + sqlx database create --database-url sqlite://db.sqlite3 + sqlx migrate run --source sync_server/src/database/migrations --database-url sqlite://db.sqlite3 + - name: Build wasm run: | - cargo install wasm-pack - cd backend wasm-pack build --target web sync_lib - name: E2E tests From f314150ff3df231c8aa6a655b2cc831c0d911535 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 16 Mar 2025 20:12:55 +0000 Subject: [PATCH 58/58] . --- .github/workflows/check.yml | 2 +- .github/workflows/e2e.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 2bd76f29..5acccc1f 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -23,7 +23,7 @@ jobs: node-version: "22.x" check-latest: true - - name: Setup + - name: Setup rust run: | cargo install sqlx-cli cd backend diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index b9fb32ad..dd2fe5d9 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -32,6 +32,7 @@ jobs: - name: Build wasm run: | + cd backend wasm-pack build --target web sync_lib - name: E2E tests