From c4f992c9d6c76311e94d00b7984b83dd49dcdf1a Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 14 Dec 2025 23:30:04 +0000 Subject: [PATCH] wip --- frontend/sync-client/src/consts.ts | 4 +- .../src/file-operations/file-operations.ts | 2 +- .../sync-client/src/persistence/database.ts | 31 +++---- .../sync-client/src/services/server-config.ts | 10 +-- .../sync-client/src/services/sync-service.ts | 24 ++--- .../src/services/websocket-manager.ts | 8 +- frontend/sync-client/src/sync-client.ts | 5 +- .../src/sync-operations/cursor-tracker.ts | 2 +- .../sync-client/src/sync-operations/syncer.ts | 62 ++++--------- .../sync-operations/unrestricted-syncer.ts | 87 +++++++++++-------- frontend/sync-client/src/utils/await-all.ts | 2 +- rustfmt.toml | 11 +++ scripts/bump-version.sh | 3 +- scripts/check.sh | 9 +- scripts/update-api-types.sh | 6 +- sync-server/config-e2e.yml | 32 +++---- sync-server/src/consts.rs | 2 +- sync-server/src/errors.rs | 2 +- sync-server/src/server/create_document.rs | 57 +++++++----- sync-server/src/server/requests.rs | 10 +-- sync-server/src/server/update_document.rs | 57 +++++++++--- 21 files changed, 233 insertions(+), 193 deletions(-) create mode 100644 rustfmt.toml diff --git a/frontend/sync-client/src/consts.ts b/frontend/sync-client/src/consts.ts index da70ba47..e0a2d60e 100644 --- a/frontend/sync-client/src/consts.ts +++ b/frontend/sync-client/src/consts.ts @@ -2,5 +2,5 @@ export const TIMEOUT_FOR_MERGING_HISTORY_ENTRIES_IN_SECONDS = 60; export const DIFF_CACHE_SIZE_MB = 2; export const MAX_LOG_MESSAGE_COUNT = 100000; export const MAX_HISTORY_ENTRY_COUNT = 5000; -export const SUPPORTED_API_VERSION = 2; -export const WEBSOCKET_DISCONNECT_TIMEOUT_IN_S = 10; +export const SUPPORTED_API_VERSION = 3; +export const WEBSOCKET_DISCONNECT_TIMEOUT_IN_SECONDS = 10; diff --git a/frontend/sync-client/src/file-operations/file-operations.ts b/frontend/sync-client/src/file-operations/file-operations.ts index fdf65d35..863f62af 100644 --- a/frontend/sync-client/src/file-operations/file-operations.ts +++ b/frontend/sync-client/src/file-operations/file-operations.ts @@ -169,9 +169,9 @@ export class FileOperations { } await this.ensureClearPath(newPath); - this.database.move(oldPath, newPath); await this.fs.rename(oldPath, newPath); + await this.deletingEmptyParentDirectoriesOfDeletedFile(oldPath); } diff --git a/frontend/sync-client/src/persistence/database.ts b/frontend/sync-client/src/persistence/database.ts index 04e0fce6..5b4e943b 100644 --- a/frontend/sync-client/src/persistence/database.ts +++ b/frontend/sync-client/src/persistence/database.ts @@ -9,6 +9,7 @@ export type DocumentId = string; export type RelativePath = string; export interface DocumentMetadata { + documentId: DocumentId; parentVersionId: VaultUpdateId; hash: string; remoteRelativePath?: RelativePath; @@ -36,7 +37,6 @@ export interface StoredDatabase { */ export interface DocumentRecord { relativePath: RelativePath; - documentId: DocumentId; metadata: DocumentMetadata | undefined; isDeleted: boolean; updates: Promise[]; @@ -57,9 +57,8 @@ export class Database { this.documents = initialState.documents?.map( - ({ relativePath, documentId, ...metadata }) => ({ + ({ relativePath, ...metadata }) => ({ relativePath, - documentId, metadata, isDeleted: false, updates: [], @@ -114,7 +113,7 @@ export class Database { i === 0 ? false : records[i - 1].parallelVersion === - current.parallelVersion + current.parallelVersion ) ) { throw new Error( @@ -127,6 +126,7 @@ export class Database { public updateDocumentMetadata( metadata: { + documentId: DocumentId; parentVersionId: VaultUpdateId; hash: string; remoteRelativePath: RelativePath; @@ -196,19 +196,18 @@ export class Database { } public createNewPendingDocument( - documentId: DocumentId, relativePath: RelativePath, promise: Promise ): DocumentRecord { this.logger.debug( - `Creating new pending document: ${relativePath} (${documentId})` + `Creating new pending document: ${relativePath}` ); const previousEntry = this.getLatestDocumentByRelativePath(relativePath); const entry = { relativePath, - documentId, + documentId: undefined, metadata: undefined, isDeleted: false, updates: [promise], @@ -231,8 +230,8 @@ export class Database { ): DocumentRecord { const entry = { relativePath, - documentId, metadata: { + documentId, parentVersionId, hash: EMPTY_HASH, remoteRelativePath: relativePath @@ -251,7 +250,7 @@ export class Database { public getDocumentByDocumentId( find: DocumentId ): DocumentRecord | undefined { - return this.documents.find(({ documentId }) => documentId === find); + return this.documents.find(({ metadata }) => metadata?.documentId === find); } public move( @@ -331,8 +330,7 @@ export class Database { public async save(): Promise { return this.saveData({ documents: this.resolvedDocuments.map( - ({ relativePath, documentId, metadata }) => ({ - documentId, + ({ relativePath, metadata }) => ({ relativePath, // eslint-disable-next-line @typescript-eslint/no-non-null-assertion ...metadata! // `resolvedDocuments` only returns docs with metadata set @@ -346,9 +344,12 @@ export class Database { private ensureConsistency(): void { const idToPath = new Map(); - this.resolvedDocuments.forEach(({ relativePath, documentId }) => { - idToPath.set(documentId, [ - ...(idToPath.get(documentId) ?? []), + this.resolvedDocuments.forEach(({ relativePath, metadata }) => { + if (metadata === undefined) { + return; + } + idToPath.set(metadata.documentId, [ + ...(idToPath.get(metadata.documentId) ?? []), relativePath ]); }); @@ -360,7 +361,7 @@ export class Database { if (duplicates.length > 0) { throw new Error( "Document IDs are not unique, found duplicates: " + - duplicates.join("; ") + duplicates.join("; ") ); } } diff --git a/frontend/sync-client/src/services/server-config.ts b/frontend/sync-client/src/services/server-config.ts index b48e9802..f19b3df8 100644 --- a/frontend/sync-client/src/services/server-config.ts +++ b/frontend/sync-client/src/services/server-config.ts @@ -14,15 +14,14 @@ export class ServerConfig { private response: Promise | undefined; private config: ServerConfigData | undefined; - public constructor(private readonly syncService: SyncService) {} + public constructor(private readonly syncService: SyncService) { } private static validateConfig(config: ServerConfigData): void { if (config.supportedApiVersion !== SUPPORTED_API_VERSION) { const shouldUpgradeClient = config.supportedApiVersion > SUPPORTED_API_VERSION; throw new ServerVersionMismatchError( - `Unsupported API version: ${config.supportedApiVersion}. Consider upgrading the ${ - shouldUpgradeClient ? "client" : "sync-server" + `Unsupported API version: ${config.supportedApiVersion}. Consider upgrading the ${shouldUpgradeClient ? "client" : "sync-server" } to ensure compatibility` ); } @@ -34,11 +33,6 @@ export class ServerConfig { } } - // warm the cache - public async initialize(): Promise { - await this.getConfig(); - } - public async checkConnection(forceUpdate = false): Promise<{ isSuccessful: boolean; message: string; diff --git a/frontend/sync-client/src/services/sync-service.ts b/frontend/sync-client/src/services/sync-service.ts index 8dd0de68..e2259876 100644 --- a/frontend/sync-client/src/services/sync-service.ts +++ b/frontend/sync-client/src/services/sync-service.ts @@ -66,27 +66,29 @@ export class SyncService { } public async create({ - documentId, relativePath, - contentBytes + contentBytes, + forceMerge }: { - documentId?: DocumentId; relativePath: RelativePath; contentBytes: Uint8Array; + forceMerge?: boolean; }): Promise { return this.retryForever(async () => { const formData = new FormData(); - if (documentId !== undefined) { - formData.append("document_id", documentId); - } + formData.append("relative_path", relativePath); + if (forceMerge === true) { + formData.append("force_merge", "true"); + } + formData.append( "content", new Blob([new Uint8Array(contentBytes)]) ); this.logger.debug( - `Creating document with id ${documentId} and relative path ${relativePath}` + `Creating document with relative path ${relativePath} (forceMerge: ${forceMerge})` ); const response = await this.client(this.getUrl("/documents"), { @@ -155,8 +157,7 @@ export class SyncService { (await response.json()) as DocumentUpdateResponse; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion this.logger.debug( - `Updated document ${JSON.stringify(result)} with id ${ - result.documentId + `Updated document ${JSON.stringify(result)} with id ${result.documentId }}` ); @@ -208,8 +209,7 @@ export class SyncService { (await response.json()) as DocumentUpdateResponse; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion this.logger.debug( - `Updated document ${JSON.stringify(result)} with id ${ - result.documentId + `Updated document ${JSON.stringify(result)} with id ${result.documentId }}` ); @@ -336,7 +336,7 @@ export class SyncService { return this.retryForever(async () => { this.logger.debug( "Getting all documents" + - (since != null ? ` since ${since}` : "") + (since != null ? ` since ${since}` : "") ); const url = new URL(this.getUrl("/documents")); diff --git a/frontend/sync-client/src/services/websocket-manager.ts b/frontend/sync-client/src/services/websocket-manager.ts index 4f47fcbe..bbbe8dfe 100644 --- a/frontend/sync-client/src/services/websocket-manager.ts +++ b/frontend/sync-client/src/services/websocket-manager.ts @@ -6,7 +6,7 @@ import type { CursorPositionFromClient } from "./types/CursorPositionFromClient" import type { ClientCursors } from "./types/ClientCursors"; import { createPromise } from "../utils/create-promise"; import type { WebSocketVaultUpdate } from "./types/WebSocketVaultUpdate"; -import { WEBSOCKET_DISCONNECT_TIMEOUT_IN_S } from "../consts"; +import { WEBSOCKET_DISCONNECT_TIMEOUT_IN_SECONDS } from "../consts"; import { removeFromArray } from "../utils/remove-from-array"; import { EventListeners } from "../utils/data-structures/event-listeners"; import { awaitAll } from "../utils/await-all"; @@ -36,7 +36,7 @@ export class WebSocketManager { private readonly logger: Logger, private readonly settings: Settings, private readonly webSocketFactoryImplementation: typeof globalThis.WebSocket = WebSocket - ) {} + ) { } public get isWebSocketConnected(): boolean { return ( @@ -69,10 +69,10 @@ export class WebSocketManager { timeoutId = setTimeout(() => { reject( new Error( - `Timeout waiting for WebSocket to close after ${WEBSOCKET_DISCONNECT_TIMEOUT_IN_S} seconds` + `Timeout waiting for WebSocket to close after ${WEBSOCKET_DISCONNECT_TIMEOUT_IN_SECONDS} seconds` ) ); - }, WEBSOCKET_DISCONNECT_TIMEOUT_IN_S * 1000); + }, WEBSOCKET_DISCONNECT_TIMEOUT_IN_SECONDS * 1000); }); try { diff --git a/frontend/sync-client/src/sync-client.ts b/frontend/sync-client/src/sync-client.ts index 5c427d7e..f9db6dc5 100644 --- a/frontend/sync-client/src/sync-client.ts +++ b/frontend/sync-client/src/sync-client.ts @@ -56,7 +56,7 @@ export class SyncClient { database: Partial; }> > - ) {} + ) { } public get documentCount(): number { return this.database.length; @@ -472,7 +472,8 @@ export class SyncClient { this.checkIfDestroyed("startSyncing"); this.fetchController.finishReset(); - await this.serverConfig.initialize(); + // warm the cache + await this.serverConfig.getConfig(); this.webSocketManager.start(); if (!this.hasStartedOfflineSync) { diff --git a/frontend/sync-client/src/sync-operations/cursor-tracker.ts b/frontend/sync-client/src/sync-operations/cursor-tracker.ts index bdd7d9b7..48b8908a 100644 --- a/frontend/sync-client/src/sync-operations/cursor-tracker.ts +++ b/frontend/sync-client/src/sync-operations/cursor-tracker.ts @@ -113,7 +113,7 @@ export class CursorTracker { documentsWithCursors.push({ relative_path: relativePath, - document_id: record.documentId, + document_id: record.metadata.documentId, vault_update_id: record.metadata.parentVersionId, cursors: cursors.map(({ start, end }) => ({ start: Math.min(start, end), diff --git a/frontend/sync-client/src/sync-operations/syncer.ts b/frontend/sync-client/src/sync-operations/syncer.ts index 5b6993f9..01bba387 100644 --- a/frontend/sync-client/src/sync-operations/syncer.ts +++ b/frontend/sync-client/src/sync-operations/syncer.ts @@ -8,13 +8,12 @@ import type { SyncService } from "../services/sync-service"; import type { Logger } from "../tracing/logger"; import PQueue from "p-queue"; import { hash } from "../utils/hash"; -import { v4 as uuidv4 } from "uuid"; import type { Settings } from "../persistence/settings"; import type { FileOperations } from "../file-operations/file-operations"; import { findMatchingFile } from "../utils/find-matching-file"; import type { UnrestrictedSyncer } from "./unrestricted-syncer"; import { createPromise } from "../utils/create-promise"; -import { SyncResetError } from "../services/sync-reset-error"; +import { SyncResetError } from "../errors/sync-reset-error"; import { Locks } from "../utils/data-structures/locks"; import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent"; import type { WebSocketVaultUpdate } from "../services/types/WebSocketVaultUpdate"; @@ -98,9 +97,7 @@ export class Syncer { const [promise, resolve, reject] = createPromise(); - const id = uuidv4(); const document = this.database.createNewPendingDocument( - id, relativePath, promise ); @@ -171,7 +168,7 @@ export class Syncer { // in that case, we mustn't move it again. if ( this.database.getLatestDocumentByRelativePath(relativePath) === - undefined || + undefined || this.database.getLatestDocumentByRelativePath(relativePath) ?.isDeleted === true ) { @@ -391,8 +388,6 @@ export class Syncer { } private async internalScheduleSyncForOfflineChanges(): Promise { - await this.createFakeDocumentsFromRemoteState(); - const allLocalFiles = await this.operations.listFilesRecursively(); this.logger.info( `Scheduling sync for ${allLocalFiles.length} local files` @@ -426,9 +421,19 @@ export class Syncer { // Perhaps the file has been moved; let's check by looking at the deleted files const contentHash = await this.syncQueue.add(async () => { - const contentBytes = - await this.operations.read(relativePath); // this can throw FileNotFoundError - return hash(contentBytes); + try { + const contentBytes = + await this.operations.read(relativePath); // this can throw FileNotFoundError + return hash(contentBytes); + } catch (e) { + if ( + e instanceof Error && + e.name === "FileNotFoundError" + ) { + return undefined; + } + throw e; + } }); if (contentHash == undefined) { @@ -481,42 +486,9 @@ export class Syncer { return this.syncLocallyDeletedFile(relativePath); }) ); - } - - /** - * Create fake documents in the database for all files that are present locally - * and also exist remotely. This will stop the subequent syncs from duplicating - * the documents by creating the same documents from multiple clients. - */ - private async createFakeDocumentsFromRemoteState(): Promise { - if (this.database.getHasInitialSyncCompleted()) { - return; - } - - const [allLocalFiles, remote] = await awaitAll([ - this.operations.listFilesRecursively(), - this.syncQueue.add(async () => this.syncService.getAll()) - ]); - - if (remote !== undefined) { - remote.latestDocuments - .filter( - (remoteDocument) => - allLocalFiles.includes(remoteDocument.relativePath) && - !remoteDocument.isDeleted && - this.database.getDocumentByDocumentId( - remoteDocument.documentId - ) === undefined - ) - .forEach((remoteDocument) => { - this.database.createNewEmptyDocument( - remoteDocument.documentId, - remoteDocument.vaultUpdateId, - remoteDocument.relativePath - ); - }); - } this.database.setHasInitialSyncCompleted(true); } + + } diff --git a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts index 5b80d75b..f277f637 100644 --- a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts +++ b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts @@ -82,9 +82,9 @@ export class UnrestrictedSyncer { const contentHash = hash(contentBytes); const response = await this.syncService.create({ - documentId: document.documentId, relativePath: originalRelativePath, - contentBytes + contentBytes, + forceMerge: !this.database.getHasInitialSyncCompleted() // don't duplicate files on first sync }); // In case a document with the same name (but different ID) had existed remotely that we haven't known about @@ -100,6 +100,7 @@ export class UnrestrictedSyncer { this.database.updateDocumentMetadata( { + documentId: response.documentId, parentVersionId: response.vaultUpdateId, hash: contentHash, remoteRelativePath: response.relativePath @@ -131,13 +132,21 @@ export class UnrestrictedSyncer { }; await this.executeSync(updateDetails, async () => { + if (document.metadata === undefined) { + this.logger.debug( + `Document ${document.relativePath} has no metadata, so it was never synced remotely` + ); + return; + } + const response = await this.syncService.delete({ - documentId: document.documentId, + documentId: document.metadata.documentId, relativePath: document.relativePath }); this.database.updateDocumentMetadata( { + ...document.metadata, parentVersionId: response.vaultUpdateId, hash: EMPTY_HASH, remoteRelativePath: document.relativePath @@ -170,14 +179,14 @@ export class UnrestrictedSyncer { const updateDetails: SyncUpdateDetails | SyncMovedDetails = oldPath !== undefined ? { - type: SyncType.MOVE, - relativePath: document.relativePath, - movedFrom: oldPath - } + type: SyncType.MOVE, + relativePath: document.relativePath, + movedFrom: oldPath + } : { - type: SyncType.UPDATE, - relativePath: document.relativePath - }; + type: SyncType.UPDATE, + relativePath: document.relativePath + }; await this.executeSync(updateDetails, async () => { const originalRelativePath = document.relativePath; @@ -216,22 +225,22 @@ export class UnrestrictedSyncer { response = isText && cachedVersion !== undefined ? await this.syncService.putText({ - documentId: document.documentId, - parentVersionId: - document.metadata.parentVersionId, - relativePath: document.relativePath, - content: diff( - new TextDecoder().decode(cachedVersion), - new TextDecoder().decode(contentBytes) - ) - }) + documentId: document.metadata.documentId, + parentVersionId: + document.metadata.parentVersionId, + relativePath: document.relativePath, + content: diff( + new TextDecoder().decode(cachedVersion), + new TextDecoder().decode(contentBytes) + ) + }) : await this.syncService.putBinary({ - documentId: document.documentId, - parentVersionId: - document.metadata.parentVersionId, - relativePath: document.relativePath, - contentBytes - }); + documentId: document.metadata.documentId, + parentVersionId: + document.metadata.parentVersionId, + relativePath: document.relativePath, + contentBytes + }); } else { if (!force) { this.logger.debug( @@ -241,7 +250,7 @@ export class UnrestrictedSyncer { } response = await this.syncService.get({ - documentId: document.documentId + documentId: document.metadata.documentId }); } @@ -290,6 +299,7 @@ export class UnrestrictedSyncer { this.database.updateDocumentMetadata( { + ...document.metadata, parentVersionId: response.vaultUpdateId, hash: contentHash, remoteRelativePath: response.relativePath @@ -317,6 +327,7 @@ export class UnrestrictedSyncer { } else { this.database.updateDocumentMetadata( { + ...document.metadata, parentVersionId: response.vaultUpdateId, hash: contentHash, remoteRelativePath: response.relativePath @@ -334,16 +345,16 @@ export class UnrestrictedSyncer { const actualUpdateDetails: SyncUpdateDetails | SyncMovedDetails = oldPath !== undefined || - response.relativePath != originalRelativePath + response.relativePath != originalRelativePath ? { - type: SyncType.MOVE, - relativePath: response.relativePath, - movedFrom: originalRelativePath - } + type: SyncType.MOVE, + relativePath: response.relativePath, + movedFrom: originalRelativePath + } : { - type: SyncType.UPDATE, - relativePath: response.relativePath - }; + type: SyncType.UPDATE, + relativePath: response.relativePath + }; if (areThereLocalChanges) { this.history.addHistoryEntry({ @@ -437,12 +448,12 @@ export class UnrestrictedSyncer { const [promise, resolve] = createPromise(); this.database.updateDocumentMetadata( { + documentId: remoteVersion.documentId, parentVersionId: remoteVersion.vaultUpdateId, hash: hash(contentBytes), remoteRelativePath: remoteVersion.relativePath }, this.database.createNewPendingDocument( - remoteVersion.documentId, remoteVersion.relativePath, promise ) @@ -541,9 +552,8 @@ export class UnrestrictedSyncer { type: SyncType.SKIPPED, relativePath }, - message: `File size of ${sizeInMB} MB exceeds the maximum file size limit of ${ - maxFileSizeMB - } MB` + message: `File size of ${sizeInMB} MB exceeds the maximum file size limit of ${maxFileSizeMB + } MB` }; } } @@ -582,6 +592,7 @@ export class UnrestrictedSyncer { this.database.delete(document.relativePath); this.database.updateDocumentMetadata( { + documentId: response.documentId, parentVersionId: response.vaultUpdateId, hash: EMPTY_HASH, remoteRelativePath: response.relativePath diff --git a/frontend/sync-client/src/utils/await-all.ts b/frontend/sync-client/src/utils/await-all.ts index 9406a6b8..43e06ce6 100644 --- a/frontend/sync-client/src/utils/await-all.ts +++ b/frontend/sync-client/src/utils/await-all.ts @@ -9,7 +9,7 @@ type ResolvedTuple = { export const awaitAll = async ( promises: PromiseTuple ): Promise> => { - // eslint-disable-next-line no-restricted-properties + // eslint-disable-next-line no-restricted-properties, @typescript-eslint/await-thenable const result = await Promise.allSettled(promises); for (const res of result) { if (res.status === "rejected") { diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 00000000..a9107050 --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,11 @@ +# Rustfmt configuration +# This should match the .editorconfig settings + +# Use spaces for indentation (matches .editorconfig indent_style = space) +hard_tabs = false + +# Use 4 spaces for indentation (matches .editorconfig indent_size = 4) +tab_spaces = 4 + +# Use Unix line endings (matches .editorconfig end_of_line = lf) +newline_style = "Unix" diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh index fb953e2a..bea3d982 100755 --- a/scripts/bump-version.sh +++ b/scripts/bump-version.sh @@ -35,7 +35,8 @@ cd .. cp frontend/obsidian-plugin/manifest.json manifest.json # for BRAT, otherwise it wouldn't update -git ls-files | xargs npx eclint fix +# Format all files across the project (frontend and backend) +npx -C frontend prettier --write "**/*.{ts,js,json,md,yml,yaml}" # Commit and tag git add . diff --git a/scripts/check.sh b/scripts/check.sh index 7c3c87e5..f7a3aa57 100755 --- a/scripts/check.sh +++ b/scripts/check.sh @@ -45,10 +45,11 @@ cd frontend npm run build npm run test npm run lint +cd .. -# Use git ls-files to only check tracked files, respecting .gitignore -# We always run in fix mode and then check with git status -git ls-files | xargs npx eclint fix +# Format all files across the project (frontend and backend) +# Prettier respects .gitignore by default +npx -C frontend prettier --write "**/*.{ts,js,json,md,yml,yaml}" if [[ "$FIX_MODE" == false ]] && [[ $(git status --porcelain) ]]; then git status --porcelain @@ -56,6 +57,4 @@ if [[ "$FIX_MODE" == false ]] && [[ $(git status --porcelain) ]]; then exit 1 fi -cd .. - echo "Success" diff --git a/scripts/update-api-types.sh b/scripts/update-api-types.sh index 4b947ee8..36ca100d 100755 --- a/scripts/update-api-types.sh +++ b/scripts/update-api-types.sh @@ -12,5 +12,7 @@ cp -r sync-server/bindings/* frontend/sync-client/src/services/types/ cd frontend npm run lint -git ls-files | xargs npx eclint fix -cd - +cd .. + +# Format all files across the project (frontend and backend) +npx -C frontend prettier --write "**/*.{ts,js,json,md,yml,yaml}" diff --git a/sync-server/config-e2e.yml b/sync-server/config-e2e.yml index e9d47559..1f235b01 100644 --- a/sync-server/config-e2e.yml +++ b/sync-server/config-e2e.yml @@ -9,24 +9,24 @@ server: max_clients_per_vault: 256 response_timeout: 30m mergeable_file_extensions: - - md - - txt + - md + - txt users: user_configs: - - name: admin - token: test-token-change-me - vault_access: - type: allow_access_to_all - - name: other-admin - token: test-token-change-me2 - vault_access: - type: allow_access_to_all - - name: test - token: other-test-token - vault_access: - type: allow_list - allowed: - - default + - name: admin + token: test-token-change-me + vault_access: + type: allow_access_to_all + - name: other-admin + token: test-token-change-me2 + vault_access: + type: allow_access_to_all + - name: test + token: other-test-token + vault_access: + type: allow_list + allowed: + - default logging: log_directory: logs log_rotation: 7days diff --git a/sync-server/src/consts.rs b/sync-server/src/consts.rs index 98ed1c1f..9e9890c0 100644 --- a/sync-server/src/consts.rs +++ b/sync-server/src/consts.rs @@ -20,4 +20,4 @@ pub const DEFAULT_LOG_LEVEL: LogLevel = LogLevel::Info; pub const DEFAULT_MERGEABLE_FILE_EXTENSIONS: &[&str] = &["md", "txt"]; -pub const SUPPORTED_API_VERSION: u32 = 2; +pub const SUPPORTED_API_VERSION: u32 = 3; diff --git a/sync-server/src/errors.rs b/sync-server/src/errors.rs index 831b0e86..c505b8ae 100644 --- a/sync-server/src/errors.rs +++ b/sync-server/src/errors.rs @@ -5,7 +5,7 @@ use axum::{ http::StatusCode, response::{IntoResponse, Response}, }; -use log::{debug, error}; +use log::debug; use serde::Serialize; use thiserror::Error; use ts_rs::TS; diff --git a/sync-server/src/server/create_document.rs b/sync-server/src/server/create_document.rs index 859c0db4..20f67193 100644 --- a/sync-server/src/server/create_document.rs +++ b/sync-server/src/server/create_document.rs @@ -11,10 +11,11 @@ use super::{device_id_header::DeviceIdHeader, requests::CreateDocumentVersion}; use crate::{ app_state::{ AppState, - database::models::{DocumentVersionWithoutContent, StoredDocumentVersion, VaultId}, + database::models::{StoredDocumentVersion, VaultId}, }, config::user_config::User, - errors::{SyncServerError, client_error, server_error}, + errors::{SyncServerError, server_error}, + server::{responses::DocumentUpdateResponse, update_document::merge_with_stored_version}, utils::{ find_first_available_path::find_first_available_path, normalize::normalize, sanitize_path::sanitize_path, @@ -37,7 +38,7 @@ pub async fn create_document( TypedHeader(device_id): TypedHeader, State(state): State, TypedMultipart(request): TypedMultipart, -) -> Result, SyncServerError> { +) -> Result, SyncServerError> { debug!("Creating document in vault `{vault_id}`"); let mut transaction = state @@ -46,24 +47,39 @@ pub async fn create_document( .await .map_err(server_error)?; - let document_id = match request.document_id { - Some(document_id) => { - let existing_version = state - .database - .get_latest_document(&vault_id, &document_id, Some(&mut transaction)) - .await - .map_err(server_error)?; + let sanitized_relative_path = sanitize_path(&request.relative_path); - if existing_version.is_some() { - return Err(client_error(anyhow::anyhow!( - "Document with the same ID `{document_id}` already exists" - ))); - } + if request.force_merge.unwrap_or_default() { + let latest_version = state + .database + .get_latest_document_by_path( + &vault_id, + &sanitized_relative_path, + Some(&mut transaction), + ) + .await + .map_err(server_error)?; + if let Some(latest_version) = latest_version { + info!( + "Document already exists at new location: `{sanitized_relative_path}` when trying to create it in vault `{vault_id}`, merging into existing document" + ); - document_id + return merge_with_stored_version( + latest_version.clone(), + latest_version, + vault_id, + user, + device_id, + state, + &sanitized_relative_path, + request.content.contents.to_vec(), + transaction, + ) + .await; } - None => uuid::Uuid::new_v4(), - }; + } + + let document_id = uuid::Uuid::new_v4(); let last_update_id = state .database @@ -71,7 +87,6 @@ pub async fn create_document( .await .map_err(server_error)?; - let sanitized_relative_path = sanitize_path(&request.relative_path); let deduped_path = find_first_available_path( &vault_id, &sanitized_relative_path, @@ -105,5 +120,7 @@ pub async fn create_document( .await .map_err(server_error)?; - Ok(Json(new_version.into())) + Ok(Json(DocumentUpdateResponse::FastForwardUpdate( + new_version.into(), + ))) } diff --git a/sync-server/src/server/requests.rs b/sync-server/src/server/requests.rs index 119ad467..574823f5 100644 --- a/sync-server/src/server/requests.rs +++ b/sync-server/src/server/requests.rs @@ -4,18 +4,16 @@ use reconcile_text::NumberOrText; use serde::{self, Deserialize}; use ts_rs::TS; -use crate::app_state::database::models::{DocumentId, VaultUpdateId}; +use crate::app_state::database::models::VaultUpdateId; #[derive(TS, Debug, TryFromMultipart)] #[ts(export)] pub struct CreateDocumentVersion { - /// The client can decide the document id (if it wishes to) in order - /// to help with syncing. If the client does not provide a document id, - /// the server will generate one. If the client provides a document id - /// it must not already exist in the database. - pub document_id: Option, pub relative_path: String, + // whether to merge with existing document at the same path if it exists + pub force_merge: Option, + #[ts(as = "Vec")] #[form_data(limit = "unlimited")] pub content: FieldData, diff --git a/sync-server/src/server/update_document.rs b/sync-server/src/server/update_document.rs index 00fbd008..c24b62c9 100644 --- a/sync-server/src/server/update_document.rs +++ b/sync-server/src/server/update_document.rs @@ -16,7 +16,10 @@ use super::{ use crate::{ app_state::{ AppState, - database::models::{DocumentId, StoredDocumentVersion, VaultId, VaultUpdateId}, + database::{ + Transaction, + models::{DocumentId, StoredDocumentVersion, VaultId, VaultUpdateId}, + }, }, config::user_config::User, errors::{SyncServerError, client_error, not_found_error, server_error}, @@ -141,12 +144,6 @@ async fn update_document( .await .map_err(server_error)?; - let last_update_id = state - .database - .get_max_update_id_in_vault(&vault_id, Some(&mut transaction)) - .await - .map_err(server_error)?; - let latest_version = state .database .get_latest_document(&vault_id, &document_id, Some(&mut transaction)) @@ -174,12 +171,39 @@ async fn update_document( ))); } + merge_with_stored_version( + parent_document, + latest_version, + vault_id, + user, + device_id, + state, + &sanitized_relative_path, + content, + transaction, + ) + .await +} + +#[allow(clippy::too_many_arguments)] +pub async fn merge_with_stored_version( + parent_document: StoredDocumentVersion, + latest_version: StoredDocumentVersion, + vault_id: VaultId, + user: User, + device_id: DeviceIdHeader, + state: AppState, + sanitized_relative_path: &str, + content: Vec, + mut transaction: Transaction<'_>, +) -> Result, SyncServerError> { // Return the latest version if the content and path are the same as the latest // version if content == latest_version.content && sanitized_relative_path == latest_version.relative_path { info!( - "Document content is the same as the latest version for `{document_id}`, skipping update" + "Document content is the same as the latest version for `{}`, skipping update", + parent_document.document_id ); transaction .rollback() @@ -193,14 +217,17 @@ async fn update_document( } let are_all_participants_mergable = is_file_type_mergable( - &sanitized_relative_path, + sanitized_relative_path, &state.config.server.mergeable_file_extensions, ) && !is_binary(&parent_document.content) && !is_binary(&latest_version.content) && !is_binary(&content); let merged_content = if are_all_participants_mergable { - info!("Merging changes for document `{document_id}` in vault `{vault_id}`"); + info!( + "Merging changes for document `{}` in vault `{vault_id}`", + parent_document.document_id + ); reconcile( str::from_utf8(&parent_document.content) .expect("parent must be valid UTF-8 because it's not binary"), @@ -227,7 +254,7 @@ async fn update_document( { let new_path = find_first_available_path( &vault_id, - &sanitized_relative_path, + sanitized_relative_path, &state.database, &mut transaction, ) @@ -245,8 +272,14 @@ async fn update_document( latest_version.relative_path.clone() }; + let last_update_id = state + .database + .get_max_update_id_in_vault(&vault_id, Some(&mut transaction)) + .await + .map_err(server_error)?; + let new_version = StoredDocumentVersion { - document_id, + document_id: parent_document.document_id, vault_update_id: last_update_id + 1, relative_path: new_relative_path, content: merged_content,