wip

frontend/sync-client/src/consts.ts

@@ -2,5 +2,5 @@ export const TIMEOUT_FOR_MERGING_HISTORY_ENTRIES_IN_SECONDS = 60;
 export const DIFF_CACHE_SIZE_MB = 2;
 export const MAX_LOG_MESSAGE_COUNT = 100000;
 export const MAX_HISTORY_ENTRY_COUNT = 5000;
-export const SUPPORTED_API_VERSION = 2;
+export const SUPPORTED_API_VERSION = 3;
-export const WEBSOCKET_DISCONNECT_TIMEOUT_IN_S = 10;
+export const WEBSOCKET_DISCONNECT_TIMEOUT_IN_SECONDS = 10;

frontend/sync-client/src/file-operations/file-operations.ts

@@ -169,9 +169,9 @@ export class FileOperations {
         }
 
         await this.ensureClearPath(newPath);
-
         this.database.move(oldPath, newPath);
         await this.fs.rename(oldPath, newPath);
+
         await this.deletingEmptyParentDirectoriesOfDeletedFile(oldPath);
     }
 

frontend/sync-client/src/persistence/database.ts

@@ -9,6 +9,7 @@ export type DocumentId = string;
 export type RelativePath = string;
 
 export interface DocumentMetadata {
+    documentId: DocumentId;
     parentVersionId: VaultUpdateId;
     hash: string;
     remoteRelativePath?: RelativePath;
@@ -36,7 +37,6 @@ export interface StoredDatabase {
  */
 export interface DocumentRecord {
     relativePath: RelativePath;
-    documentId: DocumentId;
     metadata: DocumentMetadata | undefined;
     isDeleted: boolean;
     updates: Promise<unknown>[];
@@ -57,9 +57,8 @@ export class Database {
 
         this.documents =
             initialState.documents?.map(
-                ({ relativePath, documentId, ...metadata }) => ({
+                ({ relativePath, ...metadata }) => ({
                     relativePath,
-                    documentId,
                     metadata,
                     isDeleted: false,
                     updates: [],
@@ -114,7 +113,7 @@ export class Database {
                     i === 0
                         ? false
                         : records[i - 1].parallelVersion ===
-                          current.parallelVersion
+                        current.parallelVersion
                 )
             ) {
                 throw new Error(
@@ -127,6 +126,7 @@ export class Database {
 
     public updateDocumentMetadata(
         metadata: {
+            documentId: DocumentId;
             parentVersionId: VaultUpdateId;
             hash: string;
             remoteRelativePath: RelativePath;
@@ -196,19 +196,18 @@ export class Database {
     }
 
     public createNewPendingDocument(
-        documentId: DocumentId,
         relativePath: RelativePath,
         promise: Promise<unknown>
     ): DocumentRecord {
         this.logger.debug(
-            `Creating new pending document: ${relativePath} (${documentId})`
+            `Creating new pending document: ${relativePath}`
         );
         const previousEntry =
             this.getLatestDocumentByRelativePath(relativePath);
 
         const entry = {
             relativePath,
-            documentId,
+            documentId: undefined,
             metadata: undefined,
             isDeleted: false,
             updates: [promise],
@@ -231,8 +230,8 @@ export class Database {
     ): DocumentRecord {
         const entry = {
             relativePath,
-            documentId,
             metadata: {
+                documentId,
                 parentVersionId,
                 hash: EMPTY_HASH,
                 remoteRelativePath: relativePath
@@ -251,7 +250,7 @@ export class Database {
     public getDocumentByDocumentId(
         find: DocumentId
     ): DocumentRecord | undefined {
-        return this.documents.find(({ documentId }) => documentId === find);
+        return this.documents.find(({ metadata }) => metadata?.documentId === find);
     }
 
     public move(
@@ -331,8 +330,7 @@ export class Database {
     public async save(): Promise<void> {
         return this.saveData({
             documents: this.resolvedDocuments.map(
-                ({ relativePath, documentId, metadata }) => ({
+                ({ relativePath, metadata }) => ({
-                    documentId,
                     relativePath,
                     // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
                     ...metadata! // `resolvedDocuments` only returns docs with metadata set
@@ -346,9 +344,12 @@ export class Database {
     private ensureConsistency(): void {
         const idToPath = new Map<string, string[]>();
 
-        this.resolvedDocuments.forEach(({ relativePath, documentId }) => {
+        this.resolvedDocuments.forEach(({ relativePath, metadata }) => {
-            idToPath.set(documentId, [
+            if (metadata === undefined) {
-                ...(idToPath.get(documentId) ?? []),
+                return;
+            }
+            idToPath.set(metadata.documentId, [
+                ...(idToPath.get(metadata.documentId) ?? []),
                 relativePath
             ]);
         });
@@ -360,7 +361,7 @@ export class Database {
         if (duplicates.length > 0) {
             throw new Error(
                 "Document IDs are not unique, found duplicates: " +
-                    duplicates.join("; ")
+                duplicates.join("; ")
             );
         }
     }

frontend/sync-client/src/services/server-config.ts

@@ -14,15 +14,14 @@ export class ServerConfig {
     private response: Promise<PingResponse> | undefined;
     private config: ServerConfigData | undefined;
 
-    public constructor(private readonly syncService: SyncService) {}
+    public constructor(private readonly syncService: SyncService) { }
 
     private static validateConfig(config: ServerConfigData): void {
         if (config.supportedApiVersion !== SUPPORTED_API_VERSION) {
             const shouldUpgradeClient =
                 config.supportedApiVersion > SUPPORTED_API_VERSION;
             throw new ServerVersionMismatchError(
-                `Unsupported API version: ${config.supportedApiVersion}. Consider upgrading the ${
+                `Unsupported API version: ${config.supportedApiVersion}. Consider upgrading the ${shouldUpgradeClient ? "client" : "sync-server"
-                    shouldUpgradeClient ? "client" : "sync-server"
                 } to ensure compatibility`
             );
         }
@@ -34,11 +33,6 @@ export class ServerConfig {
         }
     }
 
-    // warm the cache
-    public async initialize(): Promise<void> {
-        await this.getConfig();
-    }
-
     public async checkConnection(forceUpdate = false): Promise<{
         isSuccessful: boolean;
         message: string;

frontend/sync-client/src/services/sync-service.ts

@@ -66,27 +66,29 @@ export class SyncService {
     }
 
     public async create({
-        documentId,
         relativePath,
-        contentBytes
+        contentBytes,
+        forceMerge
     }: {
-        documentId?: DocumentId;
         relativePath: RelativePath;
         contentBytes: Uint8Array;
+        forceMerge?: boolean;
     }): Promise<DocumentVersionWithoutContent> {
         return this.retryForever(async () => {
             const formData = new FormData();
-            if (documentId !== undefined) {
+
-                formData.append("document_id", documentId);
-            }
             formData.append("relative_path", relativePath);
+            if (forceMerge === true) {
+                formData.append("force_merge", "true");
+            }
+
             formData.append(
                 "content",
                 new Blob([new Uint8Array(contentBytes)])
             );
 
             this.logger.debug(
-                `Creating document with id ${documentId} and relative path ${relativePath}`
+                `Creating document with relative path ${relativePath} (forceMerge: ${forceMerge})`
             );
 
             const response = await this.client(this.getUrl("/documents"), {
@@ -155,8 +157,7 @@ export class SyncService {
                 (await response.json()) as DocumentUpdateResponse; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
 
             this.logger.debug(
-                `Updated document ${JSON.stringify(result)} with id ${
+                `Updated document ${JSON.stringify(result)} with id ${result.documentId
-                    result.documentId
                 }}`
             );
 
@@ -208,8 +209,7 @@ export class SyncService {
                 (await response.json()) as DocumentUpdateResponse; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
 
             this.logger.debug(
-                `Updated document ${JSON.stringify(result)} with id ${
+                `Updated document ${JSON.stringify(result)} with id ${result.documentId
-                    result.documentId
                 }}`
             );
 
@@ -336,7 +336,7 @@ export class SyncService {
         return this.retryForever(async () => {
             this.logger.debug(
                 "Getting all documents" +
-                    (since != null ? ` since ${since}` : "")
+                (since != null ? ` since ${since}` : "")
             );
 
             const url = new URL(this.getUrl("/documents"));

frontend/sync-client/src/services/websocket-manager.ts

@@ -6,7 +6,7 @@ import type { CursorPositionFromClient } from "./types/CursorPositionFromClient"
 import type { ClientCursors } from "./types/ClientCursors";
 import { createPromise } from "../utils/create-promise";
 import type { WebSocketVaultUpdate } from "./types/WebSocketVaultUpdate";
-import { WEBSOCKET_DISCONNECT_TIMEOUT_IN_S } from "../consts";
+import { WEBSOCKET_DISCONNECT_TIMEOUT_IN_SECONDS } from "../consts";
 import { removeFromArray } from "../utils/remove-from-array";
 import { EventListeners } from "../utils/data-structures/event-listeners";
 import { awaitAll } from "../utils/await-all";
@@ -36,7 +36,7 @@ export class WebSocketManager {
         private readonly logger: Logger,
         private readonly settings: Settings,
         private readonly webSocketFactoryImplementation: typeof globalThis.WebSocket = WebSocket
-    ) {}
+    ) { }
 
     public get isWebSocketConnected(): boolean {
         return (
@@ -69,10 +69,10 @@ export class WebSocketManager {
             timeoutId = setTimeout(() => {
                 reject(
                     new Error(
-                        `Timeout waiting for WebSocket to close after ${WEBSOCKET_DISCONNECT_TIMEOUT_IN_S} seconds`
+                        `Timeout waiting for WebSocket to close after ${WEBSOCKET_DISCONNECT_TIMEOUT_IN_SECONDS} seconds`
                     )
                 );
-            }, WEBSOCKET_DISCONNECT_TIMEOUT_IN_S * 1000);
+            }, WEBSOCKET_DISCONNECT_TIMEOUT_IN_SECONDS * 1000);
         });
 
         try {

frontend/sync-client/src/sync-client.ts

@@ -56,7 +56,7 @@ export class SyncClient {
                 database: Partial<StoredDatabase>;
             }>
         >
-    ) {}
+    ) { }
 
     public get documentCount(): number {
         return this.database.length;
@@ -472,7 +472,8 @@ export class SyncClient {
         this.checkIfDestroyed("startSyncing");
         this.fetchController.finishReset();
 
-        await this.serverConfig.initialize();
+        // warm the cache
+        await this.serverConfig.getConfig();
         this.webSocketManager.start();
 
         if (!this.hasStartedOfflineSync) {

frontend/sync-client/src/sync-operations/cursor-tracker.ts

@@ -113,7 +113,7 @@ export class CursorTracker {
 
             documentsWithCursors.push({
                 relative_path: relativePath,
-                document_id: record.documentId,
+                document_id: record.metadata.documentId,
                 vault_update_id: record.metadata.parentVersionId,
                 cursors: cursors.map(({ start, end }) => ({
                     start: Math.min(start, end),

frontend/sync-client/src/sync-operations/syncer.ts

@@ -8,13 +8,12 @@ import type { SyncService } from "../services/sync-service";
 import type { Logger } from "../tracing/logger";
 import PQueue from "p-queue";
 import { hash } from "../utils/hash";
-import { v4 as uuidv4 } from "uuid";
 import type { Settings } from "../persistence/settings";
 import type { FileOperations } from "../file-operations/file-operations";
 import { findMatchingFile } from "../utils/find-matching-file";
 import type { UnrestrictedSyncer } from "./unrestricted-syncer";
 import { createPromise } from "../utils/create-promise";
-import { SyncResetError } from "../services/sync-reset-error";
+import { SyncResetError } from "../errors/sync-reset-error";
 import { Locks } from "../utils/data-structures/locks";
 import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent";
 import type { WebSocketVaultUpdate } from "../services/types/WebSocketVaultUpdate";
@@ -98,9 +97,7 @@ export class Syncer {
 
         const [promise, resolve, reject] = createPromise();
 
-        const id = uuidv4();
         const document = this.database.createNewPendingDocument(
-            id,
             relativePath,
             promise
         );
@@ -171,7 +168,7 @@ export class Syncer {
             // in that case, we mustn't move it again.
             if (
                 this.database.getLatestDocumentByRelativePath(relativePath) ===
-                    undefined ||
+                undefined ||
                 this.database.getLatestDocumentByRelativePath(relativePath)
                     ?.isDeleted === true
             ) {
@@ -391,8 +388,6 @@ export class Syncer {
     }
 
     private async internalScheduleSyncForOfflineChanges(): Promise<void> {
-        await this.createFakeDocumentsFromRemoteState();
-
         const allLocalFiles = await this.operations.listFilesRecursively();
         this.logger.info(
             `Scheduling sync for ${allLocalFiles.length} local files`
@@ -426,9 +421,19 @@ export class Syncer {
 
                 // Perhaps the file has been moved; let's check by looking at the deleted files
                 const contentHash = await this.syncQueue.add(async () => {
-                    const contentBytes =
+                    try {
-                        await this.operations.read(relativePath); // this can throw FileNotFoundError
+                        const contentBytes =
-                    return hash(contentBytes);
+                            await this.operations.read(relativePath); // this can throw FileNotFoundError
+                        return hash(contentBytes);
+                    } catch (e) {
+                        if (
+                            e instanceof Error &&
+                            e.name === "FileNotFoundError"
+                        ) {
+                            return undefined;
+                        }
+                        throw e;
+                    }
                 });
 
                 if (contentHash == undefined) {
@@ -481,42 +486,9 @@ export class Syncer {
                 return this.syncLocallyDeletedFile(relativePath);
             })
         );
-    }
-
-    /**
-     * Create fake documents in the database for all files that are present locally
-     * and also exist remotely. This will stop the subequent syncs from duplicating
-     * the documents by creating the same documents from multiple clients.
-     */
-    private async createFakeDocumentsFromRemoteState(): Promise<void> {
-        if (this.database.getHasInitialSyncCompleted()) {
-            return;
-        }
-
-        const [allLocalFiles, remote] = await awaitAll([
-            this.operations.listFilesRecursively(),
-            this.syncQueue.add(async () => this.syncService.getAll())
-        ]);
-
-        if (remote !== undefined) {
-            remote.latestDocuments
-                .filter(
-                    (remoteDocument) =>
-                        allLocalFiles.includes(remoteDocument.relativePath) &&
-                        !remoteDocument.isDeleted &&
-                        this.database.getDocumentByDocumentId(
-                            remoteDocument.documentId
-                        ) === undefined
-                )
-                .forEach((remoteDocument) => {
-                    this.database.createNewEmptyDocument(
-                        remoteDocument.documentId,
-                        remoteDocument.vaultUpdateId,
-                        remoteDocument.relativePath
-                    );
-                });
-        }
 
         this.database.setHasInitialSyncCompleted(true);
     }
+
+
 }

frontend/sync-client/src/sync-operations/unrestricted-syncer.ts

@@ -82,9 +82,9 @@ export class UnrestrictedSyncer {
             const contentHash = hash(contentBytes);
 
             const response = await this.syncService.create({
-                documentId: document.documentId,
                 relativePath: originalRelativePath,
-                contentBytes
+                contentBytes,
+                forceMerge: !this.database.getHasInitialSyncCompleted() // don't duplicate files on first sync
             });
 
             // In case a document with the same name (but different ID) had existed remotely that we haven't known about
@@ -100,6 +100,7 @@ export class UnrestrictedSyncer {
 
             this.database.updateDocumentMetadata(
                 {
+                    documentId: response.documentId,
                     parentVersionId: response.vaultUpdateId,
                     hash: contentHash,
                     remoteRelativePath: response.relativePath
@@ -131,13 +132,21 @@ export class UnrestrictedSyncer {
         };
 
         await this.executeSync(updateDetails, async () => {
+            if (document.metadata === undefined) {
+                this.logger.debug(
+                    `Document ${document.relativePath} has no metadata, so it was never synced remotely`
+                );
+                return;
+            }
+
             const response = await this.syncService.delete({
-                documentId: document.documentId,
+                documentId: document.metadata.documentId,
                 relativePath: document.relativePath
             });
 
             this.database.updateDocumentMetadata(
                 {
+                    ...document.metadata,
                     parentVersionId: response.vaultUpdateId,
                     hash: EMPTY_HASH,
                     remoteRelativePath: document.relativePath
@@ -170,14 +179,14 @@ export class UnrestrictedSyncer {
         const updateDetails: SyncUpdateDetails | SyncMovedDetails =
             oldPath !== undefined
                 ? {
-                      type: SyncType.MOVE,
+                    type: SyncType.MOVE,
-                      relativePath: document.relativePath,
+                    relativePath: document.relativePath,
-                      movedFrom: oldPath
+                    movedFrom: oldPath
-                  }
+                }
                 : {
-                      type: SyncType.UPDATE,
+                    type: SyncType.UPDATE,
-                      relativePath: document.relativePath
+                    relativePath: document.relativePath
-                  };
+                };
 
         await this.executeSync(updateDetails, async () => {
             const originalRelativePath = document.relativePath;
@@ -216,22 +225,22 @@ export class UnrestrictedSyncer {
                 response =
                     isText && cachedVersion !== undefined
                         ? await this.syncService.putText({
-                              documentId: document.documentId,
+                            documentId: document.metadata.documentId,
-                              parentVersionId:
+                            parentVersionId:
-                                  document.metadata.parentVersionId,
+                                document.metadata.parentVersionId,
-                              relativePath: document.relativePath,
+                            relativePath: document.relativePath,
-                              content: diff(
+                            content: diff(
-                                  new TextDecoder().decode(cachedVersion),
+                                new TextDecoder().decode(cachedVersion),
-                                  new TextDecoder().decode(contentBytes)
+                                new TextDecoder().decode(contentBytes)
-                              )
+                            )
-                          })
+                        })
                         : await this.syncService.putBinary({
-                              documentId: document.documentId,
+                            documentId: document.metadata.documentId,
-                              parentVersionId:
+                            parentVersionId:
-                                  document.metadata.parentVersionId,
+                                document.metadata.parentVersionId,
-                              relativePath: document.relativePath,
+                            relativePath: document.relativePath,
-                              contentBytes
+                            contentBytes
-                          });
+                        });
             } else {
                 if (!force) {
                     this.logger.debug(
@@ -241,7 +250,7 @@ export class UnrestrictedSyncer {
                 }
 
                 response = await this.syncService.get({
-                    documentId: document.documentId
+                    documentId: document.metadata.documentId
                 });
             }
 
@@ -290,6 +299,7 @@ export class UnrestrictedSyncer {
 
                 this.database.updateDocumentMetadata(
                     {
+                        ...document.metadata,
                         parentVersionId: response.vaultUpdateId,
                         hash: contentHash,
                         remoteRelativePath: response.relativePath
@@ -317,6 +327,7 @@ export class UnrestrictedSyncer {
             } else {
                 this.database.updateDocumentMetadata(
                     {
+                        ...document.metadata,
                         parentVersionId: response.vaultUpdateId,
                         hash: contentHash,
                         remoteRelativePath: response.relativePath
@@ -334,16 +345,16 @@ export class UnrestrictedSyncer {
 
             const actualUpdateDetails: SyncUpdateDetails | SyncMovedDetails =
                 oldPath !== undefined ||
-                response.relativePath != originalRelativePath
+                    response.relativePath != originalRelativePath
                     ? {
-                          type: SyncType.MOVE,
+                        type: SyncType.MOVE,
-                          relativePath: response.relativePath,
+                        relativePath: response.relativePath,
-                          movedFrom: originalRelativePath
+                        movedFrom: originalRelativePath
-                      }
+                    }
                     : {
-                          type: SyncType.UPDATE,
+                        type: SyncType.UPDATE,
-                          relativePath: response.relativePath
+                        relativePath: response.relativePath
-                      };
+                    };
 
             if (areThereLocalChanges) {
                 this.history.addHistoryEntry({
@@ -437,12 +448,12 @@ export class UnrestrictedSyncer {
             const [promise, resolve] = createPromise();
             this.database.updateDocumentMetadata(
                 {
+                    documentId: remoteVersion.documentId,
                     parentVersionId: remoteVersion.vaultUpdateId,
                     hash: hash(contentBytes),
                     remoteRelativePath: remoteVersion.relativePath
                 },
                 this.database.createNewPendingDocument(
-                    remoteVersion.documentId,
                     remoteVersion.relativePath,
                     promise
                 )
@@ -541,9 +552,8 @@ export class UnrestrictedSyncer {
                     type: SyncType.SKIPPED,
                     relativePath
                 },
-                message: `File size of ${sizeInMB} MB exceeds the maximum file size limit of ${
+                message: `File size of ${sizeInMB} MB exceeds the maximum file size limit of ${maxFileSizeMB
-                    maxFileSizeMB
+                    } MB`
-                } MB`
             };
         }
     }
@@ -582,6 +592,7 @@ export class UnrestrictedSyncer {
         this.database.delete(document.relativePath);
         this.database.updateDocumentMetadata(
             {
+                documentId: response.documentId,
                 parentVersionId: response.vaultUpdateId,
                 hash: EMPTY_HASH,
                 remoteRelativePath: response.relativePath

frontend/sync-client/src/utils/await-all.ts

@@ -9,7 +9,7 @@ type ResolvedTuple<T extends readonly unknown[]> = {
 export const awaitAll = async <T extends readonly unknown[]>(
     promises: PromiseTuple<T>
 ): Promise<ResolvedTuple<T>> => {
-    // eslint-disable-next-line no-restricted-properties
+    // eslint-disable-next-line no-restricted-properties, @typescript-eslint/await-thenable
     const result = await Promise.allSettled(promises);
     for (const res of result) {
         if (res.status === "rejected") {

rustfmt.toml

@@ -0,0 +1,11 @@
+# Rustfmt configuration
+# This should match the .editorconfig settings
+
+# Use spaces for indentation (matches .editorconfig indent_style = space)
+hard_tabs = false
+
+# Use 4 spaces for indentation (matches .editorconfig indent_size = 4)
+tab_spaces = 4
+
+# Use Unix line endings (matches .editorconfig end_of_line = lf)
+newline_style = "Unix"

scripts/bump-version.sh

@@ -35,7 +35,8 @@ cd ..
 
 cp frontend/obsidian-plugin/manifest.json manifest.json  # for BRAT, otherwise it wouldn't update
 
-git ls-files | xargs npx eclint fix
+# Format all files across the project (frontend and backend)
+npx -C frontend prettier --write "**/*.{ts,js,json,md,yml,yaml}"
 
 # Commit and tag
 git add .

scripts/check.sh

@@ -45,10 +45,11 @@ cd frontend
 npm run build
 npm run test
 npm run lint
+cd ..
 
-# Use git ls-files to only check tracked files, respecting .gitignore
+# Format all files across the project (frontend and backend)
-# We always run in fix mode and then check with git status
+# Prettier respects .gitignore by default
-git ls-files | xargs npx eclint fix
+npx -C frontend prettier --write "**/*.{ts,js,json,md,yml,yaml}"
 
 if [[ "$FIX_MODE" == false ]] && [[ $(git status --porcelain) ]]; then
     git status --porcelain
@@ -56,6 +57,4 @@ if [[ "$FIX_MODE" == false ]] && [[ $(git status --porcelain) ]]; then
     exit 1
 fi
 
-cd ..
-
 echo "Success"

scripts/update-api-types.sh

@@ -12,5 +12,7 @@ cp -r sync-server/bindings/* frontend/sync-client/src/services/types/
 
 cd frontend
 npm run lint
-git ls-files | xargs npx eclint fix
+cd ..
-cd -
+
+# Format all files across the project (frontend and backend)
+npx -C frontend prettier --write "**/*.{ts,js,json,md,yml,yaml}"

sync-server/config-e2e.yml

@@ -9,24 +9,24 @@ server:
   max_clients_per_vault: 256
   response_timeout: 30m
   mergeable_file_extensions:
-    - md
+  - md
-    - txt
+  - txt
 users:
   user_configs:
-    - name: admin
+  - name: admin
-      token: test-token-change-me
+    token: test-token-change-me
-      vault_access:
+    vault_access:
-        type: allow_access_to_all
+      type: allow_access_to_all
-    - name: other-admin
+  - name: other-admin
-      token: test-token-change-me2
+    token: test-token-change-me2
-      vault_access:
+    vault_access:
-        type: allow_access_to_all
+      type: allow_access_to_all
-    - name: test
+  - name: test
-      token: other-test-token
+    token: other-test-token
-      vault_access:
+    vault_access:
-        type: allow_list
+      type: allow_list
-        allowed:
+      allowed:
-          - default
+      - default
 logging:
   log_directory: logs
   log_rotation: 7days

sync-server/src/consts.rs

@@ -20,4 +20,4 @@ pub const DEFAULT_LOG_LEVEL: LogLevel = LogLevel::Info;
 
 pub const DEFAULT_MERGEABLE_FILE_EXTENSIONS: &[&str] = &["md", "txt"];
 
-pub const SUPPORTED_API_VERSION: u32 = 2;
+pub const SUPPORTED_API_VERSION: u32 = 3;

sync-server/src/errors.rs

@@ -5,7 +5,7 @@ use axum::{
     http::StatusCode,
     response::{IntoResponse, Response},
 };
-use log::{debug, error};
+use log::debug;
 use serde::Serialize;
 use thiserror::Error;
 use ts_rs::TS;

sync-server/src/server/create_document.rs

@@ -11,10 +11,11 @@ use super::{device_id_header::DeviceIdHeader, requests::CreateDocumentVersion};
 use crate::{
     app_state::{
         AppState,
-        database::models::{DocumentVersionWithoutContent, StoredDocumentVersion, VaultId},
+        database::models::{StoredDocumentVersion, VaultId},
     },
     config::user_config::User,
-    errors::{SyncServerError, client_error, server_error},
+    errors::{SyncServerError, server_error},
+    server::{responses::DocumentUpdateResponse, update_document::merge_with_stored_version},
     utils::{
         find_first_available_path::find_first_available_path, normalize::normalize,
         sanitize_path::sanitize_path,
@@ -37,7 +38,7 @@ pub async fn create_document(
     TypedHeader(device_id): TypedHeader<DeviceIdHeader>,
     State(state): State<AppState>,
     TypedMultipart(request): TypedMultipart<CreateDocumentVersion>,
-) -> Result<Json<DocumentVersionWithoutContent>, SyncServerError> {
+) -> Result<Json<DocumentUpdateResponse>, SyncServerError> {
     debug!("Creating document in vault `{vault_id}`");
 
     let mut transaction = state
@@ -46,24 +47,39 @@ pub async fn create_document(
         .await
         .map_err(server_error)?;
 
-    let document_id = match request.document_id {
+    let sanitized_relative_path = sanitize_path(&request.relative_path);
-        Some(document_id) => {
-            let existing_version = state
-                .database
-                .get_latest_document(&vault_id, &document_id, Some(&mut transaction))
-                .await
-                .map_err(server_error)?;
 
-            if existing_version.is_some() {
+    if request.force_merge.unwrap_or_default() {
-                return Err(client_error(anyhow::anyhow!(
+        let latest_version = state
-                    "Document with the same ID `{document_id}` already exists"
+            .database
-                )));
+            .get_latest_document_by_path(
-            }
+                &vault_id,
+                &sanitized_relative_path,
+                Some(&mut transaction),
+            )
+            .await
+            .map_err(server_error)?;
+        if let Some(latest_version) = latest_version {
+            info!(
+                "Document already exists at new location: `{sanitized_relative_path}` when trying to create it in vault `{vault_id}`, merging into existing document"
+            );
 
-            document_id
+            return merge_with_stored_version(
+                latest_version.clone(),
+                latest_version,
+                vault_id,
+                user,
+                device_id,
+                state,
+                &sanitized_relative_path,
+                request.content.contents.to_vec(),
+                transaction,
+            )
+            .await;
         }
-        None => uuid::Uuid::new_v4(),
+    }
-    };
+
+    let document_id = uuid::Uuid::new_v4();
 
     let last_update_id = state
         .database
@@ -71,7 +87,6 @@ pub async fn create_document(
         .await
         .map_err(server_error)?;
 
-    let sanitized_relative_path = sanitize_path(&request.relative_path);
     let deduped_path = find_first_available_path(
         &vault_id,
         &sanitized_relative_path,
@@ -105,5 +120,7 @@ pub async fn create_document(
         .await
         .map_err(server_error)?;
 
-    Ok(Json(new_version.into()))
+    Ok(Json(DocumentUpdateResponse::FastForwardUpdate(
+        new_version.into(),
+    )))
 }

sync-server/src/server/requests.rs

@@ -4,18 +4,16 @@ use reconcile_text::NumberOrText;
 use serde::{self, Deserialize};
 use ts_rs::TS;
 
-use crate::app_state::database::models::{DocumentId, VaultUpdateId};
+use crate::app_state::database::models::VaultUpdateId;
 
 #[derive(TS, Debug, TryFromMultipart)]
 #[ts(export)]
 pub struct CreateDocumentVersion {
-    /// The client can decide the document id (if it wishes to) in order
-    /// to help with syncing. If the client does not provide a document id,
-    /// the server will generate one. If the client provides a document id
-    /// it must not already exist in the database.
-    pub document_id: Option<DocumentId>,
     pub relative_path: String,
 
+    // whether to merge with existing document at the same path if it exists
+    pub force_merge: Option<bool>,
+
     #[ts(as = "Vec<u8>")]
     #[form_data(limit = "unlimited")]
     pub content: FieldData<Bytes>,

sync-server/src/server/update_document.rs

@@ -16,7 +16,10 @@ use super::{
 use crate::{
     app_state::{
         AppState,
-        database::models::{DocumentId, StoredDocumentVersion, VaultId, VaultUpdateId},
+        database::{
+            Transaction,
+            models::{DocumentId, StoredDocumentVersion, VaultId, VaultUpdateId},
+        },
     },
     config::user_config::User,
     errors::{SyncServerError, client_error, not_found_error, server_error},
@@ -141,12 +144,6 @@ async fn update_document(
         .await
         .map_err(server_error)?;
 
-    let last_update_id = state
-        .database
-        .get_max_update_id_in_vault(&vault_id, Some(&mut transaction))
-        .await
-        .map_err(server_error)?;
-
     let latest_version = state
         .database
         .get_latest_document(&vault_id, &document_id, Some(&mut transaction))
@@ -174,12 +171,39 @@ async fn update_document(
         )));
     }
 
+    merge_with_stored_version(
+        parent_document,
+        latest_version,
+        vault_id,
+        user,
+        device_id,
+        state,
+        &sanitized_relative_path,
+        content,
+        transaction,
+    )
+    .await
+}
+
+#[allow(clippy::too_many_arguments)]
+pub async fn merge_with_stored_version(
+    parent_document: StoredDocumentVersion,
+    latest_version: StoredDocumentVersion,
+    vault_id: VaultId,
+    user: User,
+    device_id: DeviceIdHeader,
+    state: AppState,
+    sanitized_relative_path: &str,
+    content: Vec<u8>,
+    mut transaction: Transaction<'_>,
+) -> Result<Json<DocumentUpdateResponse>, SyncServerError> {
     // Return the latest version if the content and path are the same as the latest
     // version
     if content == latest_version.content && sanitized_relative_path == latest_version.relative_path
     {
         info!(
-            "Document content is the same as the latest version for `{document_id}`, skipping update"
+            "Document content is the same as the latest version for `{}`, skipping update",
+            parent_document.document_id
         );
         transaction
             .rollback()
@@ -193,14 +217,17 @@ async fn update_document(
     }
 
     let are_all_participants_mergable = is_file_type_mergable(
-        &sanitized_relative_path,
+        sanitized_relative_path,
         &state.config.server.mergeable_file_extensions,
     ) && !is_binary(&parent_document.content)
         && !is_binary(&latest_version.content)
         && !is_binary(&content);
 
     let merged_content = if are_all_participants_mergable {
-        info!("Merging changes for document `{document_id}` in vault `{vault_id}`");
+        info!(
+            "Merging changes for document `{}` in vault `{vault_id}`",
+            parent_document.document_id
+        );
         reconcile(
             str::from_utf8(&parent_document.content)
                 .expect("parent must be valid UTF-8 because it's not binary"),
@@ -227,7 +254,7 @@ async fn update_document(
     {
         let new_path = find_first_available_path(
             &vault_id,
-            &sanitized_relative_path,
+            sanitized_relative_path,
             &state.database,
             &mut transaction,
         )
@@ -245,8 +272,14 @@ async fn update_document(
         latest_version.relative_path.clone()
     };
 
+    let last_update_id = state
+        .database
+        .get_max_update_id_in_vault(&vault_id, Some(&mut transaction))
+        .await
+        .map_err(server_error)?;
+
     let new_version = StoredDocumentVersion {
-        document_id,
+        document_id: parent_document.document_id,
         vault_update_id: last_update_id + 1,
         relative_path: new_relative_path,
         content: merged_content,