diff --git a/frontend/sync-client/src/persistence/database.ts b/frontend/sync-client/src/persistence/database.ts index f3b6011..824ac6e 100644 --- a/frontend/sync-client/src/persistence/database.ts +++ b/frontend/sync-client/src/persistence/database.ts @@ -9,12 +9,14 @@ export type RelativePath = string; export interface DocumentMetadata { parentVersionId: VaultUpdateId; hash: string; + remoteRelativePath?: RelativePath; } export interface StoredDocumentMetadata { relativePath: RelativePath; documentId: DocumentId; parentVersionId: VaultUpdateId; + remoteRelativePath?: RelativePath; hash: string; } @@ -120,6 +122,7 @@ export class Database { metadata: { parentVersionId: VaultUpdateId; hash: string; + remoteRelativePath: RelativePath; }, toUpdate: DocumentRecord ): void { @@ -221,7 +224,8 @@ export class Database { documentId, metadata: { parentVersionId, - hash: EMPTY_HASH + hash: EMPTY_HASH, + remoteRelativePath: relativePath }, isDeleted: false, updates: [], diff --git a/frontend/sync-client/src/sync-operations/syncer.ts b/frontend/sync-client/src/sync-operations/syncer.ts index 7b4ce46..e141ce9 100644 --- a/frontend/sync-client/src/sync-operations/syncer.ts +++ b/frontend/sync-client/src/sync-operations/syncer.ts @@ -200,25 +200,38 @@ export class Syncer { oldPath?: RelativePath; relativePath: RelativePath; }): Promise { - if ( - oldPath !== undefined && - (this.database.getLatestDocumentByRelativePath(relativePath) === - undefined || + if (oldPath !== undefined) { + // We might have moved the document in the database before calling this method, + // in that case, we mustn't move it again. + if ( + this.database.getLatestDocumentByRelativePath(relativePath) === + undefined || this.database.getLatestDocumentByRelativePath(relativePath) - ?.isDeleted === true) - ) { - if (oldPath === relativePath) { - throw new Error( - `Old path and new path are the same: ${oldPath}` - ); - } + ?.isDeleted === true + ) { + if (oldPath === relativePath) { + throw new Error( + `Old path and new path are the same: ${oldPath}` + ); + } - this.database.move(oldPath, relativePath); + this.database.move(oldPath, relativePath); + } } let document = this.database.getLatestDocumentByRelativePath(relativePath); + if ( + oldPath !== undefined && + document?.metadata?.remoteRelativePath === relativePath + ) { + this.logger.debug( + `Document ${relativePath} has been moved as a result of a remote update, skipping sync` + ); + return; + } + if (document === undefined) { this.logger.debug( `Cannot find document ${relativePath} in the database, skipping` diff --git a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts index 58e8894..b978093 100644 --- a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts +++ b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts @@ -6,7 +6,15 @@ import type { import type { SyncService } from "../services/sync-service"; import type { Logger } from "../tracing/logger"; -import type { SyncHistory } from "../tracing/sync-history"; +import type { + CommonHistoryEntry, + SyncCreateDetails, + SyncDeleteDetails, + SyncDetails, + SyncHistory, + SyncMovedDetails, + SyncUpdateDetails +} from "../tracing/sync-history"; import { SyncStatus, SyncType } from "../tracing/sync-history"; import { EMPTY_HASH, hash } from "../utils/hash"; import type { components } from "../services/types"; @@ -16,7 +24,7 @@ import type { FileOperations } from "../file-operations/file-operations"; import { createPromise } from "../utils/create-promise"; import { FileNotFoundError } from "../file-operations/file-not-found-error"; import { SyncResetError } from "../services/sync-reset-error"; -import { makeRe } from "minimatch"; +import { globsToRegexes } from "../utils/globs-to-regexes"; export class UnrestrictedSyncer { private ignorePatterns: RegExp[]; @@ -29,90 +37,97 @@ export class UnrestrictedSyncer { private readonly operations: FileOperations, private readonly history: SyncHistory ) { - this.ignorePatterns = this.globsToRegex( - this.settings.getSettings().ignorePatterns + this.ignorePatterns = globsToRegexes( + this.settings.getSettings().ignorePatterns, + this.logger ); this.settings.addOnSettingsChangeListener((newSettings) => { - this.ignorePatterns = this.globsToRegex(newSettings.ignorePatterns); + this.ignorePatterns = globsToRegexes( + newSettings.ignorePatterns, + this.logger + ); }); } public async unrestrictedSyncLocallyCreatedFile( document: DocumentRecord ): Promise { - return this.executeSync( - document.relativePath, - SyncType.CREATE, - async () => { - if (document.isDeleted) { - this.logger.debug( - `Document ${document.relativePath} has been already deleted, no need to update it` - ); - return; - } + const updateDetails: SyncCreateDetails = { + type: SyncType.CREATE, + relativePath: document.relativePath + }; - const contentBytes = await this.operations.read( - document.relativePath - ); // this can throw FileNotFoundError - const contentHash = hash(contentBytes); - - const response = await this.syncService.create({ - documentId: document.documentId, - relativePath: document.relativePath, - contentBytes - }); - - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - relativePath: document.relativePath, - message: `Successfully uploaded locally created file`, - type: SyncType.CREATE - }); - - this.database.updateDocumentMetadata( - { - parentVersionId: response.vaultUpdateId, - hash: contentHash - }, - document + return this.executeSync(updateDetails, async () => { + if (document.isDeleted) { + this.logger.debug( + `Document ${document.relativePath} has been already deleted, no need to create it` ); - - this.database.addLastSeenUpdateId(response.vaultUpdateId); + return; } - ); + + const contentBytes = await this.operations.read( + document.relativePath + ); // this can throw FileNotFoundError + const contentHash = hash(contentBytes); + + const response = await this.syncService.create({ + documentId: document.documentId, + relativePath: document.relativePath, + contentBytes + }); + + this.database.updateDocumentMetadata( + { + parentVersionId: response.vaultUpdateId, + hash: contentHash, + remoteRelativePath: response.relativePath + }, + document + ); + + this.database.addSeenUpdateId(response.vaultUpdateId); + + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + details: updateDetails, + message: `Successfully uploaded locally created file` + }); + }); } public async unrestrictedSyncLocallyDeletedFile( document: DocumentRecord ): Promise { - await this.executeSync( - document.relativePath, - SyncType.DELETE, - async () => { - const response = await this.syncService.delete({ - documentId: document.documentId, - relativePath: document.relativePath - }); + const updateDetails: SyncDeleteDetails = { + type: SyncType.DELETE, + relativePath: document.relativePath + }; - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - relativePath: document.relativePath, - message: `Successfully deleted locally deleted file on the remote server`, - type: SyncType.DELETE - }); + await this.executeSync(updateDetails, async () => { + const response = await this.syncService.delete({ + documentId: document.documentId, + relativePath: document.relativePath + }); - this.database.updateDocumentMetadata( - { - parentVersionId: response.vaultUpdateId, - hash: EMPTY_HASH - }, - document - ); + this.database.updateDocumentMetadata( + { + parentVersionId: response.vaultUpdateId, + hash: EMPTY_HASH, + remoteRelativePath: document.relativePath + }, + document + ); - this.database.addLastSeenUpdateId(response.vaultUpdateId); - } - ); + this.database.addSeenUpdateId(response.vaultUpdateId); + + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + details: updateDetails, + message: `Successfully deleted locally deleted file on the server`, + author: response.userId + }); + }); } public async unrestrictedSyncLocallyUpdatedFile({ @@ -126,299 +141,327 @@ export class UnrestrictedSyncer { force?: boolean; document: DocumentRecord; }): Promise { - await this.executeSync( - document.relativePath, - SyncType.UPDATE, - async () => { - const originalRelativePath = document.relativePath; - - if (document.isDeleted || document.metadata === undefined) { - this.logger.debug( - `Document ${document.relativePath} has been already deleted, no need to update it` - ); - return; - } - - const contentBytes = await this.operations.read( - document.relativePath - ); // this can throw FileNotFoundError - let contentHash = hash(contentBytes); - - let response: - | components["schemas"]["DocumentVersion"] - | components["schemas"]["DocumentUpdateResponse"] - | undefined = undefined; - if ( - document.metadata.hash === contentHash && - oldPath === undefined - ) { - if (!force) { - this.logger.debug( - `File hash of ${document.relativePath} matches with last synced version and the path hasn't changed; no need to sync` - ); - return; - } - - response = await this.syncService.get({ - documentId: document.documentId - }); - } else { - response = await this.syncService.put({ - documentId: document.documentId, - parentVersionId: document.metadata.parentVersionId, + const updateDetails: SyncUpdateDetails | SyncMovedDetails = + oldPath !== undefined + ? { + type: SyncType.MOVE, relativePath: document.relativePath, - contentBytes - }); - } + movedFrom: oldPath + } + : { + type: SyncType.UPDATE, + relativePath: document.relativePath + }; - // `document` is mutable and reflects the latest state in the local database - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - if (document.isDeleted) { - this.logger.info( - `Document ${document.relativePath} has been deleted before we could finish updating it` - ); - this.database.addLastSeenUpdateId(response.vaultUpdateId); - return; - } + await this.executeSync(updateDetails, async () => { + const originalRelativePath = document.relativePath; - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - if (document.metadata === undefined) { - throw new Error( - `Document ${document.relativePath} no longer has metadata after updating it, this cannot happen` - ); - } + if (document.isDeleted || document.metadata === undefined) { + this.logger.debug( + `Document ${document.relativePath} has been already deleted, no need to update it` + ); + return; + } - if ( - // `Syncer` creates fake local document metadata for all remote docs with invalid hashes. The parent IDs will likely match - // the latest versions so we still need to update the local versions to turn the fakes into real metadata. - document.metadata.parentVersionId > response.vaultUpdateId - ) { + const contentBytes = await this.operations.read( + document.relativePath + ); // this can throw FileNotFoundError + let contentHash = hash(contentBytes); + + const areThereLocalChanges = !( + document.metadata.hash === contentHash && oldPath === undefined + ); + + let response: + | components["schemas"]["DocumentVersion"] + | components["schemas"]["DocumentUpdateResponse"] + | undefined = undefined; + + if (areThereLocalChanges) { + response = await this.syncService.put({ + documentId: document.documentId, + parentVersionId: document.metadata.parentVersionId, + relativePath: document.relativePath, + contentBytes + }); + } else { + if (!force) { this.logger.debug( - `Document ${document.relativePath} is already more up to date than the fetched version` + `File hash of ${document.relativePath} matches with last synced version and the path hasn't changed; no need to sync` ); - this.database.addLastSeenUpdateId(response.vaultUpdateId); // in case the previous `vaultUpdateId` update hasn't made it through return; } + response = await this.syncService.get({ + documentId: document.documentId + }); + } + + // `document` is mutable and reflects the latest state in the local database + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + if (document.isDeleted) { + this.logger.info( + `Document ${document.relativePath} has been deleted before we could finish updating it` + ); + this.database.addSeenUpdateId(response.vaultUpdateId); + return; + } + + if ( + // `Syncer` creates fake local document metadata for all remote docs with invalid hashes. The parent IDs will likely match + // the latest versions so we still need to update the local versions to turn the fakes into real metadata. + document.metadata.parentVersionId > response.vaultUpdateId + ) { + this.logger.debug( + `Document ${document.relativePath} is already more up to date than the fetched version` + ); + this.database.addSeenUpdateId(response.vaultUpdateId); // in case the previous `vaultUpdateId` update hasn't made it through + return; + } + + if (response.isDeleted) { + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + details: { + type: SyncType.DELETE, + relativePath: document.relativePath + }, + message: + "File has been deleted remotely, so we deleted it locally", + author: response.userId + }); + + this.database.delete(document.relativePath); + this.database.updateDocumentMetadata( + { + parentVersionId: response.vaultUpdateId, + hash: EMPTY_HASH, + remoteRelativePath: response.relativePath + }, + document + ); + + await this.operations.delete(document.relativePath); + + this.database.addSeenUpdateId(response.vaultUpdateId); + + return; + } + + let actualPath = document.relativePath; + + if (response.relativePath != originalRelativePath) { + actualPath = response.relativePath; + // Make sure to update the remote relative path to avoid uploading + // the file as a result of this filesystem event. + document.metadata.remoteRelativePath = response.relativePath; + await this.operations.move( + document.relativePath, + response.relativePath + ); // this can throw FileNotFoundError + } + + if (!("type" in response) || response.type === "MergingUpdate") { + const responseBytes = deserialize(response.contentBase64); + contentHash = hash(responseBytes); + + this.database.updateDocumentMetadata( + { + parentVersionId: response.vaultUpdateId, + hash: contentHash, + remoteRelativePath: response.relativePath + }, + document + ); + + await this.operations.write( + actualPath, + contentBytes, + responseBytes + ); + if (!force) { this.history.addHistoryEntry({ status: SyncStatus.SUCCESS, - relativePath: document.relativePath, - message: `Successfully uploaded locally updated file to the remote server`, - type: SyncType.UPDATE + details: updateDetails, + message: `The file we updated had been updated remotely, so we downloaded the merged version` }); } - - if (response.isDeleted) { - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - relativePath: document.relativePath, - message: - "The file we tried to update had been deleted remotely, therefore, we have deleted it locally", - type: SyncType.DELETE - }); - - this.database.delete(document.relativePath); - this.database.updateDocumentMetadata( - { - parentVersionId: response.vaultUpdateId, - hash: EMPTY_HASH - }, - document - ); - - await this.operations.delete(document.relativePath); - - this.database.addLastSeenUpdateId(response.vaultUpdateId); - - return; - } - - let actualPath = document.relativePath; - - if (response.relativePath != originalRelativePath) { - actualPath = response.relativePath; - await this.operations.move( - document.relativePath, - response.relativePath - ); // this can throw FileNotFoundError - } - - if ( - !("type" in response) || - response.type === "MergingUpdate" - ) { - const responseBytes = deserialize(response.contentBase64); - contentHash = hash(responseBytes); - - this.database.updateDocumentMetadata( - { - parentVersionId: response.vaultUpdateId, - hash: contentHash - }, - document - ); - - await this.operations.write( - actualPath, - contentBytes, - responseBytes - ); - - if (!force) { - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - relativePath: document.relativePath, - message: `The file we updated had been updated remotely, so we downloaded the merged version`, - type: SyncType.UPDATE - }); - } - } else { - this.database.updateDocumentMetadata( - { - parentVersionId: response.vaultUpdateId, - hash: contentHash - }, - document - ); - } - - this.database.addLastSeenUpdateId(response.vaultUpdateId); + } else { + this.database.updateDocumentMetadata( + { + parentVersionId: response.vaultUpdateId, + hash: contentHash, + remoteRelativePath: response.relativePath + }, + document + ); } - ); + + this.database.addSeenUpdateId(response.vaultUpdateId); + + const actualUpdateDetails: SyncUpdateDetails | SyncMovedDetails = + oldPath !== undefined || + response.relativePath != originalRelativePath + ? { + type: SyncType.MOVE, + relativePath: response.relativePath, + movedFrom: originalRelativePath + } + : { + type: SyncType.UPDATE, + relativePath: response.relativePath + }; + + if (areThereLocalChanges) { + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + details: actualUpdateDetails, + message: `Successfully uploaded locally updated file to the server`, + author: response.userId + }); + } else { + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + details: actualUpdateDetails, + message: `Successfully downloaded remotely updated file from the server`, + author: response.userId + }); + } + }); } public async unrestrictedSyncRemotelyUpdatedFile( remoteVersion: components["schemas"]["DocumentVersionWithoutContent"], document?: DocumentRecord ): Promise { - await this.executeSync( - remoteVersion.relativePath, - SyncType.UPDATE, - async () => { - if (document?.metadata !== undefined) { - // If the file exists locally, let's pretend the user has updated it - // and deal with remote update/deletion within `unrestrictedSyncLocallyUpdatedFile` - if ( - document.metadata.parentVersionId >= - remoteVersion.vaultUpdateId - ) { - this.logger.debug( - `Document ${remoteVersion.relativePath} is already at least as up to date as the fetched version` - ); - - return; - } - - return this.unrestrictedSyncLocallyUpdatedFile({ - document, - force: true - }); - } else if (remoteVersion.isDeleted) { - // Either the doc hasn't made it to us before and therefore we don't need to delete it, - // or we already have it, in which case the preceeding if will deal with it - this.logger.debug( - `Document ${remoteVersion.relativePath} has been deleted remotely, no need to sync` - ); - return; - } - - const content = ( - await this.syncService.get({ - documentId: remoteVersion.documentId - }) - ).contentBase64; - - document = this.database.getDocumentByDocumentId( - remoteVersion.documentId - ); - - if (document?.isDeleted === true) { - this.logger.info( - `Document ${remoteVersion.relativePath} has been deleted locally before we could finish updating it` - ); - return; - } + const updateDetails: SyncCreateDetails = { + type: SyncType.CREATE, + relativePath: remoteVersion.relativePath + }; + await this.executeSync(updateDetails, async () => { + if (document?.metadata !== undefined) { + // If the file exists locally, let's pretend the user has updated it + // and deal with remote update/deletion within `unrestrictedSyncLocallyUpdatedFile` if ( - (document?.metadata?.parentVersionId ?? -1) >= + document.metadata.parentVersionId >= remoteVersion.vaultUpdateId ) { this.logger.debug( - `Document ${remoteVersion.relativePath} is already more up to date than the fetched version` + `Document ${remoteVersion.relativePath} is already at least as up to date as the fetched version` ); + return; } - const contentBytes = deserialize(content); + return this.unrestrictedSyncLocallyUpdatedFile({ + document, + force: true + }); + } else if (remoteVersion.isDeleted) { + // Either the document hasn't made it to us before and therefore we don't need to delete it, + // or we already have it, in which case the preceeding if would've dealt with it + this.logger.debug( + `Document ${remoteVersion.relativePath} has been deleted remotely, no need to sync` + ); + return; + } - await this.operations.ensureClearPath( + // Don't download oversized files + const historyEntryForSkippedOversizedFile = + this.getHistoryEntryForSkippedOversizedFile( + remoteVersion.contentSize, remoteVersion.relativePath ); - - const [promise, resolve] = createPromise(); - this.database.updateDocumentMetadata( - { - parentVersionId: remoteVersion.vaultUpdateId, - hash: hash(contentBytes) - }, - this.database.createNewPendingDocument( - remoteVersion.documentId, - remoteVersion.relativePath, - promise - ) + if (historyEntryForSkippedOversizedFile !== undefined) { + this.history.addHistoryEntry( + historyEntryForSkippedOversizedFile ); - - await this.operations.create( - remoteVersion.relativePath, - contentBytes - ); - - resolve(); - this.database.removeDocumentPromise(promise); - - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - relativePath: remoteVersion.relativePath, - message: `Successfully downloaded remote file which hadn't existed locally`, - type: SyncType.CREATE - }); + return; } - ); + + const content = ( + await this.syncService.get({ + documentId: remoteVersion.documentId + }) + ).contentBase64; + + // We're trying to create an entirely new document that didn't exist locally + document = this.database.getDocumentByDocumentId( + remoteVersion.documentId + ); + // It can happen that a concurrent sync operation has already created the document, so we can bail here + if (document !== undefined) { + this.logger.debug( + `Document ${remoteVersion.relativePath} has already been created locally, no need to create it again` + ); + return; + } + + const contentBytes = deserialize(content); + + await this.operations.ensureClearPath(remoteVersion.relativePath); + + const [promise, resolve] = createPromise(); + this.database.updateDocumentMetadata( + { + parentVersionId: remoteVersion.vaultUpdateId, + hash: hash(contentBytes), + remoteRelativePath: remoteVersion.relativePath + }, + this.database.createNewPendingDocument( + remoteVersion.documentId, + remoteVersion.relativePath, + promise + ) + ); + + await this.operations.create( + remoteVersion.relativePath, + contentBytes + ); + + resolve(); + this.database.removeDocumentPromise(promise); + + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + details: updateDetails, + message: `Successfully downloaded remote file which hadn't existed locally`, + author: remoteVersion.userId + }); + }); } public async executeSync( - relativePath: RelativePath, - syncType: SyncType, + details: SyncDetails, fn: () => Promise ): Promise { for (const pattern of this.ignorePatterns) { - if (pattern.test(relativePath)) { + if (pattern.test(details.relativePath)) { this.logger.debug( - `File '${relativePath}' is ignored by the ignore pattern: ${pattern}` + `File '${details.relativePath}' is ignored by the ignore pattern: ${pattern}` ); return; // bail without SKIPPED status because we were told to ignore this file and we shouldn't clutter up the history } } try { - if (await this.operations.exists(relativePath)) { - const sizeInMB = Math.round( - (await this.operations.getFileSize(relativePath)) / - 1024 / - 1024 + // Only check the size of files which already exist locally. + if (await this.operations.exists(details.relativePath)) { + const sizeInBytes = await this.operations.getFileSize( + details.relativePath ); - - if (sizeInMB > this.settings.getSettings().maxFileSizeMB) { - this.history.addHistoryEntry({ - status: SyncStatus.SKIPPED, - relativePath, - message: `File size of ${sizeInMB} MB exceeds the maximum file size limit of ${ - this.settings.getSettings().maxFileSizeMB - } MB`, - type: syncType - }); - + const historyEntryForSkippedOversizedFile = + this.getHistoryEntryForSkippedOversizedFile( + sizeInBytes, + details.relativePath + ); + if (historyEntryForSkippedOversizedFile !== undefined) { + this.history.addHistoryEntry( + historyEntryForSkippedOversizedFile + ); return; } } @@ -428,7 +471,7 @@ export class UnrestrictedSyncer { if (e instanceof FileNotFoundError) { // A subsequent sync operation must have been creating to deal with this this.logger.info( - `Skiping file '${relativePath}' because it no longer exists when trying to ${syncType.toLocaleLowerCase()} it` + `Skiping file '${details.relativePath}' because it no longer exists when trying to ${details.type.toLocaleLowerCase()} it` ); return; } @@ -440,26 +483,31 @@ export class UnrestrictedSyncer { } else { this.history.addHistoryEntry({ status: SyncStatus.ERROR, - relativePath, - message: `Failed to sync file '${relativePath}' because of ${e} when trying to ${syncType.toLocaleLowerCase()} it`, - type: syncType + details, + message: `Failed to sync file '${details.relativePath}' because of ${e} when trying to ${details.type.toLocaleLowerCase()} it` }); throw e; } } } - private globsToRegex(globs: string[]): RegExp[] { - return globs - .map((pattern) => { - const result = makeRe(pattern); - if (result === false) { - this.logger.warn( - `Failed to parse ${pattern}' as a glob pattern, skipping it` - ); - } - return result; - }) - .filter((pattern) => pattern !== false); + private getHistoryEntryForSkippedOversizedFile( + sizeInBytes: number, + relativePath: RelativePath + ): CommonHistoryEntry | undefined { + const sizeInMB = Math.round(sizeInBytes / 1024 / 1024); + const { maxFileSizeMB } = this.settings.getSettings(); + if (sizeInMB > maxFileSizeMB) { + return { + status: SyncStatus.SKIPPED, + details: { + type: SyncType.SKIPPED, + relativePath + }, + message: `File size of ${sizeInMB} MB exceeds the maximum file size limit of ${ + maxFileSizeMB + } MB` + }; + } } }