Improve vault history messages
This commit is contained in:
parent
a340039301
commit
d59203b5b9
3 changed files with 404 additions and 339 deletions
|
|
@ -9,12 +9,14 @@ export type RelativePath = string;
|
||||||
export interface DocumentMetadata {
|
export interface DocumentMetadata {
|
||||||
parentVersionId: VaultUpdateId;
|
parentVersionId: VaultUpdateId;
|
||||||
hash: string;
|
hash: string;
|
||||||
|
remoteRelativePath?: RelativePath;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface StoredDocumentMetadata {
|
export interface StoredDocumentMetadata {
|
||||||
relativePath: RelativePath;
|
relativePath: RelativePath;
|
||||||
documentId: DocumentId;
|
documentId: DocumentId;
|
||||||
parentVersionId: VaultUpdateId;
|
parentVersionId: VaultUpdateId;
|
||||||
|
remoteRelativePath?: RelativePath;
|
||||||
hash: string;
|
hash: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -120,6 +122,7 @@ export class Database {
|
||||||
metadata: {
|
metadata: {
|
||||||
parentVersionId: VaultUpdateId;
|
parentVersionId: VaultUpdateId;
|
||||||
hash: string;
|
hash: string;
|
||||||
|
remoteRelativePath: RelativePath;
|
||||||
},
|
},
|
||||||
toUpdate: DocumentRecord
|
toUpdate: DocumentRecord
|
||||||
): void {
|
): void {
|
||||||
|
|
@ -221,7 +224,8 @@ export class Database {
|
||||||
documentId,
|
documentId,
|
||||||
metadata: {
|
metadata: {
|
||||||
parentVersionId,
|
parentVersionId,
|
||||||
hash: EMPTY_HASH
|
hash: EMPTY_HASH,
|
||||||
|
remoteRelativePath: relativePath
|
||||||
},
|
},
|
||||||
isDeleted: false,
|
isDeleted: false,
|
||||||
updates: [],
|
updates: [],
|
||||||
|
|
|
||||||
|
|
@ -200,25 +200,38 @@ export class Syncer {
|
||||||
oldPath?: RelativePath;
|
oldPath?: RelativePath;
|
||||||
relativePath: RelativePath;
|
relativePath: RelativePath;
|
||||||
}): Promise<void> {
|
}): Promise<void> {
|
||||||
if (
|
if (oldPath !== undefined) {
|
||||||
oldPath !== undefined &&
|
// We might have moved the document in the database before calling this method,
|
||||||
(this.database.getLatestDocumentByRelativePath(relativePath) ===
|
// in that case, we mustn't move it again.
|
||||||
undefined ||
|
if (
|
||||||
|
this.database.getLatestDocumentByRelativePath(relativePath) ===
|
||||||
|
undefined ||
|
||||||
this.database.getLatestDocumentByRelativePath(relativePath)
|
this.database.getLatestDocumentByRelativePath(relativePath)
|
||||||
?.isDeleted === true)
|
?.isDeleted === true
|
||||||
) {
|
) {
|
||||||
if (oldPath === relativePath) {
|
if (oldPath === relativePath) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`Old path and new path are the same: ${oldPath}`
|
`Old path and new path are the same: ${oldPath}`
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
this.database.move(oldPath, relativePath);
|
this.database.move(oldPath, relativePath);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let document =
|
let document =
|
||||||
this.database.getLatestDocumentByRelativePath(relativePath);
|
this.database.getLatestDocumentByRelativePath(relativePath);
|
||||||
|
|
||||||
|
if (
|
||||||
|
oldPath !== undefined &&
|
||||||
|
document?.metadata?.remoteRelativePath === relativePath
|
||||||
|
) {
|
||||||
|
this.logger.debug(
|
||||||
|
`Document ${relativePath} has been moved as a result of a remote update, skipping sync`
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (document === undefined) {
|
if (document === undefined) {
|
||||||
this.logger.debug(
|
this.logger.debug(
|
||||||
`Cannot find document ${relativePath} in the database, skipping`
|
`Cannot find document ${relativePath} in the database, skipping`
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,15 @@ import type {
|
||||||
|
|
||||||
import type { SyncService } from "../services/sync-service";
|
import type { SyncService } from "../services/sync-service";
|
||||||
import type { Logger } from "../tracing/logger";
|
import type { Logger } from "../tracing/logger";
|
||||||
import type { SyncHistory } from "../tracing/sync-history";
|
import type {
|
||||||
|
CommonHistoryEntry,
|
||||||
|
SyncCreateDetails,
|
||||||
|
SyncDeleteDetails,
|
||||||
|
SyncDetails,
|
||||||
|
SyncHistory,
|
||||||
|
SyncMovedDetails,
|
||||||
|
SyncUpdateDetails
|
||||||
|
} from "../tracing/sync-history";
|
||||||
import { SyncStatus, SyncType } from "../tracing/sync-history";
|
import { SyncStatus, SyncType } from "../tracing/sync-history";
|
||||||
import { EMPTY_HASH, hash } from "../utils/hash";
|
import { EMPTY_HASH, hash } from "../utils/hash";
|
||||||
import type { components } from "../services/types";
|
import type { components } from "../services/types";
|
||||||
|
|
@ -16,7 +24,7 @@ import type { FileOperations } from "../file-operations/file-operations";
|
||||||
import { createPromise } from "../utils/create-promise";
|
import { createPromise } from "../utils/create-promise";
|
||||||
import { FileNotFoundError } from "../file-operations/file-not-found-error";
|
import { FileNotFoundError } from "../file-operations/file-not-found-error";
|
||||||
import { SyncResetError } from "../services/sync-reset-error";
|
import { SyncResetError } from "../services/sync-reset-error";
|
||||||
import { makeRe } from "minimatch";
|
import { globsToRegexes } from "../utils/globs-to-regexes";
|
||||||
|
|
||||||
export class UnrestrictedSyncer {
|
export class UnrestrictedSyncer {
|
||||||
private ignorePatterns: RegExp[];
|
private ignorePatterns: RegExp[];
|
||||||
|
|
@ -29,90 +37,97 @@ export class UnrestrictedSyncer {
|
||||||
private readonly operations: FileOperations,
|
private readonly operations: FileOperations,
|
||||||
private readonly history: SyncHistory
|
private readonly history: SyncHistory
|
||||||
) {
|
) {
|
||||||
this.ignorePatterns = this.globsToRegex(
|
this.ignorePatterns = globsToRegexes(
|
||||||
this.settings.getSettings().ignorePatterns
|
this.settings.getSettings().ignorePatterns,
|
||||||
|
this.logger
|
||||||
);
|
);
|
||||||
|
|
||||||
this.settings.addOnSettingsChangeListener((newSettings) => {
|
this.settings.addOnSettingsChangeListener((newSettings) => {
|
||||||
this.ignorePatterns = this.globsToRegex(newSettings.ignorePatterns);
|
this.ignorePatterns = globsToRegexes(
|
||||||
|
newSettings.ignorePatterns,
|
||||||
|
this.logger
|
||||||
|
);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
public async unrestrictedSyncLocallyCreatedFile(
|
public async unrestrictedSyncLocallyCreatedFile(
|
||||||
document: DocumentRecord
|
document: DocumentRecord
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
return this.executeSync(
|
const updateDetails: SyncCreateDetails = {
|
||||||
document.relativePath,
|
type: SyncType.CREATE,
|
||||||
SyncType.CREATE,
|
relativePath: document.relativePath
|
||||||
async () => {
|
};
|
||||||
if (document.isDeleted) {
|
|
||||||
this.logger.debug(
|
|
||||||
`Document ${document.relativePath} has been already deleted, no need to update it`
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const contentBytes = await this.operations.read(
|
return this.executeSync(updateDetails, async () => {
|
||||||
document.relativePath
|
if (document.isDeleted) {
|
||||||
); // this can throw FileNotFoundError
|
this.logger.debug(
|
||||||
const contentHash = hash(contentBytes);
|
`Document ${document.relativePath} has been already deleted, no need to create it`
|
||||||
|
|
||||||
const response = await this.syncService.create({
|
|
||||||
documentId: document.documentId,
|
|
||||||
relativePath: document.relativePath,
|
|
||||||
contentBytes
|
|
||||||
});
|
|
||||||
|
|
||||||
this.history.addHistoryEntry({
|
|
||||||
status: SyncStatus.SUCCESS,
|
|
||||||
relativePath: document.relativePath,
|
|
||||||
message: `Successfully uploaded locally created file`,
|
|
||||||
type: SyncType.CREATE
|
|
||||||
});
|
|
||||||
|
|
||||||
this.database.updateDocumentMetadata(
|
|
||||||
{
|
|
||||||
parentVersionId: response.vaultUpdateId,
|
|
||||||
hash: contentHash
|
|
||||||
},
|
|
||||||
document
|
|
||||||
);
|
);
|
||||||
|
return;
|
||||||
this.database.addLastSeenUpdateId(response.vaultUpdateId);
|
|
||||||
}
|
}
|
||||||
);
|
|
||||||
|
const contentBytes = await this.operations.read(
|
||||||
|
document.relativePath
|
||||||
|
); // this can throw FileNotFoundError
|
||||||
|
const contentHash = hash(contentBytes);
|
||||||
|
|
||||||
|
const response = await this.syncService.create({
|
||||||
|
documentId: document.documentId,
|
||||||
|
relativePath: document.relativePath,
|
||||||
|
contentBytes
|
||||||
|
});
|
||||||
|
|
||||||
|
this.database.updateDocumentMetadata(
|
||||||
|
{
|
||||||
|
parentVersionId: response.vaultUpdateId,
|
||||||
|
hash: contentHash,
|
||||||
|
remoteRelativePath: response.relativePath
|
||||||
|
},
|
||||||
|
document
|
||||||
|
);
|
||||||
|
|
||||||
|
this.database.addSeenUpdateId(response.vaultUpdateId);
|
||||||
|
|
||||||
|
this.history.addHistoryEntry({
|
||||||
|
status: SyncStatus.SUCCESS,
|
||||||
|
details: updateDetails,
|
||||||
|
message: `Successfully uploaded locally created file`
|
||||||
|
});
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
public async unrestrictedSyncLocallyDeletedFile(
|
public async unrestrictedSyncLocallyDeletedFile(
|
||||||
document: DocumentRecord
|
document: DocumentRecord
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
await this.executeSync(
|
const updateDetails: SyncDeleteDetails = {
|
||||||
document.relativePath,
|
type: SyncType.DELETE,
|
||||||
SyncType.DELETE,
|
relativePath: document.relativePath
|
||||||
async () => {
|
};
|
||||||
const response = await this.syncService.delete({
|
|
||||||
documentId: document.documentId,
|
|
||||||
relativePath: document.relativePath
|
|
||||||
});
|
|
||||||
|
|
||||||
this.history.addHistoryEntry({
|
await this.executeSync(updateDetails, async () => {
|
||||||
status: SyncStatus.SUCCESS,
|
const response = await this.syncService.delete({
|
||||||
relativePath: document.relativePath,
|
documentId: document.documentId,
|
||||||
message: `Successfully deleted locally deleted file on the remote server`,
|
relativePath: document.relativePath
|
||||||
type: SyncType.DELETE
|
});
|
||||||
});
|
|
||||||
|
|
||||||
this.database.updateDocumentMetadata(
|
this.database.updateDocumentMetadata(
|
||||||
{
|
{
|
||||||
parentVersionId: response.vaultUpdateId,
|
parentVersionId: response.vaultUpdateId,
|
||||||
hash: EMPTY_HASH
|
hash: EMPTY_HASH,
|
||||||
},
|
remoteRelativePath: document.relativePath
|
||||||
document
|
},
|
||||||
);
|
document
|
||||||
|
);
|
||||||
|
|
||||||
this.database.addLastSeenUpdateId(response.vaultUpdateId);
|
this.database.addSeenUpdateId(response.vaultUpdateId);
|
||||||
}
|
|
||||||
);
|
this.history.addHistoryEntry({
|
||||||
|
status: SyncStatus.SUCCESS,
|
||||||
|
details: updateDetails,
|
||||||
|
message: `Successfully deleted locally deleted file on the server`,
|
||||||
|
author: response.userId
|
||||||
|
});
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
public async unrestrictedSyncLocallyUpdatedFile({
|
public async unrestrictedSyncLocallyUpdatedFile({
|
||||||
|
|
@ -126,299 +141,327 @@ export class UnrestrictedSyncer {
|
||||||
force?: boolean;
|
force?: boolean;
|
||||||
document: DocumentRecord;
|
document: DocumentRecord;
|
||||||
}): Promise<void> {
|
}): Promise<void> {
|
||||||
await this.executeSync(
|
const updateDetails: SyncUpdateDetails | SyncMovedDetails =
|
||||||
document.relativePath,
|
oldPath !== undefined
|
||||||
SyncType.UPDATE,
|
? {
|
||||||
async () => {
|
type: SyncType.MOVE,
|
||||||
const originalRelativePath = document.relativePath;
|
|
||||||
|
|
||||||
if (document.isDeleted || document.metadata === undefined) {
|
|
||||||
this.logger.debug(
|
|
||||||
`Document ${document.relativePath} has been already deleted, no need to update it`
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const contentBytes = await this.operations.read(
|
|
||||||
document.relativePath
|
|
||||||
); // this can throw FileNotFoundError
|
|
||||||
let contentHash = hash(contentBytes);
|
|
||||||
|
|
||||||
let response:
|
|
||||||
| components["schemas"]["DocumentVersion"]
|
|
||||||
| components["schemas"]["DocumentUpdateResponse"]
|
|
||||||
| undefined = undefined;
|
|
||||||
if (
|
|
||||||
document.metadata.hash === contentHash &&
|
|
||||||
oldPath === undefined
|
|
||||||
) {
|
|
||||||
if (!force) {
|
|
||||||
this.logger.debug(
|
|
||||||
`File hash of ${document.relativePath} matches with last synced version and the path hasn't changed; no need to sync`
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
response = await this.syncService.get({
|
|
||||||
documentId: document.documentId
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
response = await this.syncService.put({
|
|
||||||
documentId: document.documentId,
|
|
||||||
parentVersionId: document.metadata.parentVersionId,
|
|
||||||
relativePath: document.relativePath,
|
relativePath: document.relativePath,
|
||||||
contentBytes
|
movedFrom: oldPath
|
||||||
});
|
}
|
||||||
}
|
: {
|
||||||
|
type: SyncType.UPDATE,
|
||||||
|
relativePath: document.relativePath
|
||||||
|
};
|
||||||
|
|
||||||
// `document` is mutable and reflects the latest state in the local database
|
await this.executeSync(updateDetails, async () => {
|
||||||
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
const originalRelativePath = document.relativePath;
|
||||||
if (document.isDeleted) {
|
|
||||||
this.logger.info(
|
|
||||||
`Document ${document.relativePath} has been deleted before we could finish updating it`
|
|
||||||
);
|
|
||||||
this.database.addLastSeenUpdateId(response.vaultUpdateId);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
if (document.isDeleted || document.metadata === undefined) {
|
||||||
if (document.metadata === undefined) {
|
this.logger.debug(
|
||||||
throw new Error(
|
`Document ${document.relativePath} has been already deleted, no need to update it`
|
||||||
`Document ${document.relativePath} no longer has metadata after updating it, this cannot happen`
|
);
|
||||||
);
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (
|
const contentBytes = await this.operations.read(
|
||||||
// `Syncer` creates fake local document metadata for all remote docs with invalid hashes. The parent IDs will likely match
|
document.relativePath
|
||||||
// the latest versions so we still need to update the local versions to turn the fakes into real metadata.
|
); // this can throw FileNotFoundError
|
||||||
document.metadata.parentVersionId > response.vaultUpdateId
|
let contentHash = hash(contentBytes);
|
||||||
) {
|
|
||||||
|
const areThereLocalChanges = !(
|
||||||
|
document.metadata.hash === contentHash && oldPath === undefined
|
||||||
|
);
|
||||||
|
|
||||||
|
let response:
|
||||||
|
| components["schemas"]["DocumentVersion"]
|
||||||
|
| components["schemas"]["DocumentUpdateResponse"]
|
||||||
|
| undefined = undefined;
|
||||||
|
|
||||||
|
if (areThereLocalChanges) {
|
||||||
|
response = await this.syncService.put({
|
||||||
|
documentId: document.documentId,
|
||||||
|
parentVersionId: document.metadata.parentVersionId,
|
||||||
|
relativePath: document.relativePath,
|
||||||
|
contentBytes
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
if (!force) {
|
||||||
this.logger.debug(
|
this.logger.debug(
|
||||||
`Document ${document.relativePath} is already more up to date than the fetched version`
|
`File hash of ${document.relativePath} matches with last synced version and the path hasn't changed; no need to sync`
|
||||||
);
|
);
|
||||||
this.database.addLastSeenUpdateId(response.vaultUpdateId); // in case the previous `vaultUpdateId` update hasn't made it through
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
response = await this.syncService.get({
|
||||||
|
documentId: document.documentId
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// `document` is mutable and reflects the latest state in the local database
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
||||||
|
if (document.isDeleted) {
|
||||||
|
this.logger.info(
|
||||||
|
`Document ${document.relativePath} has been deleted before we could finish updating it`
|
||||||
|
);
|
||||||
|
this.database.addSeenUpdateId(response.vaultUpdateId);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
// `Syncer` creates fake local document metadata for all remote docs with invalid hashes. The parent IDs will likely match
|
||||||
|
// the latest versions so we still need to update the local versions to turn the fakes into real metadata.
|
||||||
|
document.metadata.parentVersionId > response.vaultUpdateId
|
||||||
|
) {
|
||||||
|
this.logger.debug(
|
||||||
|
`Document ${document.relativePath} is already more up to date than the fetched version`
|
||||||
|
);
|
||||||
|
this.database.addSeenUpdateId(response.vaultUpdateId); // in case the previous `vaultUpdateId` update hasn't made it through
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (response.isDeleted) {
|
||||||
|
this.history.addHistoryEntry({
|
||||||
|
status: SyncStatus.SUCCESS,
|
||||||
|
details: {
|
||||||
|
type: SyncType.DELETE,
|
||||||
|
relativePath: document.relativePath
|
||||||
|
},
|
||||||
|
message:
|
||||||
|
"File has been deleted remotely, so we deleted it locally",
|
||||||
|
author: response.userId
|
||||||
|
});
|
||||||
|
|
||||||
|
this.database.delete(document.relativePath);
|
||||||
|
this.database.updateDocumentMetadata(
|
||||||
|
{
|
||||||
|
parentVersionId: response.vaultUpdateId,
|
||||||
|
hash: EMPTY_HASH,
|
||||||
|
remoteRelativePath: response.relativePath
|
||||||
|
},
|
||||||
|
document
|
||||||
|
);
|
||||||
|
|
||||||
|
await this.operations.delete(document.relativePath);
|
||||||
|
|
||||||
|
this.database.addSeenUpdateId(response.vaultUpdateId);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let actualPath = document.relativePath;
|
||||||
|
|
||||||
|
if (response.relativePath != originalRelativePath) {
|
||||||
|
actualPath = response.relativePath;
|
||||||
|
// Make sure to update the remote relative path to avoid uploading
|
||||||
|
// the file as a result of this filesystem event.
|
||||||
|
document.metadata.remoteRelativePath = response.relativePath;
|
||||||
|
await this.operations.move(
|
||||||
|
document.relativePath,
|
||||||
|
response.relativePath
|
||||||
|
); // this can throw FileNotFoundError
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!("type" in response) || response.type === "MergingUpdate") {
|
||||||
|
const responseBytes = deserialize(response.contentBase64);
|
||||||
|
contentHash = hash(responseBytes);
|
||||||
|
|
||||||
|
this.database.updateDocumentMetadata(
|
||||||
|
{
|
||||||
|
parentVersionId: response.vaultUpdateId,
|
||||||
|
hash: contentHash,
|
||||||
|
remoteRelativePath: response.relativePath
|
||||||
|
},
|
||||||
|
document
|
||||||
|
);
|
||||||
|
|
||||||
|
await this.operations.write(
|
||||||
|
actualPath,
|
||||||
|
contentBytes,
|
||||||
|
responseBytes
|
||||||
|
);
|
||||||
|
|
||||||
if (!force) {
|
if (!force) {
|
||||||
this.history.addHistoryEntry({
|
this.history.addHistoryEntry({
|
||||||
status: SyncStatus.SUCCESS,
|
status: SyncStatus.SUCCESS,
|
||||||
relativePath: document.relativePath,
|
details: updateDetails,
|
||||||
message: `Successfully uploaded locally updated file to the remote server`,
|
message: `The file we updated had been updated remotely, so we downloaded the merged version`
|
||||||
type: SyncType.UPDATE
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
if (response.isDeleted) {
|
this.database.updateDocumentMetadata(
|
||||||
this.history.addHistoryEntry({
|
{
|
||||||
status: SyncStatus.SUCCESS,
|
parentVersionId: response.vaultUpdateId,
|
||||||
relativePath: document.relativePath,
|
hash: contentHash,
|
||||||
message:
|
remoteRelativePath: response.relativePath
|
||||||
"The file we tried to update had been deleted remotely, therefore, we have deleted it locally",
|
},
|
||||||
type: SyncType.DELETE
|
document
|
||||||
});
|
);
|
||||||
|
|
||||||
this.database.delete(document.relativePath);
|
|
||||||
this.database.updateDocumentMetadata(
|
|
||||||
{
|
|
||||||
parentVersionId: response.vaultUpdateId,
|
|
||||||
hash: EMPTY_HASH
|
|
||||||
},
|
|
||||||
document
|
|
||||||
);
|
|
||||||
|
|
||||||
await this.operations.delete(document.relativePath);
|
|
||||||
|
|
||||||
this.database.addLastSeenUpdateId(response.vaultUpdateId);
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
let actualPath = document.relativePath;
|
|
||||||
|
|
||||||
if (response.relativePath != originalRelativePath) {
|
|
||||||
actualPath = response.relativePath;
|
|
||||||
await this.operations.move(
|
|
||||||
document.relativePath,
|
|
||||||
response.relativePath
|
|
||||||
); // this can throw FileNotFoundError
|
|
||||||
}
|
|
||||||
|
|
||||||
if (
|
|
||||||
!("type" in response) ||
|
|
||||||
response.type === "MergingUpdate"
|
|
||||||
) {
|
|
||||||
const responseBytes = deserialize(response.contentBase64);
|
|
||||||
contentHash = hash(responseBytes);
|
|
||||||
|
|
||||||
this.database.updateDocumentMetadata(
|
|
||||||
{
|
|
||||||
parentVersionId: response.vaultUpdateId,
|
|
||||||
hash: contentHash
|
|
||||||
},
|
|
||||||
document
|
|
||||||
);
|
|
||||||
|
|
||||||
await this.operations.write(
|
|
||||||
actualPath,
|
|
||||||
contentBytes,
|
|
||||||
responseBytes
|
|
||||||
);
|
|
||||||
|
|
||||||
if (!force) {
|
|
||||||
this.history.addHistoryEntry({
|
|
||||||
status: SyncStatus.SUCCESS,
|
|
||||||
relativePath: document.relativePath,
|
|
||||||
message: `The file we updated had been updated remotely, so we downloaded the merged version`,
|
|
||||||
type: SyncType.UPDATE
|
|
||||||
});
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
this.database.updateDocumentMetadata(
|
|
||||||
{
|
|
||||||
parentVersionId: response.vaultUpdateId,
|
|
||||||
hash: contentHash
|
|
||||||
},
|
|
||||||
document
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
this.database.addLastSeenUpdateId(response.vaultUpdateId);
|
|
||||||
}
|
}
|
||||||
);
|
|
||||||
|
this.database.addSeenUpdateId(response.vaultUpdateId);
|
||||||
|
|
||||||
|
const actualUpdateDetails: SyncUpdateDetails | SyncMovedDetails =
|
||||||
|
oldPath !== undefined ||
|
||||||
|
response.relativePath != originalRelativePath
|
||||||
|
? {
|
||||||
|
type: SyncType.MOVE,
|
||||||
|
relativePath: response.relativePath,
|
||||||
|
movedFrom: originalRelativePath
|
||||||
|
}
|
||||||
|
: {
|
||||||
|
type: SyncType.UPDATE,
|
||||||
|
relativePath: response.relativePath
|
||||||
|
};
|
||||||
|
|
||||||
|
if (areThereLocalChanges) {
|
||||||
|
this.history.addHistoryEntry({
|
||||||
|
status: SyncStatus.SUCCESS,
|
||||||
|
details: actualUpdateDetails,
|
||||||
|
message: `Successfully uploaded locally updated file to the server`,
|
||||||
|
author: response.userId
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
this.history.addHistoryEntry({
|
||||||
|
status: SyncStatus.SUCCESS,
|
||||||
|
details: actualUpdateDetails,
|
||||||
|
message: `Successfully downloaded remotely updated file from the server`,
|
||||||
|
author: response.userId
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
public async unrestrictedSyncRemotelyUpdatedFile(
|
public async unrestrictedSyncRemotelyUpdatedFile(
|
||||||
remoteVersion: components["schemas"]["DocumentVersionWithoutContent"],
|
remoteVersion: components["schemas"]["DocumentVersionWithoutContent"],
|
||||||
document?: DocumentRecord
|
document?: DocumentRecord
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
await this.executeSync(
|
const updateDetails: SyncCreateDetails = {
|
||||||
remoteVersion.relativePath,
|
type: SyncType.CREATE,
|
||||||
SyncType.UPDATE,
|
relativePath: remoteVersion.relativePath
|
||||||
async () => {
|
};
|
||||||
if (document?.metadata !== undefined) {
|
|
||||||
// If the file exists locally, let's pretend the user has updated it
|
|
||||||
// and deal with remote update/deletion within `unrestrictedSyncLocallyUpdatedFile`
|
|
||||||
if (
|
|
||||||
document.metadata.parentVersionId >=
|
|
||||||
remoteVersion.vaultUpdateId
|
|
||||||
) {
|
|
||||||
this.logger.debug(
|
|
||||||
`Document ${remoteVersion.relativePath} is already at least as up to date as the fetched version`
|
|
||||||
);
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
return this.unrestrictedSyncLocallyUpdatedFile({
|
|
||||||
document,
|
|
||||||
force: true
|
|
||||||
});
|
|
||||||
} else if (remoteVersion.isDeleted) {
|
|
||||||
// Either the doc hasn't made it to us before and therefore we don't need to delete it,
|
|
||||||
// or we already have it, in which case the preceeding if will deal with it
|
|
||||||
this.logger.debug(
|
|
||||||
`Document ${remoteVersion.relativePath} has been deleted remotely, no need to sync`
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const content = (
|
|
||||||
await this.syncService.get({
|
|
||||||
documentId: remoteVersion.documentId
|
|
||||||
})
|
|
||||||
).contentBase64;
|
|
||||||
|
|
||||||
document = this.database.getDocumentByDocumentId(
|
|
||||||
remoteVersion.documentId
|
|
||||||
);
|
|
||||||
|
|
||||||
if (document?.isDeleted === true) {
|
|
||||||
this.logger.info(
|
|
||||||
`Document ${remoteVersion.relativePath} has been deleted locally before we could finish updating it`
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
await this.executeSync(updateDetails, async () => {
|
||||||
|
if (document?.metadata !== undefined) {
|
||||||
|
// If the file exists locally, let's pretend the user has updated it
|
||||||
|
// and deal with remote update/deletion within `unrestrictedSyncLocallyUpdatedFile`
|
||||||
if (
|
if (
|
||||||
(document?.metadata?.parentVersionId ?? -1) >=
|
document.metadata.parentVersionId >=
|
||||||
remoteVersion.vaultUpdateId
|
remoteVersion.vaultUpdateId
|
||||||
) {
|
) {
|
||||||
this.logger.debug(
|
this.logger.debug(
|
||||||
`Document ${remoteVersion.relativePath} is already more up to date than the fetched version`
|
`Document ${remoteVersion.relativePath} is already at least as up to date as the fetched version`
|
||||||
);
|
);
|
||||||
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const contentBytes = deserialize(content);
|
return this.unrestrictedSyncLocallyUpdatedFile({
|
||||||
|
document,
|
||||||
|
force: true
|
||||||
|
});
|
||||||
|
} else if (remoteVersion.isDeleted) {
|
||||||
|
// Either the document hasn't made it to us before and therefore we don't need to delete it,
|
||||||
|
// or we already have it, in which case the preceeding if would've dealt with it
|
||||||
|
this.logger.debug(
|
||||||
|
`Document ${remoteVersion.relativePath} has been deleted remotely, no need to sync`
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
await this.operations.ensureClearPath(
|
// Don't download oversized files
|
||||||
|
const historyEntryForSkippedOversizedFile =
|
||||||
|
this.getHistoryEntryForSkippedOversizedFile(
|
||||||
|
remoteVersion.contentSize,
|
||||||
remoteVersion.relativePath
|
remoteVersion.relativePath
|
||||||
);
|
);
|
||||||
|
if (historyEntryForSkippedOversizedFile !== undefined) {
|
||||||
const [promise, resolve] = createPromise();
|
this.history.addHistoryEntry(
|
||||||
this.database.updateDocumentMetadata(
|
historyEntryForSkippedOversizedFile
|
||||||
{
|
|
||||||
parentVersionId: remoteVersion.vaultUpdateId,
|
|
||||||
hash: hash(contentBytes)
|
|
||||||
},
|
|
||||||
this.database.createNewPendingDocument(
|
|
||||||
remoteVersion.documentId,
|
|
||||||
remoteVersion.relativePath,
|
|
||||||
promise
|
|
||||||
)
|
|
||||||
);
|
);
|
||||||
|
return;
|
||||||
await this.operations.create(
|
|
||||||
remoteVersion.relativePath,
|
|
||||||
contentBytes
|
|
||||||
);
|
|
||||||
|
|
||||||
resolve();
|
|
||||||
this.database.removeDocumentPromise(promise);
|
|
||||||
|
|
||||||
this.history.addHistoryEntry({
|
|
||||||
status: SyncStatus.SUCCESS,
|
|
||||||
relativePath: remoteVersion.relativePath,
|
|
||||||
message: `Successfully downloaded remote file which hadn't existed locally`,
|
|
||||||
type: SyncType.CREATE
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
);
|
|
||||||
|
const content = (
|
||||||
|
await this.syncService.get({
|
||||||
|
documentId: remoteVersion.documentId
|
||||||
|
})
|
||||||
|
).contentBase64;
|
||||||
|
|
||||||
|
// We're trying to create an entirely new document that didn't exist locally
|
||||||
|
document = this.database.getDocumentByDocumentId(
|
||||||
|
remoteVersion.documentId
|
||||||
|
);
|
||||||
|
// It can happen that a concurrent sync operation has already created the document, so we can bail here
|
||||||
|
if (document !== undefined) {
|
||||||
|
this.logger.debug(
|
||||||
|
`Document ${remoteVersion.relativePath} has already been created locally, no need to create it again`
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const contentBytes = deserialize(content);
|
||||||
|
|
||||||
|
await this.operations.ensureClearPath(remoteVersion.relativePath);
|
||||||
|
|
||||||
|
const [promise, resolve] = createPromise();
|
||||||
|
this.database.updateDocumentMetadata(
|
||||||
|
{
|
||||||
|
parentVersionId: remoteVersion.vaultUpdateId,
|
||||||
|
hash: hash(contentBytes),
|
||||||
|
remoteRelativePath: remoteVersion.relativePath
|
||||||
|
},
|
||||||
|
this.database.createNewPendingDocument(
|
||||||
|
remoteVersion.documentId,
|
||||||
|
remoteVersion.relativePath,
|
||||||
|
promise
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
await this.operations.create(
|
||||||
|
remoteVersion.relativePath,
|
||||||
|
contentBytes
|
||||||
|
);
|
||||||
|
|
||||||
|
resolve();
|
||||||
|
this.database.removeDocumentPromise(promise);
|
||||||
|
|
||||||
|
this.history.addHistoryEntry({
|
||||||
|
status: SyncStatus.SUCCESS,
|
||||||
|
details: updateDetails,
|
||||||
|
message: `Successfully downloaded remote file which hadn't existed locally`,
|
||||||
|
author: remoteVersion.userId
|
||||||
|
});
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
public async executeSync<T>(
|
public async executeSync<T>(
|
||||||
relativePath: RelativePath,
|
details: SyncDetails,
|
||||||
syncType: SyncType,
|
|
||||||
fn: () => Promise<T>
|
fn: () => Promise<T>
|
||||||
): Promise<T | undefined> {
|
): Promise<T | undefined> {
|
||||||
for (const pattern of this.ignorePatterns) {
|
for (const pattern of this.ignorePatterns) {
|
||||||
if (pattern.test(relativePath)) {
|
if (pattern.test(details.relativePath)) {
|
||||||
this.logger.debug(
|
this.logger.debug(
|
||||||
`File '${relativePath}' is ignored by the ignore pattern: ${pattern}`
|
`File '${details.relativePath}' is ignored by the ignore pattern: ${pattern}`
|
||||||
);
|
);
|
||||||
return; // bail without SKIPPED status because we were told to ignore this file and we shouldn't clutter up the history
|
return; // bail without SKIPPED status because we were told to ignore this file and we shouldn't clutter up the history
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
if (await this.operations.exists(relativePath)) {
|
// Only check the size of files which already exist locally.
|
||||||
const sizeInMB = Math.round(
|
if (await this.operations.exists(details.relativePath)) {
|
||||||
(await this.operations.getFileSize(relativePath)) /
|
const sizeInBytes = await this.operations.getFileSize(
|
||||||
1024 /
|
details.relativePath
|
||||||
1024
|
|
||||||
);
|
);
|
||||||
|
const historyEntryForSkippedOversizedFile =
|
||||||
if (sizeInMB > this.settings.getSettings().maxFileSizeMB) {
|
this.getHistoryEntryForSkippedOversizedFile(
|
||||||
this.history.addHistoryEntry({
|
sizeInBytes,
|
||||||
status: SyncStatus.SKIPPED,
|
details.relativePath
|
||||||
relativePath,
|
);
|
||||||
message: `File size of ${sizeInMB} MB exceeds the maximum file size limit of ${
|
if (historyEntryForSkippedOversizedFile !== undefined) {
|
||||||
this.settings.getSettings().maxFileSizeMB
|
this.history.addHistoryEntry(
|
||||||
} MB`,
|
historyEntryForSkippedOversizedFile
|
||||||
type: syncType
|
);
|
||||||
});
|
|
||||||
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -428,7 +471,7 @@ export class UnrestrictedSyncer {
|
||||||
if (e instanceof FileNotFoundError) {
|
if (e instanceof FileNotFoundError) {
|
||||||
// A subsequent sync operation must have been creating to deal with this
|
// A subsequent sync operation must have been creating to deal with this
|
||||||
this.logger.info(
|
this.logger.info(
|
||||||
`Skiping file '${relativePath}' because it no longer exists when trying to ${syncType.toLocaleLowerCase()} it`
|
`Skiping file '${details.relativePath}' because it no longer exists when trying to ${details.type.toLocaleLowerCase()} it`
|
||||||
);
|
);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
@ -440,26 +483,31 @@ export class UnrestrictedSyncer {
|
||||||
} else {
|
} else {
|
||||||
this.history.addHistoryEntry({
|
this.history.addHistoryEntry({
|
||||||
status: SyncStatus.ERROR,
|
status: SyncStatus.ERROR,
|
||||||
relativePath,
|
details,
|
||||||
message: `Failed to sync file '${relativePath}' because of ${e} when trying to ${syncType.toLocaleLowerCase()} it`,
|
message: `Failed to sync file '${details.relativePath}' because of ${e} when trying to ${details.type.toLocaleLowerCase()} it`
|
||||||
type: syncType
|
|
||||||
});
|
});
|
||||||
throw e;
|
throw e;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private globsToRegex(globs: string[]): RegExp[] {
|
private getHistoryEntryForSkippedOversizedFile(
|
||||||
return globs
|
sizeInBytes: number,
|
||||||
.map((pattern) => {
|
relativePath: RelativePath
|
||||||
const result = makeRe(pattern);
|
): CommonHistoryEntry | undefined {
|
||||||
if (result === false) {
|
const sizeInMB = Math.round(sizeInBytes / 1024 / 1024);
|
||||||
this.logger.warn(
|
const { maxFileSizeMB } = this.settings.getSettings();
|
||||||
`Failed to parse ${pattern}' as a glob pattern, skipping it`
|
if (sizeInMB > maxFileSizeMB) {
|
||||||
);
|
return {
|
||||||
}
|
status: SyncStatus.SKIPPED,
|
||||||
return result;
|
details: {
|
||||||
})
|
type: SyncType.SKIPPED,
|
||||||
.filter((pattern) => pattern !== false);
|
relativePath
|
||||||
|
},
|
||||||
|
message: `File size of ${sizeInMB} MB exceeds the maximum file size limit of ${
|
||||||
|
maxFileSizeMB
|
||||||
|
} MB`
|
||||||
|
};
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue