diff --git a/frontend/sync-client/src/consts.ts b/frontend/sync-client/src/consts.ts index 64f581f1..dbab3de0 100644 --- a/frontend/sync-client/src/consts.ts +++ b/frontend/sync-client/src/consts.ts @@ -1,5 +1,3 @@ -export const MERGABLE_FILE_TYPES = ["md", "txt"]; - export const TIMEOUT_FOR_MERGING_HISTORY_ENTRIES_IN_SECONDS = 60; export const DIFF_CACHE_SIZE_MB = 2; export const MAX_LOG_MESSAGE_COUNT = 100000; diff --git a/frontend/sync-client/src/file-operations/file-operations.ts b/frontend/sync-client/src/file-operations/file-operations.ts index 387178f4..7c9a45cf 100644 --- a/frontend/sync-client/src/file-operations/file-operations.ts +++ b/frontend/sync-client/src/file-operations/file-operations.ts @@ -6,6 +6,7 @@ import type { TextWithCursors } from "reconcile-text"; import { reconcile } from "reconcile-text"; import { isFileTypeMergable } from "../utils/is-file-type-mergable"; import { isBinary } from "../utils/is-binary"; +import type { ServerConfig } from "../services/server-config"; export class FileOperations { private static readonly PARENTHESES_REGEX = / \((\d+)\)$/; @@ -15,6 +16,7 @@ export class FileOperations { private readonly logger: Logger, private readonly database: Database, fs: FileSystemOperations, + private readonly serverConfig: ServerConfig, private readonly nativeLineEndings = "\n" ) { this.fs = new SafeFileSystemOperations(fs, logger); @@ -89,7 +91,10 @@ export class FileOperations { } if ( - !isFileTypeMergable(path) || + !isFileTypeMergable( + path, + this.serverConfig.getConfig().mergeableFileExtensions + ) || isBinary(expectedContent) || isBinary(newContent) ) { diff --git a/frontend/sync-client/src/services/server-config.ts b/frontend/sync-client/src/services/server-config.ts new file mode 100644 index 00000000..b5ba5b15 --- /dev/null +++ b/frontend/sync-client/src/services/server-config.ts @@ -0,0 +1,67 @@ +import { createPromise } from "../utils/create-promise"; +import type { SyncService } from "./sync-service"; +import type { PingResponse } from "./types/PingResponse"; + +export interface ServerConfigData { + mergeableFileExtensions: string[]; +} + +export class ServerConfig { + private response: Promise | undefined; + private config: ServerConfigData | undefined; + + public constructor(private readonly syncService: SyncService) {} + + public async initialize(): Promise { + this.response = this.syncService.ping(); + this.config = await this.response; + } + + public async checkConnection(forceUpdate = false): Promise<{ + isSuccessful: boolean; + message: string; + }> { + try { + let { response } = this; + if (!response && !forceUpdate) { + throw new Error("ServerConfig not initialized"); + } else if (forceUpdate) { + response = this.response = this.syncService.ping(); + } + + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const result: PingResponse = (await response)!; // it must be defined, otherwise we would have thrown above + this.config = result; + + if (result.isAuthenticated) { + return { + isSuccessful: true, + message: `Successfully connected to server (version: ${result.serverVersion}) and authenticated` + }; + } + + return { + isSuccessful: false, + message: `Successfully connected to server (version: ${result.serverVersion}) but failed to authenticate` + }; + } catch (e) { + return { + isSuccessful: false, + message: `Failed to connect to server: ${e}` + }; + } + } + + public getConfig(): ServerConfigData { + if (!this.config) { + throw new Error("ServerConfig not initialized"); + } + + return this.config; + } + + public reset(): void { + this.response = undefined; + this.config = undefined; + } +} diff --git a/frontend/sync-client/src/services/sync-service.ts b/frontend/sync-client/src/services/sync-service.ts index c23fe95b..ba047b5e 100644 --- a/frontend/sync-client/src/services/sync-service.ts +++ b/frontend/sync-client/src/services/sync-service.ts @@ -302,40 +302,24 @@ export class SyncService { }); } - public async checkConnection(): Promise<{ - isSuccessful: boolean; - message: string; - }> { - try { - const response = await this.pingClient(this.getUrl("/ping"), { - headers: this.getDefaultHeaders() - }); - const result: PingResponse | SerializedError = - (await response.json()) as PingResponse | SerializedError; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion + public async ping(): Promise { + const response = await this.pingClient(this.getUrl("/ping"), { + headers: this.getDefaultHeaders() + }); + const result: PingResponse | SerializedError = + (await response.json()) as PingResponse | SerializedError; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - if ("errorType" in result) { - throw new Error( - `Failed to ping server: ${SyncService.formatError(result)}` - ); - } - - if (result.isAuthenticated) { - return { - isSuccessful: true, - message: `Successfully connected to server (version: ${result.serverVersion}) and authenticated` - }; - } - - return { - isSuccessful: false, - message: `Successfully connected to server (version: ${result.serverVersion}) but failed to authenticate` - }; - } catch (e) { - return { - isSuccessful: false, - message: `Failed to connect to server: ${e}` - }; + if ("errorType" in result) { + throw new Error( + `Failed to ping server: ${SyncService.formatError(result)}` + ); } + + this.logger.debug( + `Pinged server, got response: ${JSON.stringify(result)}` + ); + + return result; } private getUrl(path: string): string { diff --git a/frontend/sync-client/src/services/types/PingResponse.ts b/frontend/sync-client/src/services/types/PingResponse.ts index b0d993f2..ea691a93 100644 --- a/frontend/sync-client/src/services/types/PingResponse.ts +++ b/frontend/sync-client/src/services/types/PingResponse.ts @@ -13,4 +13,8 @@ export interface PingResponse { * header. */ isAuthenticated: boolean; + /** + * List of file extensions that are allowed to be merged. + */ + mergeableFileExtensions: string[]; } diff --git a/frontend/sync-client/src/sync-client.ts b/frontend/sync-client/src/sync-client.ts index 6c6bb137..d0af6bfe 100644 --- a/frontend/sync-client/src/sync-client.ts +++ b/frontend/sync-client/src/sync-client.ts @@ -25,6 +25,7 @@ import { FileChangeNotifier } from "./sync-operations/file-change-notifier"; import { FixedSizeDocumentCache } from "./utils/data-structures/fix-sized-cache"; import { setUpTelemetry } from "./utils/set-up-telemetry"; import { DIFF_CACHE_SIZE_MB } from "./consts"; +import { ServerConfig } from "./services/server-config"; export class SyncClient { private hasStartedOfflineSync = false; @@ -46,6 +47,7 @@ export class SyncClient { private readonly fileChangeNotifier: FileChangeNotifier, private readonly contentCache: FixedSizeDocumentCache, private readonly fileOperations: FileOperations, + private readonly serverConfig: ServerConfig, private readonly persistence: PersistenceProvider< Partial<{ settings: Partial; @@ -139,10 +141,13 @@ export class SyncClient { fetch ); + const serverConfig = new ServerConfig(syncService); + const fileOperations = new FileOperations( logger, database, fs, + serverConfig, nativeLineEndings ); @@ -156,7 +161,8 @@ export class SyncClient { syncService, fileOperations, history, - contentCache + contentCache, + serverConfig ); const webSocketManager = new WebSocketManager( @@ -197,6 +203,7 @@ export class SyncClient { fileChangeNotifier, contentCache, fileOperations, + serverConfig, persistence ); @@ -213,6 +220,8 @@ export class SyncClient { } this.hasStarted = true; + await this.serverConfig.initialize(); + if ( !this.unloadTelemetry && this.settings.getSettings().enableTelemetry @@ -260,7 +269,7 @@ export class SyncClient { public async checkConnection(): Promise { this.checkIfDestroyed(); - const server = await this.syncService.checkConnection(); + const server = await this.serverConfig.checkConnection(true); return { isSuccessful: server.isSuccessful, serverMessage: server.message, diff --git a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts index 4f33fe9e..4e4243cc 100644 --- a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts +++ b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts @@ -32,6 +32,7 @@ import type { DocumentVersionWithoutContent } from "../services/types/DocumentVe import type { FixedSizeDocumentCache } from "../utils/data-structures/fix-sized-cache"; import { isFileTypeMergable } from "../utils/is-file-type-mergable"; import { isBinary } from "../utils/is-binary"; +import type { ServerConfig } from "../services/server-config"; export class UnrestrictedSyncer { private ignorePatterns: RegExp[]; @@ -43,7 +44,8 @@ export class UnrestrictedSyncer { private readonly syncService: SyncService, private readonly operations: FileOperations, private readonly history: SyncHistory, - private readonly contentCache: FixedSizeDocumentCache + private readonly contentCache: FixedSizeDocumentCache, + private readonly serverConfig: ServerConfig ) { this.ignorePatterns = globsToRegexes( this.settings.getSettings().ignorePatterns, @@ -200,7 +202,10 @@ export class UnrestrictedSyncer { if (areThereLocalChanges) { const isText = !isBinary(contentBytes) && - isFileTypeMergable(document.relativePath); + isFileTypeMergable( + document.relativePath, + this.serverConfig.getConfig().mergeableFileExtensions + ); const cachedVersion = this.contentCache.get( document.metadata.parentVersionId ); @@ -547,7 +552,13 @@ export class UnrestrictedSyncer { contentBytes: Uint8Array, filePath: RelativePath ): void { - if (isFileTypeMergable(filePath) && !isBinary(contentBytes)) { + if ( + isFileTypeMergable( + filePath, + this.serverConfig.getConfig().mergeableFileExtensions + ) && + !isBinary(contentBytes) + ) { this.contentCache.put(updateId, contentBytes); } } diff --git a/frontend/sync-client/src/utils/is-file-type-mergable.test.ts b/frontend/sync-client/src/utils/is-file-type-mergable.test.ts index 3f3fffbb..a2268d19 100644 --- a/frontend/sync-client/src/utils/is-file-type-mergable.test.ts +++ b/frontend/sync-client/src/utils/is-file-type-mergable.test.ts @@ -2,41 +2,72 @@ import { describe, it } from "node:test"; import assert from "node:assert"; import { isFileTypeMergable } from "./is-file-type-mergable"; +const mergableExtensions = ["md", "txt"]; describe("isFileTypeMergable", () => { it("should return true for .md files", () => { - assert.strictEqual(isFileTypeMergable(".md"), true); - assert.strictEqual(isFileTypeMergable("hi.md"), true); + assert.strictEqual(isFileTypeMergable(".md", mergableExtensions), true); assert.strictEqual( - isFileTypeMergable("my/path/to/my/document.md"), + isFileTypeMergable("hi.md", mergableExtensions), + true + ); + assert.strictEqual( + isFileTypeMergable("my/path/to/my/document.md", mergableExtensions), true ); }); it("should return true for .txt files", () => { - assert.strictEqual(isFileTypeMergable(".txt"), true); - assert.strictEqual(isFileTypeMergable("hi.txt"), true); assert.strictEqual( - isFileTypeMergable("my/path/to/my/document.txt"), + isFileTypeMergable(".txt", mergableExtensions), + true + ); + assert.strictEqual( + isFileTypeMergable("hi.txt", mergableExtensions), + true + ); + assert.strictEqual( + isFileTypeMergable( + "my/path/to/my/document.txt", + mergableExtensions + ), true ); }); it("should be case insensitive", () => { - assert.strictEqual(isFileTypeMergable("hi.MD"), true); assert.strictEqual( - isFileTypeMergable("my/path/to/my/DOCUMENT.MD"), + isFileTypeMergable("hi.MD", mergableExtensions), true ); - assert.strictEqual(isFileTypeMergable("hi.TXT"), true); assert.strictEqual( - isFileTypeMergable("my/path/to/my/DOCUMENT.TXT"), + isFileTypeMergable("my/path/to/my/DOCUMENT.MD", mergableExtensions), + true + ); + assert.strictEqual( + isFileTypeMergable("hi.TXT", mergableExtensions), + true + ); + assert.strictEqual( + isFileTypeMergable( + "my/path/to/my/DOCUMENT.TXT", + mergableExtensions + ), true ); }); it("should return false for non-mergable file types", () => { - assert.strictEqual(isFileTypeMergable(".json"), false); - assert.strictEqual(isFileTypeMergable("HELLO.JSON"), false); - assert.strictEqual(isFileTypeMergable("my/config.yml"), false); + assert.strictEqual( + isFileTypeMergable(".json", mergableExtensions), + false + ); + assert.strictEqual( + isFileTypeMergable("HELLO.JSON", mergableExtensions), + false + ); + assert.strictEqual( + isFileTypeMergable("my/config.yml", mergableExtensions), + false + ); }); }); diff --git a/frontend/sync-client/src/utils/is-file-type-mergable.ts b/frontend/sync-client/src/utils/is-file-type-mergable.ts index 943dc1cd..4eec2733 100644 --- a/frontend/sync-client/src/utils/is-file-type-mergable.ts +++ b/frontend/sync-client/src/utils/is-file-type-mergable.ts @@ -1,8 +1,9 @@ -import { MERGABLE_FILE_TYPES } from "../consts"; - -export function isFileTypeMergable(pathOrFileName: string): boolean { +export function isFileTypeMergable( + pathOrFileName: string, + mergeableExtensions: string[] +): boolean { const parts = pathOrFileName.split("."); const fileExtension = parts.at(-1) ?? ""; - return MERGABLE_FILE_TYPES.includes(fileExtension.toLowerCase()); + return mergeableExtensions.includes(fileExtension.toLowerCase()); } diff --git a/sync-server/config-e2e.yml b/sync-server/config-e2e.yml index 0b8491ee..58410948 100644 --- a/sync-server/config-e2e.yml +++ b/sync-server/config-e2e.yml @@ -8,6 +8,9 @@ server: max_body_size_mb: 512 max_clients_per_vault: 256 response_timeout_seconds: 60 + mergeable_file_extensions: + - md + - txt users: user_configs: - name: admin diff --git a/sync-server/src/config/server_config.rs b/sync-server/src/config/server_config.rs index ce922fb9..07dc61b3 100644 --- a/sync-server/src/config/server_config.rs +++ b/sync-server/src/config/server_config.rs @@ -2,8 +2,8 @@ use log::debug; use serde::{Deserialize, Serialize}; use crate::consts::{ - DEFAULT_HOST, DEFAULT_MAX_BODY_SIZE_MB, DEFAULT_MAX_CLIENTS_PER_VAULT, DEFAULT_PORT, - DEFAULT_RESPONSE_TIMEOUT_SECONDS, + DEFAULT_HOST, DEFAULT_MAX_BODY_SIZE_MB, DEFAULT_MAX_CLIENTS_PER_VAULT, + DEFAULT_MERGEABLE_FILE_EXTENSIONS, DEFAULT_PORT, DEFAULT_RESPONSE_TIMEOUT_SECONDS, }; #[derive(Debug, Deserialize, Serialize, Clone, Default)] @@ -22,6 +22,9 @@ pub struct ServerConfig { #[serde(default = "default_response_timeout_seconds")] pub response_timeout_seconds: u64, + + #[serde(default = "default_mergeable_file_extensions")] + pub mergeable_file_extensions: Vec, } fn default_host() -> String { @@ -48,3 +51,11 @@ fn default_response_timeout_seconds() -> u64 { debug!("Using default response timeout (seconds): {DEFAULT_RESPONSE_TIMEOUT_SECONDS}"); DEFAULT_RESPONSE_TIMEOUT_SECONDS } + +fn default_mergeable_file_extensions() -> Vec { + debug!("Using default mergeable file extensions: {DEFAULT_MERGEABLE_FILE_EXTENSIONS:?}"); + DEFAULT_MERGEABLE_FILE_EXTENSIONS + .iter() + .map(|s| (*s).to_owned()) + .collect() +} diff --git a/sync-server/src/consts.rs b/sync-server/src/consts.rs index d973ca4a..881bd727 100644 --- a/sync-server/src/consts.rs +++ b/sync-server/src/consts.rs @@ -14,3 +14,5 @@ pub const DEFAULT_MAX_CLIENTS_PER_VAULT: usize = 256; pub const DEFAULT_LOG_DIRECTORY: &str = "logs"; pub const DEFAULT_LOG_ROTATION_INTERVAL: Duration = Duration::from_secs(60 * 60 * 24); // 1 day + +pub const DEFAULT_MERGEABLE_FILE_EXTENSIONS: &[&str] = &["md", "txt"]; diff --git a/sync-server/src/server/ping.rs b/sync-server/src/server/ping.rs index 620ef0d4..ec019a1d 100644 --- a/sync-server/src/server/ping.rs +++ b/sync-server/src/server/ping.rs @@ -33,5 +33,6 @@ pub async fn ping( Ok(Json(PingResponse { server_version: env!("CARGO_PKG_VERSION").to_owned(), is_authenticated, + mergeable_file_extensions: state.config.server.mergeable_file_extensions.clone(), })) } diff --git a/sync-server/src/server/responses.rs b/sync-server/src/server/responses.rs index 5cfaa5d5..22918106 100644 --- a/sync-server/src/server/responses.rs +++ b/sync-server/src/server/responses.rs @@ -16,6 +16,9 @@ pub struct PingResponse { /// Whether the client is authenticated based on the sent Authorization /// header. pub is_authenticated: bool, + + /// List of file extensions that are allowed to be merged. + pub mergeable_file_extensions: Vec, } /// Response to a fetch latest documents request. diff --git a/sync-server/src/server/update_document.rs b/sync-server/src/server/update_document.rs index a3b0f1a0..b8a17c11 100644 --- a/sync-server/src/server/update_document.rs +++ b/sync-server/src/server/update_document.rs @@ -185,8 +185,10 @@ async fn update_document( ))); } - let are_all_participants_mergable = is_file_type_mergable(&sanitized_relative_path) - && !is_binary(&parent_document.content) + let are_all_participants_mergable = is_file_type_mergable( + &sanitized_relative_path, + &state.config.server.mergeable_file_extensions, + ) && !is_binary(&parent_document.content) && !is_binary(&latest_version.content) && !is_binary(&content); diff --git a/sync-server/src/utils/is_file_type_mergable.rs b/sync-server/src/utils/is_file_type_mergable.rs index fba4b323..7aabb393 100644 --- a/sync-server/src/utils/is_file_type_mergable.rs +++ b/sync-server/src/utils/is_file_type_mergable.rs @@ -1,7 +1,10 @@ -pub fn is_file_type_mergable(path_or_file_name: &str) -> bool { +pub fn is_file_type_mergable(path_or_file_name: &str, mergeable_extensions: &[String]) -> bool { let file_extension = path_or_file_name.split('.').next_back().unwrap_or_default(); + let file_extension_lower = file_extension.to_lowercase(); - matches!(file_extension.to_lowercase().as_str(), "md" | "txt") + mergeable_extensions + .iter() + .any(|ext| ext.to_lowercase() == file_extension_lower) } #[cfg(test)] @@ -10,14 +13,22 @@ mod tests { #[test] fn test_is_file_type_mergable() { - assert!(is_file_type_mergable(".md")); - assert!(is_file_type_mergable("hi.md")); - assert!(is_file_type_mergable("my/path/to/my/document.md")); - assert!(is_file_type_mergable("hi.MD")); - assert!(is_file_type_mergable("my/path/to/my/DOCUMENT.MD")); + let mergeable = vec!["md".to_owned(), "txt".to_owned()]; - assert!(!is_file_type_mergable(".json")); - assert!(!is_file_type_mergable("HELLO.JSON")); - assert!(!is_file_type_mergable("my/config.yml")); + assert!(is_file_type_mergable(".md", &mergeable)); + assert!(is_file_type_mergable("hi.md", &mergeable)); + assert!(is_file_type_mergable( + "my/path/to/my/document.md", + &mergeable + )); + assert!(is_file_type_mergable("hi.MD", &mergeable)); + assert!(is_file_type_mergable( + "my/path/to/my/DOCUMENT.MD", + &mergeable + )); + + assert!(!is_file_type_mergable(".json", &mergeable)); + assert!(!is_file_type_mergable("HELLO.JSON", &mergeable)); + assert!(!is_file_type_mergable("my/config.yml", &mergeable)); } }