andras/vault-link
1
0
Fork 0
Code Issues Pull requests 18 Projects Releases 55 Packages Wiki Activity Actions 1

Fix syncing logic

Browse source
This commit is contained in:
Andras Schmelczer 2026-01-04 14:08:33 +00:00
parent 0d7d36e971
commit 7c991c3b4d
10 changed files with 223 additions and 184 deletions
Download patch file Download diff file Expand all files Collapse all files

1
frontend/sync-client/src/consts.ts
View file

@ -4,3 +4,4 @@ export const MAX_LOG_MESSAGE_COUNT = 100000;
export const MAX_HISTORY_ENTRY_COUNT = 5000;
export const SUPPORTED_API_VERSION = 3;
export const WEBSOCKET_DISCONNECT_TIMEOUT_IN_SECONDS = 10;
export const WEBSOCKET_CONNECTION_TIMEOUT_IN_SECONDS = 10;

42
frontend/sync-client/src/persistence/database.ts
View file

@ -26,7 +26,6 @@ export interface StoredDocumentMetadata {
export interface StoredDatabase {
documents: StoredDocumentMetadata[];
lastSeenUpdateId: VaultUpdateId | undefined;
hasInitialSyncCompleted: boolean;
}
/**
@ -46,7 +45,6 @@ export interface DocumentRecord {
export class Database {
private documents: DocumentRecord[];
private lastSeenUpdateIds: CoveredValues;
private hasInitialSyncCompleted: boolean;
public constructor(
private readonly logger: Logger,
@ -56,15 +54,13 @@ export class Database {
initialState ??= {};
this.documents =
initialState.documents?.map(
({ relativePath, ...metadata }) => ({
relativePath,
metadata,
isDeleted: false,
updates: [],
parallelVersion: 0
})
) ?? [];
initialState.documents?.map(({ relativePath, ...metadata }) => ({
relativePath,
metadata,
isDeleted: false,
updates: [],
parallelVersion: 0
})) ?? [];
this.ensureConsistency();
this.logger.debug(`Loaded ${this.documents.length} documents`);
@ -79,11 +75,6 @@ export class Database {
this.lastSeenUpdateIds.add(doc.metadata?.parentVersionId);
});
this.hasInitialSyncCompleted =
initialState.hasInitialSyncCompleted ?? false;
this.logger.debug(
`Loaded hasInitialSyncCompleted: ${this.hasInitialSyncCompleted}`
);
}
public get length(): number {
@ -199,15 +190,12 @@ export class Database {
relativePath: RelativePath,
promise: Promise<unknown>
): DocumentRecord {
this.logger.debug(
`Creating new pending document: ${relativePath}`
);
this.logger.debug(`Creating new pending document: ${relativePath}`);
const previousEntry =
this.getLatestDocumentByRelativePath(relativePath);
const entry = {
relativePath,
documentId: undefined,
metadata: undefined,
isDeleted: false,
updates: [promise],
@ -250,7 +238,9 @@ export class Database {
public getDocumentByDocumentId(
find: DocumentId
): DocumentRecord | undefined {
return this.documents.find(({ metadata }) => metadata?.documentId === find);
return this.documents.find(
({ metadata }) => metadata?.documentId === find
);
}
public move(
@ -292,14 +282,6 @@ export class Database {
candidate.isDeleted = true;
}
public getHasInitialSyncCompleted(): boolean {
return this.hasInitialSyncCompleted;
}
public setHasInitialSyncCompleted(value: boolean): void {
this.hasInitialSyncCompleted = value;
this.saveInTheBackground();
}
public getLastSeenUpdateId(): VaultUpdateId {
return this.lastSeenUpdateIds.min;
@ -323,7 +305,6 @@ export class Database {
this.lastSeenUpdateIds = new CoveredValues(
0 // the first updateId will be 1 which is the first integer after -1
);
this.hasInitialSyncCompleted = false;
this.saveInTheBackground();
}
@ -337,7 +318,6 @@ export class Database {
})
),
lastSeenUpdateId: this.lastSeenUpdateIds.min,
hasInitialSyncCompleted: this.hasInitialSyncCompleted
});
}

45
frontend/sync-client/src/services/websocket-manager.ts
View file

@ -6,7 +6,10 @@ import type { CursorPositionFromClient } from "./types/CursorPositionFromClient"
import type { ClientCursors } from "./types/ClientCursors";
import { createPromise } from "../utils/create-promise";
import type { WebSocketVaultUpdate } from "./types/WebSocketVaultUpdate";
import { WEBSOCKET_DISCONNECT_TIMEOUT_IN_SECONDS } from "../consts";
import {
WEBSOCKET_DISCONNECT_TIMEOUT_IN_SECONDS,
WEBSOCKET_CONNECTION_TIMEOUT_IN_SECONDS
} from "../consts";
import { removeFromArray } from "../utils/remove-from-array";
import { EventListeners } from "../utils/data-structures/event-listeners";
import { awaitAll } from "../utils/await-all";
@ -27,6 +30,7 @@ export class WebSocketManager {
private isStopped = true;
private resolveDisconnectingPromise: null | (() => unknown) = null;
private reconnectTimeoutId: ReturnType<typeof setTimeout> | undefined;
private connectionTimeoutId: ReturnType<typeof setTimeout> | undefined;
private readonly outstandingPromises: Promise<unknown>[] = [];
@ -36,7 +40,7 @@ export class WebSocketManager {
private readonly logger: Logger,
private readonly settings: Settings,
private readonly webSocketFactoryImplementation: typeof globalThis.WebSocket = WebSocket
) {}
) { }
public get isWebSocketConnected(): boolean {
return (
@ -61,6 +65,11 @@ export class WebSocketManager {
this.reconnectTimeoutId = undefined;
}
if (this.connectionTimeoutId !== undefined) {
clearTimeout(this.connectionTimeoutId);
this.connectionTimeoutId = undefined;
}
this.webSocket?.close(1000, "WebSocketManager has been stopped");
// eslint-disable-next-line @typescript-eslint/init-declarations
@ -171,7 +180,22 @@ export class WebSocketManager {
this.webSocket = new this.webSocketFactoryImplementation(wsUri);
// Set connection timeout to handle cases where server is down and the WebSocket connection won't open
this.connectionTimeoutId = setTimeout(() => {
this.connectionTimeoutId = undefined;
this.logger.warn(
`WebSocket connection timeout after ${WEBSOCKET_CONNECTION_TIMEOUT_IN_SECONDS} seconds`
);
// Force close to trigger onclose handler which will schedule reconnection
this.webSocket?.close();
}, WEBSOCKET_CONNECTION_TIMEOUT_IN_SECONDS * 1000);
this.webSocket.onopen = (): void => {
if (this.connectionTimeoutId !== undefined) {
clearTimeout(this.connectionTimeoutId);
this.connectionTimeoutId = undefined;
}
// Check if we've been stopped while connecting
if (this.isStopped) {
this.webSocket?.close(
@ -215,7 +239,18 @@ export class WebSocketManager {
}
};
this.webSocket.onerror = (error): void => {
this.logger.error(
`WebSocket error occurred: ${error instanceof ErrorEvent ? error.message : "Unknown error"}`
);
};
this.webSocket.onclose = (event): void => {
if (this.connectionTimeoutId !== undefined) {
clearTimeout(this.connectionTimeoutId);
this.connectionTimeoutId = undefined;
}
this.logger.warn(
`WebSocket closed with code ${event.code} (${event.reason == "" ? "unknown reason" : event.reason})`
);
@ -225,10 +260,14 @@ export class WebSocketManager {
this.resolveDisconnectingPromise?.();
this.resolveDisconnectingPromise = null;
} else {
const delay = this.settings.getSettings().webSocketRetryIntervalMs;
this.logger.info(
`Reconnecting to WebSocket in ${delay}ms...`
);
this.reconnectTimeoutId = setTimeout(() => {
this.reconnectTimeoutId = undefined;
this.initializeWebSocket();
}, this.settings.getSettings().webSocketRetryIntervalMs);
}, delay);
}
};
}

6
frontend/sync-client/src/sync-client.ts
View file

@ -56,7 +56,7 @@ export class SyncClient {
database: Partial<StoredDatabase>;
}>
>
) {}
) { }
public get documentCount(): number {
return this.database.length;
@ -339,7 +339,9 @@ export class SyncClient {
this.hasFinishedOfflineSync = false;
this.serverConfig.reset();
await this.startSyncing();
if (this.settings.getSettings().isSyncEnabled) {
await this.startSyncing();
}
}
public getSettings(): SyncSettings {

247
frontend/sync-client/src/sync-operations/unrestricted-syncer.ts
View file

@ -84,36 +84,16 @@ export class UnrestrictedSyncer {
const response = await this.syncService.create({
relativePath: originalRelativePath,
contentBytes,
forceMerge: !this.database.getHasInitialSyncCompleted() // don't duplicate files on first sync
forceMerge: true
});
// In case a document with the same name (but different ID) had existed remotely that we haven't known about
if (response.relativePath != originalRelativePath) {
this.logger.debug(
`Document ${originalRelativePath} has been created remotely at a different path: ${response.relativePath}, moving it locally`
);
await this.operations.move(
document.relativePath,
response.relativePath
); // this can throw FileNotFoundError
}
this.database.updateDocumentMetadata(
{
documentId: response.documentId,
parentVersionId: response.vaultUpdateId,
hash: contentHash,
remoteRelativePath: response.relativePath
},
document
);
this.database.addSeenUpdateId(response.vaultUpdateId);
await this.updateCache(
response.vaultUpdateId,
contentBytes,
response.relativePath
);
this.handleMaybeMergingResponse({
document,
response,
contentHash,
originalRelativePath,
originalContentBytes: contentBytes
});
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
@ -134,7 +114,7 @@ export class UnrestrictedSyncer {
await this.executeSync(updateDetails, async () => {
if (document.metadata === undefined) {
this.logger.debug(
`Document ${document.relativePath} has no metadata, so it was never synced remotely`
`Document ${document.relativePath} has no metadata, so it has never got synced remotely; no need to delete it remotely`
);
return;
}
@ -254,69 +234,16 @@ export class UnrestrictedSyncer {
});
}
// `document` is mutable and reflects the latest state in the local database
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
if (document.isDeleted) {
this.logger.info(
`Document ${document.relativePath} has been deleted before we could finish updating it`
);
this.database.addSeenUpdateId(response.vaultUpdateId);
return;
}
if (
// `Syncer` creates fake local document metadata for all remote docs with invalid hashes. The parent IDs will likely match
// the latest versions so we still need to update the local versions to turn the fakes into real metadata.
document.metadata.parentVersionId > response.vaultUpdateId
) {
this.logger.debug(
`Document ${document.relativePath} is already more up to date than the fetched version`
);
this.database.addSeenUpdateId(response.vaultUpdateId); // in case the previous `vaultUpdateId` update hasn't made it through
return;
}
if (response.isDeleted) {
return this.applyRemoteDeleteLocally(document, response);
}
let actualPath = document.relativePath;
if (response.relativePath != originalRelativePath) {
actualPath = response.relativePath;
// Make sure to update the remote relative path to avoid uploading
// the file as a result of this filesystem event.
document.metadata.remoteRelativePath = response.relativePath;
await this.operations.move(
document.relativePath,
response.relativePath
); // this can throw FileNotFoundError
}
this.handleMaybeMergingResponse({
document,
response: response!,
contentHash,
originalRelativePath,
originalContentBytes: contentBytes
});
if (!("type" in response) || response.type === "MergingUpdate") {
const responseBytes = base64ToBytes(response.contentBase64);
contentHash = hash(responseBytes);
this.database.updateDocumentMetadata(
{
...document.metadata,
parentVersionId: response.vaultUpdateId,
hash: contentHash,
remoteRelativePath: response.relativePath
},
document
);
await this.operations.write(
actualPath,
contentBytes,
responseBytes
);
await this.updateCache(
response.vaultUpdateId,
responseBytes,
actualPath
);
if (!force) {
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
@ -324,32 +251,15 @@ export class UnrestrictedSyncer {
message: `The file we updated had been updated remotely, so we downloaded the merged version`
});
}
} else {
this.database.updateDocumentMetadata(
{
...document.metadata,
parentVersionId: response.vaultUpdateId,
hash: contentHash,
remoteRelativePath: response.relativePath
},
document
);
await this.updateCache(
response.vaultUpdateId,
contentBytes,
actualPath
);
}
this.database.addSeenUpdateId(response.vaultUpdateId);
const actualUpdateDetails: SyncUpdateDetails | SyncMovedDetails =
oldPath !== undefined ||
response.relativePath != originalRelativePath
? {
type: SyncType.MOVE,
relativePath: response.relativePath,
movedFrom: originalRelativePath
movedFrom: oldPath ?? originalRelativePath
}
: {
type: SyncType.UPDATE,
@ -363,7 +273,7 @@ export class UnrestrictedSyncer {
message: `Successfully uploaded locally updated file to the server`,
author: response.userId
});
} else {
} else if (!response.isDeleted) {
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
details: actualUpdateDetails,
@ -371,6 +281,17 @@ export class UnrestrictedSyncer {
author: response.userId,
timestamp: new Date(response.updatedDate)
});
} else {
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
details: {
type: SyncType.DELETE,
relativePath: document.relativePath
},
message: "File has been deleted remotely, so we deleted it locally",
author: response.userId,
timestamp: new Date(response.updatedDate)
});
}
});
}
@ -539,6 +460,105 @@ export class UnrestrictedSyncer {
}
}
private async handleMaybeMergingResponse(
{
document,
response,
contentHash,
originalRelativePath,
originalContentBytes
}: {
document: DocumentRecord;
response: DocumentVersion | DocumentUpdateResponse,
contentHash: string,
originalRelativePath: string,
originalContentBytes: Uint8Array
}
): Promise<void> {
// `document` is mutable and reflects the latest state in the local database
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
if (document.isDeleted) {
this.logger.info(
`Document ${document.relativePath} has been deleted before we could finish updating it`
);
this.database.addSeenUpdateId(response.vaultUpdateId);
return;
}
if (
(document.metadata?.parentVersionId ?? 0) > response.vaultUpdateId
) {
this.logger.debug(
`Document ${document.relativePath} is already more up to date than the fetched version`
);
this.database.addSeenUpdateId(response.vaultUpdateId); // in case the previous `vaultUpdateId` update hasn't made it through
return;
}
if (response.isDeleted) {
return this.applyRemoteDeleteLocally(document, response);
}
let actualPath = document.relativePath;
// this can't happen on the creation path as we can only get a merging response if a document already exists remotely on the same path
if (response.relativePath != originalRelativePath) {
actualPath = response.relativePath;
// Make sure to update the remote relative path to avoid uploading
// the file as a result of this filesystem event.
if (document.metadata !== undefined) {
document.metadata.remoteRelativePath = response.relativePath;
}
await this.operations.move(
document.relativePath,
response.relativePath
); // this can throw FileNotFoundError
}
if (!("type" in response) || response.type === "MergingUpdate") {
const responseBytes = base64ToBytes(response.contentBase64);
contentHash = hash(responseBytes);
this.database.updateDocumentMetadata(
{
documentId: response.documentId,
parentVersionId: response.vaultUpdateId,
hash: contentHash,
remoteRelativePath: response.relativePath
},
document
);
await this.operations.write(
actualPath,
originalContentBytes,
responseBytes
);
await this.updateCache(
response.vaultUpdateId,
responseBytes,
actualPath
);
} else {
this.database.updateDocumentMetadata(
{
documentId: response.documentId,
parentVersionId: response.vaultUpdateId,
hash: contentHash,
remoteRelativePath: response.relativePath
},
document
);
await this.updateCache(
response.vaultUpdateId,
originalContentBytes,
actualPath
);
}
this.database.addSeenUpdateId(response.vaultUpdateId);
}
private getHistoryEntryForSkippedOversizedFile(
sizeInBytes: number,
relativePath: RelativePath
@ -578,16 +598,7 @@ export class UnrestrictedSyncer {
document: DocumentRecord,
response: DocumentVersion | DocumentUpdateResponse
): Promise<void> {
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
details: {
type: SyncType.DELETE,
relativePath: document.relativePath
},
message: "File has been deleted remotely, so we deleted it locally",
author: response.userId,
timestamp: new Date(response.updatedDate)
});
this.database.delete(document.relativePath);
this.database.updateDocumentMetadata(