andras/vault-link
1
0
Fork 0
Code Issues Pull requests 18 Projects Releases 55 Packages Wiki Activity Actions 1

Good catches

Browse source
This commit is contained in:
Andras Schmelczer 2026-04-26 19:35:46 +01:00
parent 0ab6984cdf
commit debe7cfc37
14 changed files with 201 additions and 63 deletions
Download patch file Download diff file Expand all files Collapse all files

1
frontend/sync-client/src/consts.ts
View file

@ -1,5 +1,4 @@
export const TIMEOUT_FOR_MERGING_HISTORY_ENTRIES_IN_SECONDS = 60;
export const DIFF_CACHE_SIZE_MB = 2;
export const MAX_LOG_MESSAGE_COUNT = 100000;
export const MAX_HISTORY_ENTRY_COUNT = 5000;
export const SUPPORTED_API_VERSION = 3;

40
frontend/sync-client/src/file-operations/file-operations.ts
View file

@ -80,7 +80,15 @@ export class FileOperations {
// existed, or it was just renamed away. The upcoming write therefore
// looks like a fresh create to the watcher.
this.expectedFsEvents.expectCreate(actualPath);
await this.fs.write(actualPath, this.toNativeLineEndings(newContent));
try {
await this.fs.write(
actualPath,
this.toNativeLineEndings(newContent)
);
} catch (e) {
this.expectedFsEvents.unexpectCreate(actualPath);
throw e;
}
return actualPath;
}
@ -102,12 +110,12 @@ export class FileOperations {
return;
}
// The exists() check above is racy: between it returning true and
// any of the writes below running, the file can be deleted. The
// safe wrapper around `atomicUpdateText` raises FileNotFoundError
// in that window — treat it the same as the upfront-missing case
// (skip silently) so callers see one consistent outcome regardless
// of when the deletion happened to occur.
// Single-source the expectation registration: register exactly once
// per call, and unexpect from the catch if the underlying fs op
// throws (FileNotFoundError or otherwise). The previous shape
// registered inside each branch and let the catch swallow
// FileNotFoundError, leaking the expectation into the map.
this.expectedFsEvents.expectUpdate(path);
try {
if (
!isFileTypeMergable(
@ -120,7 +128,6 @@ export class FileOperations {
this.logger.debug(
`The expected content is not mergable, so we won't perform a 3-way merge, just overwrite it`
);
this.expectedFsEvents.expectUpdate(path);
await this.fs.write(
path,
// `newContent` might not be binary so we still have to ensure the line endings are correct
@ -142,12 +149,10 @@ export class FileOperations {
this.logger.warn(
`3-way merge aborted for ${path}: one of expected/new is not valid UTF-8 (${decodeError}); falling back to overwrite`
);
this.expectedFsEvents.expectUpdate(path);
await this.fs.write(path, this.toNativeLineEndings(newContent));
return;
}
this.expectedFsEvents.expectUpdate(path);
await this.fs.atomicUpdateText(
path,
({ text, cursors }: TextWithCursors): TextWithCursors => {
@ -174,6 +179,7 @@ export class FileOperations {
}
);
} catch (e) {
this.expectedFsEvents.unexpectUpdate(path);
if (e instanceof FileNotFoundError) {
this.logger.debug(
`File ${path} disappeared during write; not recreating`
@ -187,7 +193,12 @@ export class FileOperations {
public async delete(path: RelativePath): Promise<void> {
if (await this.exists(path)) {
this.expectedFsEvents.expectDelete(path);
await this.fs.delete(path);
try {
await this.fs.delete(path);
} catch (e) {
this.expectedFsEvents.unexpectDelete(path);
throw e;
}
await this.deletingEmptyParentDirectoriesOfDeletedFile(path);
} else {
this.logger.debug(`No need to delete '${path}', it doesn't exist`);
@ -216,7 +227,12 @@ export class FileOperations {
const actualPath = await this.ensureClearPath(newPath, moveOnConflict);
this.expectedFsEvents.expectRename(oldPath, actualPath);
await this.fs.rename(oldPath, actualPath);
try {
await this.fs.rename(oldPath, actualPath);
} catch (e) {
this.expectedFsEvents.unexpectRename(oldPath, actualPath);
throw e;
}
await this.deletingEmptyParentDirectoriesOfDeletedFile(oldPath);
return actualPath;
}

2
frontend/sync-client/src/persistence/settings.ts
View file

@ -6,7 +6,6 @@ export interface SyncSettings {
remoteUri: string;
token: string;
vaultName: string;
syncConcurrency: number;
isSyncEnabled: boolean;
maxFileSizeMB: number;
ignorePatterns: string[];
@ -20,7 +19,6 @@ export const DEFAULT_SETTINGS: SyncSettings = {
remoteUri: "",
token: "",
vaultName: "default",
syncConcurrency: 1,
isSyncEnabled: false,
maxFileSizeMB: 10,
ignorePatterns: [],

8
frontend/sync-client/src/services/build-vault-url.ts Normal file
View file

@ -0,0 +1,8 @@
import type { Settings } from "../persistence/settings";
export function buildVaultUrl(settings: Settings, path: string): string {
const { vaultName, remoteUri } = settings.getSettings();
const remoteUriWithoutTrailingSlash = remoteUri.replace(/\/+$/, "");
const encodedVaultName = encodeURIComponent(vaultName.trim());
return `${remoteUriWithoutTrailingSlash}/vaults/${encodedVaultName}${path}`;
}

16
frontend/sync-client/src/services/server-config.ts
View file

@ -1,6 +1,7 @@
import { SUPPORTED_API_VERSION } from "../consts";
import { AuthenticationError } from "../errors/authentication-error";
import { ServerVersionMismatchError } from "../errors/server-version-mismatch-error";
import type { Settings } from "../persistence/settings";
import type { SyncService } from "./sync-service";
import type { PingResponse } from "./types/PingResponse";
@ -14,7 +15,20 @@ export class ServerConfig {
private response: Promise<PingResponse> | undefined;
private config: ServerConfigData | undefined;
public constructor(private readonly syncService: SyncService) { }
public constructor(
private readonly syncService: SyncService,
settings: Settings
) {
settings.onSettingsChanged.add((newSettings, oldSettings) => {
if (
newSettings.token !== oldSettings.token ||
newSettings.vaultName !== oldSettings.vaultName ||
newSettings.remoteUri !== oldSettings.remoteUri
) {
this.reset();
}
});
}
private static validateConfig(config: ServerConfigData): void {
if (config.supportedApiVersion !== SUPPORTED_API_VERSION) {

6
frontend/sync-client/src/services/sync-service.ts
View file

@ -17,6 +17,7 @@ import type { DocumentVersion } from "./types/DocumentVersion";
import type { FetchLatestDocumentsResponse } from "./types/FetchLatestDocumentsResponse";
import type { PingResponse } from "./types/PingResponse";
import type { UpdateTextDocumentVersion } from "./types/UpdateTextDocumentVersion";
import { buildVaultUrl } from "./build-vault-url";
export class SyncService {
private readonly client: typeof globalThis.fetch;
@ -385,10 +386,7 @@ export class SyncService {
}
private getUrl(path: string): string {
const { vaultName, remoteUri } = this.settings.getSettings();
const remoteUriWithoutTrailingSlash = remoteUri.replace(/\/+$/, "");
const encodedVaultName = encodeURIComponent(vaultName.trim());
return `${remoteUriWithoutTrailingSlash}/vaults/${encodedVaultName}${path}`;
return buildVaultUrl(this.settings, path);
}
private getDefaultHeaders(

9
frontend/sync-client/src/services/websocket-manager.ts
View file

@ -12,6 +12,7 @@ import {
import { removeFromArray } from "../utils/remove-from-array";
import { EventListeners } from "../utils/data-structures/event-listeners";
import { awaitAll } from "../utils/await-all";
import { buildVaultUrl } from "./build-vault-url";
export class WebSocketManager {
public readonly onWebSocketStatusChanged = new EventListeners<
@ -198,9 +199,11 @@ export class WebSocketManager {
this.outstandingPromises.length = 0;
}
const wsUri = new URL(this.settings.getSettings().remoteUri);
wsUri.protocol = wsUri.protocol === "https" ? "wss" : "ws";
wsUri.pathname = `/vaults/${this.settings.getSettings().vaultName}/ws`;
// Build the WS URL through the same vault-URL helper the HTTP client
// uses so vault-name encoding, trailing-slash stripping, and any path
// prefix in `remoteUri` stay in sync between transports.
const wsUri = new URL(buildVaultUrl(this.settings, "/ws"));
wsUri.protocol = wsUri.protocol.startsWith("https") ? "wss" : "ws";
this.logger.info(`Connecting to WebSocket at ${wsUri.toString()}`);

20
frontend/sync-client/src/sync-client.ts
View file

@ -23,7 +23,6 @@ import type { MaybeOutdatedClientCursors } from "./types/maybe-outdated-client-c
import { FileChangeNotifier } from "./sync-operations/file-change-notifier";
import { FixedSizeDocumentCache } from "./utils/data-structures/fix-sized-cache";
import { setUpTelemetry } from "./utils/set-up-telemetry";
import { DIFF_CACHE_SIZE_MB } from "./consts";
import { ServerConfig } from "./services/server-config";
import type { EventListeners } from "./utils/data-structures/event-listeners";
import { Lock } from "./utils/data-structures/locks";
@ -174,7 +173,7 @@ export class SyncClient {
fetch
);
const serverConfig = new ServerConfig(syncService);
const serverConfig = new ServerConfig(syncService, settings);
const expectedFsEvents = new ExpectedFsEvents();
@ -187,7 +186,7 @@ export class SyncClient {
);
const contentCache = new FixedSizeDocumentCache(
1024 * 1024 * DIFF_CACHE_SIZE_MB
1024 * 1024 * settings.getSettings().diffCacheSizeMB
);
const webSocketManager = new WebSocketManager(
@ -443,10 +442,21 @@ export class SyncClient {
* without tripping the public `checkIfDestroyed` guard, which exists
* only to keep external callers from continuing to use a disposed
* client.
*
* Loops because a WebSocket message handler completing is what enqueues
* a `RemoteChange` into the syncer; if we awaited the syncer first and
* the WS handler second, a message arriving mid-wait would leave a fresh
* drain pending while `save()` ran. Each iteration waits for both, then
* re-checks; we exit only once both report idle in the same pass.
*/
private async waitUntilFinishedInternal(): Promise<void> {
await this.syncer.waitUntilFinished();
await this.webSocketManager.waitUntilFinished();
while (
this.webSocketManager.hasOutstandingWork ||
this.syncer.hasPendingWork
) {
await this.webSocketManager.waitUntilFinished();
await this.syncer.waitUntilFinished();
}
await this.syncEventQueue.save();
}

7
frontend/sync-client/src/sync-operations/cursor-tracker.ts
View file

@ -93,6 +93,13 @@ export class CursorTracker {
await this.getDocumentsUpToDateness(clientCursor);
}
}
// Drop the local-cursor send-cache so the next call re-reads
// the file. The first cache key is the editor's input, which
// doesn't change when the file content does — without this,
// a remote update flipping the file from dirty back to clean
// would never re-send the cursor with a fresh `vaultUpdateId`.
this.lastLocalCursorStateJson = "";
this.lastLocalCursorStateWithoutDirtyDocumentsJson = "";
})
);
}

38
frontend/sync-client/src/sync-operations/expected-fs-events.ts
View file

@ -46,6 +46,38 @@ export class ExpectedFsEvents {
this.bump(this.renames, ExpectedFsEvents.renameKey(oldPath, newPath));
}
/**
* Cancel a previously-registered expectation when the fs op that registered
* it failed before any watcher event could fire. Without this, a leaked
* expectation silently swallows the next genuine user event at the same
* path (or, for renames, the same `oldPath → newPath` pair).
*
* Floored at zero: if the watcher *did* fire (op partially completed) and
* already consumed the entry, the unexpect is a no-op. The fallback is
* acceptable — at worst we re-upload a real edit we'd otherwise filter.
*/
public unexpectCreate(path: RelativePath): void {
this.decrement(this.creates, path);
}
public unexpectUpdate(path: RelativePath): void {
this.decrement(this.updates, path);
}
public unexpectDelete(path: RelativePath): void {
this.decrement(this.deletes, path);
}
public unexpectRename(
oldPath: RelativePath,
newPath: RelativePath
): void {
this.decrement(
this.renames,
ExpectedFsEvents.renameKey(oldPath, newPath)
);
}
public matchCreate(path: RelativePath): boolean {
return this.consume(this.creates, path);
}
@ -95,4 +127,10 @@ export class ExpectedFsEvents {
else {map.set(key, count - 1);}
return true;
}
private decrement(map: Map<RelativePath, number>, key: RelativePath): void {
const count = map.get(key) ?? 0;
if (count <= 1) {map.delete(key);}
else {map.set(key, count - 1);}
}
}

35
frontend/sync-client/src/sync-operations/sync-event-queue.ts
View file

@ -3,6 +3,7 @@ import type { Logger } from "../tracing/logger";
import { globsToRegexes } from "../utils/globs-to-regexes";
import { CONFLICT_PATH_REGEX } from "./conflict-path";
import { removeFromArray } from "../utils/remove-from-array";
import { EventListeners } from "../utils/data-structures/event-listeners";
import type { DocumentWithPath } from "./types";
import {
SyncEventType,
@ -17,6 +18,14 @@ import {
import { MinCovered } from "../utils/data-structures/min-covered";
export class SyncEventQueue {
// Fires synchronously whenever the events array length changes (push, pop,
// remove, bulk-clear). The Syncer mirrors this into its public count
// listener; without this hook, listeners only saw deltas at consume time
// and missed the "queue grew" / "queue cleared on reset" transitions.
public readonly onPendingUpdateCountChanged = new EventListeners<
(count: number) => unknown
>();
private readonly _lastSeenUpdateId: MinCovered;
// Latest state of the filesystem as we know it, excluding
@ -123,6 +132,7 @@ export class SyncEventQueue {
if (input.type === SyncEventType.RemoteChange) {
this.events.push(input);
this.notifyPendingUpdateCountChanged();
return;
}
@ -154,6 +164,7 @@ export class SyncEventQueue {
originalPath: path,
resolvers: Promise.withResolvers()
});
this.notifyPendingUpdateCountChanged();
return;
}
@ -180,6 +191,7 @@ export class SyncEventQueue {
type: SyncEventType.LocalDelete,
documentId: (pendingDocumentId ?? documentId)!
});
this.notifyPendingUpdateCountChanged();
return;
}
@ -219,6 +231,7 @@ export class SyncEventQueue {
path,
originalPath: path
});
this.notifyPendingUpdateCountChanged();
if (needsSave) {
await this.save();
@ -226,7 +239,11 @@ export class SyncEventQueue {
}
public async next(): Promise<SyncEvent | undefined> {
return this.events.shift();
const event = this.events.shift();
if (event !== undefined) {
this.notifyPendingUpdateCountChanged();
}
return event;
}
@ -250,7 +267,9 @@ export class SyncEventQueue {
* remote-create handler just absorbed).
*/
public consumeEvent(event: SyncEvent): void {
removeFromArray(this.events, event);
if (removeFromArray(this.events, event)) {
this.notifyPendingUpdateCountChanged();
}
}
@ -261,7 +280,9 @@ export class SyncEventQueue {
event: Extract<SyncEvent, { type: SyncEventType.LocalCreate }>,
record: DocumentRecord
): Promise<void> {
removeFromArray(this.events, event); // in case the create event is still pending
if (removeFromArray(this.events, event)) {
this.notifyPendingUpdateCountChanged();
}
await this.setDocument(event.path, record);
event.resolvers.resolve(record.documentId);
}
@ -376,8 +397,16 @@ export class SyncEventQueue {
}
public clearPending(): void {
const hadEvents = this.events.length > 0;
this.rejectAllPendingCreates();
this.events.length = 0;
if (hadEvents) {
this.notifyPendingUpdateCountChanged();
}
}
private notifyPendingUpdateCountChanged(): void {
this.onPendingUpdateCountChanged.trigger(this.events.length);
}
public findLatestCreateForPath(

40
frontend/sync-client/src/sync-operations/syncer.ts
View file

@ -28,7 +28,7 @@ import type { SyncHistory } from "../tracing/sync-history";
import {
SyncStatus,
SyncType,
type CommonHistoryEntry
type HistoryEntry
} from "../tracing/sync-history";
import { isBinary } from "../utils/is-binary";
import { isFileTypeMergable } from "../utils/is-file-type-mergable";
@ -72,6 +72,12 @@ export class Syncer {
this.webSocketManager.onRemoteVaultUpdateReceived.add(
this.syncRemotelyUpdatedFile.bind(this)
);
// Funnel every queue mutation (enqueue, consume, clearPending) through
// the public count notifier so listeners see grow/shrink transitions
// immediately rather than only when a drain consumes an event.
this.queue.onPendingUpdateCountChanged.add(() => {
this.notifyRemainingOperationsChanged();
});
}
public syncLocallyCreatedFile(relativePath: RelativePath): void {
@ -152,9 +158,24 @@ export class Syncer {
}
}
/**
* True while there is queued or in-flight work the syncer needs to handle:
* a running offline scan, an active drain, or pending events. Used by
* `SyncClient.waitUntilFinishedInternal` to detect WebSocket-fed work that
* landed in the queue after the syncer's first quiescence point.
*/
public get hasPendingWork(): boolean {
return (
this.runningScheduleSyncForOfflineChanges !== undefined ||
this.drainPromise !== undefined ||
this.queue.pendingUpdateCount > 0
);
}
public reset(): void {
this.queue.clearPending();
this.clearOfflineScanGate();
this.previousRemainingOperationsCount = 0;
}
/**
@ -350,13 +371,19 @@ export class Syncer {
event.resolvers.reject(new Error("Create was cancelled"));
}
// Advance the cursor so the server doesn't replay this update on every
// reconnect — the skip is permanent for this version.
if (event.type === SyncEventType.RemoteChange) {
this.queue.lastSeenUpdateId = event.remoteVersion.vaultUpdateId;
}
return true;
}
private getHistoryEntryForSkippedOversizedFile(
sizeInBytes: number,
relativePath: RelativePath
): CommonHistoryEntry | undefined {
): HistoryEntry | undefined {
const sizeInMB = Math.round(sizeInBytes / 1024 / 1024);
const { maxFileSizeMB } = this.settings.getSettings();
if (sizeInMB > maxFileSizeMB) {
@ -366,7 +393,8 @@ export class Syncer {
type: SyncType.SKIPPED as const,
relativePath
},
message: `File size of ${sizeInMB} MB exceeds the maximum file size limit of ${maxFileSizeMB} MB`
message: `File size of ${sizeInMB} MB exceeds the maximum file size limit of ${maxFileSizeMB} MB`,
timestamp: new Date()
};
}
}
@ -429,7 +457,6 @@ export class Syncer {
// and history entry. Keeping the entry in the map until then lets
// late remote updates be recognised as "file is missing" and
// skipped, instead of resurrecting the doc.
//
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
details: {
@ -437,7 +464,8 @@ export class Syncer {
relativePath: doc.path
},
message: "Successfully deleted file on the server",
author: response.userId
author: response.userId,
timestamp: new Date(response.updatedDate)
});
}
@ -482,8 +510,6 @@ export class Syncer {
return;
}
this.queue.lastSeenUpdateId = response.vaultUpdateId;
await this.handleMaybeMergingResponse({
path: diskPath,
response,

27
frontend/sync-client/src/tracing/sync-history.ts
View file

@ -40,12 +40,15 @@ export type SyncDetails =
| SyncMovedDetails
| SyncSkippedDetails;
export interface CommonHistoryEntry {
export interface HistoryEntry {
status: SyncStatus;
message: string;
details: SyncDetails;
timestamp: Date;
// `author` is the server-side user id and only exists for entries that
// round-tripped through the server. Local-only entries (e.g. SKIPPED)
// legitimately have no author.
author?: string;
timestamp?: Date;
}
export enum SyncType {
@ -62,7 +65,6 @@ export enum SyncStatus {
SKIPPED = "SKIPPED"
}
export type HistoryEntry = CommonHistoryEntry & { timestamp: Date };
export interface HistoryStats {
success: number;
@ -81,7 +83,7 @@ export class SyncHistory {
error: 0
};
public constructor(private readonly logger: Logger) {}
public constructor(private readonly logger: Logger) { }
public get entries(): readonly HistoryEntry[] {
return this._entries;
@ -93,25 +95,20 @@ export class SyncHistory {
*
* If the entry list is too long, the oldest entry will be removed.
*/
public addHistoryEntry(entry: CommonHistoryEntry): void {
const historyEntry = {
...entry,
timestamp: entry.timestamp ?? new Date()
};
const candidate = this.findSimilarRecentUpdateEntry(historyEntry);
public addHistoryEntry(entry: HistoryEntry): void {
const candidate = this.findSimilarRecentUpdateEntry(entry);
if (candidate !== undefined) {
removeFromArray(this._entries, candidate);
}
// Insert the entry at the beginning
this._entries.unshift(historyEntry);
this._entries.unshift(entry);
if (this._entries.length > MAX_HISTORY_ENTRY_COUNT) {
this._entries.pop();
}
this.updateSuccessCount(historyEntry);
this.updateSuccessCount(entry);
}
public reset(): void {
@ -139,8 +136,8 @@ export class SyncHistory {
candidate !== undefined &&
(this._entries[0] === candidate ||
candidate.timestamp.getTime() +
TIMEOUT_FOR_MERGING_HISTORY_ENTRIES_IN_SECONDS * 1000 >
entry.timestamp.getTime())
TIMEOUT_FOR_MERGING_HISTORY_ENTRIES_IN_SECONDS * 1000 >
entry.timestamp.getTime())
) {
return candidate;
}

15
frontend/test-client/src/agent/mock-agent.ts
View file

@ -261,11 +261,11 @@ export class MockAgent extends MockClient {
);
otherAgent.client.logger.info(
"Other agent's data: " +
JSON.stringify(otherAgent.data, null, 2)
JSON.stringify(otherAgent.data, null, 2)
);
otherAgent.client.logger.info(
"Other agent's files: " +
Array.from(otherAgent.files.keys()).join(", ")
Array.from(otherAgent.files.keys()).join(", ")
);
throw e;
@ -339,12 +339,7 @@ export class MockAgent extends MockClient {
);
}
if (!this.useSlowFileEvents && !this.doDeletes) {
assert(
found.length >= 1,
`[${this.name}] Binary content ${content} not found in any files`
);
}
// can't assert(found.length >= 1, ...); because binary files have LWW semantics
}
}
@ -531,9 +526,9 @@ export class MockAgent extends MockClient {
private removeBinaryUuid(file: string): void {
const existing = this.files.get(file);
if (existing === undefined) {return;}
if (existing === undefined) { return; }
const content = new TextDecoder().decode(existing);
if (!content.startsWith("BINARY:")) {return;}
if (!content.startsWith("BINARY:")) { return; }
const uuid = content.slice("BINARY:".length);
utils.removeFromArray(this.writtenBinaryContents, uuid);
}