Good catches
This commit is contained in:
parent
0ab6984cdf
commit
debe7cfc37
14 changed files with 201 additions and 63 deletions
|
|
@ -1,5 +1,4 @@
|
||||||
export const TIMEOUT_FOR_MERGING_HISTORY_ENTRIES_IN_SECONDS = 60;
|
export const TIMEOUT_FOR_MERGING_HISTORY_ENTRIES_IN_SECONDS = 60;
|
||||||
export const DIFF_CACHE_SIZE_MB = 2;
|
|
||||||
export const MAX_LOG_MESSAGE_COUNT = 100000;
|
export const MAX_LOG_MESSAGE_COUNT = 100000;
|
||||||
export const MAX_HISTORY_ENTRY_COUNT = 5000;
|
export const MAX_HISTORY_ENTRY_COUNT = 5000;
|
||||||
export const SUPPORTED_API_VERSION = 3;
|
export const SUPPORTED_API_VERSION = 3;
|
||||||
|
|
|
||||||
|
|
@ -80,7 +80,15 @@ export class FileOperations {
|
||||||
// existed, or it was just renamed away. The upcoming write therefore
|
// existed, or it was just renamed away. The upcoming write therefore
|
||||||
// looks like a fresh create to the watcher.
|
// looks like a fresh create to the watcher.
|
||||||
this.expectedFsEvents.expectCreate(actualPath);
|
this.expectedFsEvents.expectCreate(actualPath);
|
||||||
await this.fs.write(actualPath, this.toNativeLineEndings(newContent));
|
try {
|
||||||
|
await this.fs.write(
|
||||||
|
actualPath,
|
||||||
|
this.toNativeLineEndings(newContent)
|
||||||
|
);
|
||||||
|
} catch (e) {
|
||||||
|
this.expectedFsEvents.unexpectCreate(actualPath);
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
return actualPath;
|
return actualPath;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -102,12 +110,12 @@ export class FileOperations {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// The exists() check above is racy: between it returning true and
|
// Single-source the expectation registration: register exactly once
|
||||||
// any of the writes below running, the file can be deleted. The
|
// per call, and unexpect from the catch if the underlying fs op
|
||||||
// safe wrapper around `atomicUpdateText` raises FileNotFoundError
|
// throws (FileNotFoundError or otherwise). The previous shape
|
||||||
// in that window — treat it the same as the upfront-missing case
|
// registered inside each branch and let the catch swallow
|
||||||
// (skip silently) so callers see one consistent outcome regardless
|
// FileNotFoundError, leaking the expectation into the map.
|
||||||
// of when the deletion happened to occur.
|
this.expectedFsEvents.expectUpdate(path);
|
||||||
try {
|
try {
|
||||||
if (
|
if (
|
||||||
!isFileTypeMergable(
|
!isFileTypeMergable(
|
||||||
|
|
@ -120,7 +128,6 @@ export class FileOperations {
|
||||||
this.logger.debug(
|
this.logger.debug(
|
||||||
`The expected content is not mergable, so we won't perform a 3-way merge, just overwrite it`
|
`The expected content is not mergable, so we won't perform a 3-way merge, just overwrite it`
|
||||||
);
|
);
|
||||||
this.expectedFsEvents.expectUpdate(path);
|
|
||||||
await this.fs.write(
|
await this.fs.write(
|
||||||
path,
|
path,
|
||||||
// `newContent` might not be binary so we still have to ensure the line endings are correct
|
// `newContent` might not be binary so we still have to ensure the line endings are correct
|
||||||
|
|
@ -142,12 +149,10 @@ export class FileOperations {
|
||||||
this.logger.warn(
|
this.logger.warn(
|
||||||
`3-way merge aborted for ${path}: one of expected/new is not valid UTF-8 (${decodeError}); falling back to overwrite`
|
`3-way merge aborted for ${path}: one of expected/new is not valid UTF-8 (${decodeError}); falling back to overwrite`
|
||||||
);
|
);
|
||||||
this.expectedFsEvents.expectUpdate(path);
|
|
||||||
await this.fs.write(path, this.toNativeLineEndings(newContent));
|
await this.fs.write(path, this.toNativeLineEndings(newContent));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
this.expectedFsEvents.expectUpdate(path);
|
|
||||||
await this.fs.atomicUpdateText(
|
await this.fs.atomicUpdateText(
|
||||||
path,
|
path,
|
||||||
({ text, cursors }: TextWithCursors): TextWithCursors => {
|
({ text, cursors }: TextWithCursors): TextWithCursors => {
|
||||||
|
|
@ -174,6 +179,7 @@ export class FileOperations {
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
|
this.expectedFsEvents.unexpectUpdate(path);
|
||||||
if (e instanceof FileNotFoundError) {
|
if (e instanceof FileNotFoundError) {
|
||||||
this.logger.debug(
|
this.logger.debug(
|
||||||
`File ${path} disappeared during write; not recreating`
|
`File ${path} disappeared during write; not recreating`
|
||||||
|
|
@ -187,7 +193,12 @@ export class FileOperations {
|
||||||
public async delete(path: RelativePath): Promise<void> {
|
public async delete(path: RelativePath): Promise<void> {
|
||||||
if (await this.exists(path)) {
|
if (await this.exists(path)) {
|
||||||
this.expectedFsEvents.expectDelete(path);
|
this.expectedFsEvents.expectDelete(path);
|
||||||
await this.fs.delete(path);
|
try {
|
||||||
|
await this.fs.delete(path);
|
||||||
|
} catch (e) {
|
||||||
|
this.expectedFsEvents.unexpectDelete(path);
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
await this.deletingEmptyParentDirectoriesOfDeletedFile(path);
|
await this.deletingEmptyParentDirectoriesOfDeletedFile(path);
|
||||||
} else {
|
} else {
|
||||||
this.logger.debug(`No need to delete '${path}', it doesn't exist`);
|
this.logger.debug(`No need to delete '${path}', it doesn't exist`);
|
||||||
|
|
@ -216,7 +227,12 @@ export class FileOperations {
|
||||||
|
|
||||||
const actualPath = await this.ensureClearPath(newPath, moveOnConflict);
|
const actualPath = await this.ensureClearPath(newPath, moveOnConflict);
|
||||||
this.expectedFsEvents.expectRename(oldPath, actualPath);
|
this.expectedFsEvents.expectRename(oldPath, actualPath);
|
||||||
await this.fs.rename(oldPath, actualPath);
|
try {
|
||||||
|
await this.fs.rename(oldPath, actualPath);
|
||||||
|
} catch (e) {
|
||||||
|
this.expectedFsEvents.unexpectRename(oldPath, actualPath);
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
await this.deletingEmptyParentDirectoriesOfDeletedFile(oldPath);
|
await this.deletingEmptyParentDirectoriesOfDeletedFile(oldPath);
|
||||||
return actualPath;
|
return actualPath;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,6 @@ export interface SyncSettings {
|
||||||
remoteUri: string;
|
remoteUri: string;
|
||||||
token: string;
|
token: string;
|
||||||
vaultName: string;
|
vaultName: string;
|
||||||
syncConcurrency: number;
|
|
||||||
isSyncEnabled: boolean;
|
isSyncEnabled: boolean;
|
||||||
maxFileSizeMB: number;
|
maxFileSizeMB: number;
|
||||||
ignorePatterns: string[];
|
ignorePatterns: string[];
|
||||||
|
|
@ -20,7 +19,6 @@ export const DEFAULT_SETTINGS: SyncSettings = {
|
||||||
remoteUri: "",
|
remoteUri: "",
|
||||||
token: "",
|
token: "",
|
||||||
vaultName: "default",
|
vaultName: "default",
|
||||||
syncConcurrency: 1,
|
|
||||||
isSyncEnabled: false,
|
isSyncEnabled: false,
|
||||||
maxFileSizeMB: 10,
|
maxFileSizeMB: 10,
|
||||||
ignorePatterns: [],
|
ignorePatterns: [],
|
||||||
|
|
|
||||||
8
frontend/sync-client/src/services/build-vault-url.ts
Normal file
8
frontend/sync-client/src/services/build-vault-url.ts
Normal file
|
|
@ -0,0 +1,8 @@
|
||||||
|
import type { Settings } from "../persistence/settings";
|
||||||
|
|
||||||
|
export function buildVaultUrl(settings: Settings, path: string): string {
|
||||||
|
const { vaultName, remoteUri } = settings.getSettings();
|
||||||
|
const remoteUriWithoutTrailingSlash = remoteUri.replace(/\/+$/, "");
|
||||||
|
const encodedVaultName = encodeURIComponent(vaultName.trim());
|
||||||
|
return `${remoteUriWithoutTrailingSlash}/vaults/${encodedVaultName}${path}`;
|
||||||
|
}
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
import { SUPPORTED_API_VERSION } from "../consts";
|
import { SUPPORTED_API_VERSION } from "../consts";
|
||||||
import { AuthenticationError } from "../errors/authentication-error";
|
import { AuthenticationError } from "../errors/authentication-error";
|
||||||
import { ServerVersionMismatchError } from "../errors/server-version-mismatch-error";
|
import { ServerVersionMismatchError } from "../errors/server-version-mismatch-error";
|
||||||
|
import type { Settings } from "../persistence/settings";
|
||||||
import type { SyncService } from "./sync-service";
|
import type { SyncService } from "./sync-service";
|
||||||
import type { PingResponse } from "./types/PingResponse";
|
import type { PingResponse } from "./types/PingResponse";
|
||||||
|
|
||||||
|
|
@ -14,7 +15,20 @@ export class ServerConfig {
|
||||||
private response: Promise<PingResponse> | undefined;
|
private response: Promise<PingResponse> | undefined;
|
||||||
private config: ServerConfigData | undefined;
|
private config: ServerConfigData | undefined;
|
||||||
|
|
||||||
public constructor(private readonly syncService: SyncService) { }
|
public constructor(
|
||||||
|
private readonly syncService: SyncService,
|
||||||
|
settings: Settings
|
||||||
|
) {
|
||||||
|
settings.onSettingsChanged.add((newSettings, oldSettings) => {
|
||||||
|
if (
|
||||||
|
newSettings.token !== oldSettings.token ||
|
||||||
|
newSettings.vaultName !== oldSettings.vaultName ||
|
||||||
|
newSettings.remoteUri !== oldSettings.remoteUri
|
||||||
|
) {
|
||||||
|
this.reset();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
private static validateConfig(config: ServerConfigData): void {
|
private static validateConfig(config: ServerConfigData): void {
|
||||||
if (config.supportedApiVersion !== SUPPORTED_API_VERSION) {
|
if (config.supportedApiVersion !== SUPPORTED_API_VERSION) {
|
||||||
|
|
|
||||||
|
|
@ -17,6 +17,7 @@ import type { DocumentVersion } from "./types/DocumentVersion";
|
||||||
import type { FetchLatestDocumentsResponse } from "./types/FetchLatestDocumentsResponse";
|
import type { FetchLatestDocumentsResponse } from "./types/FetchLatestDocumentsResponse";
|
||||||
import type { PingResponse } from "./types/PingResponse";
|
import type { PingResponse } from "./types/PingResponse";
|
||||||
import type { UpdateTextDocumentVersion } from "./types/UpdateTextDocumentVersion";
|
import type { UpdateTextDocumentVersion } from "./types/UpdateTextDocumentVersion";
|
||||||
|
import { buildVaultUrl } from "./build-vault-url";
|
||||||
|
|
||||||
export class SyncService {
|
export class SyncService {
|
||||||
private readonly client: typeof globalThis.fetch;
|
private readonly client: typeof globalThis.fetch;
|
||||||
|
|
@ -385,10 +386,7 @@ export class SyncService {
|
||||||
}
|
}
|
||||||
|
|
||||||
private getUrl(path: string): string {
|
private getUrl(path: string): string {
|
||||||
const { vaultName, remoteUri } = this.settings.getSettings();
|
return buildVaultUrl(this.settings, path);
|
||||||
const remoteUriWithoutTrailingSlash = remoteUri.replace(/\/+$/, "");
|
|
||||||
const encodedVaultName = encodeURIComponent(vaultName.trim());
|
|
||||||
return `${remoteUriWithoutTrailingSlash}/vaults/${encodedVaultName}${path}`;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private getDefaultHeaders(
|
private getDefaultHeaders(
|
||||||
|
|
|
||||||
|
|
@ -12,6 +12,7 @@ import {
|
||||||
import { removeFromArray } from "../utils/remove-from-array";
|
import { removeFromArray } from "../utils/remove-from-array";
|
||||||
import { EventListeners } from "../utils/data-structures/event-listeners";
|
import { EventListeners } from "../utils/data-structures/event-listeners";
|
||||||
import { awaitAll } from "../utils/await-all";
|
import { awaitAll } from "../utils/await-all";
|
||||||
|
import { buildVaultUrl } from "./build-vault-url";
|
||||||
|
|
||||||
export class WebSocketManager {
|
export class WebSocketManager {
|
||||||
public readonly onWebSocketStatusChanged = new EventListeners<
|
public readonly onWebSocketStatusChanged = new EventListeners<
|
||||||
|
|
@ -198,9 +199,11 @@ export class WebSocketManager {
|
||||||
this.outstandingPromises.length = 0;
|
this.outstandingPromises.length = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
const wsUri = new URL(this.settings.getSettings().remoteUri);
|
// Build the WS URL through the same vault-URL helper the HTTP client
|
||||||
wsUri.protocol = wsUri.protocol === "https" ? "wss" : "ws";
|
// uses so vault-name encoding, trailing-slash stripping, and any path
|
||||||
wsUri.pathname = `/vaults/${this.settings.getSettings().vaultName}/ws`;
|
// prefix in `remoteUri` stay in sync between transports.
|
||||||
|
const wsUri = new URL(buildVaultUrl(this.settings, "/ws"));
|
||||||
|
wsUri.protocol = wsUri.protocol.startsWith("https") ? "wss" : "ws";
|
||||||
|
|
||||||
this.logger.info(`Connecting to WebSocket at ${wsUri.toString()}`);
|
this.logger.info(`Connecting to WebSocket at ${wsUri.toString()}`);
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -23,7 +23,6 @@ import type { MaybeOutdatedClientCursors } from "./types/maybe-outdated-client-c
|
||||||
import { FileChangeNotifier } from "./sync-operations/file-change-notifier";
|
import { FileChangeNotifier } from "./sync-operations/file-change-notifier";
|
||||||
import { FixedSizeDocumentCache } from "./utils/data-structures/fix-sized-cache";
|
import { FixedSizeDocumentCache } from "./utils/data-structures/fix-sized-cache";
|
||||||
import { setUpTelemetry } from "./utils/set-up-telemetry";
|
import { setUpTelemetry } from "./utils/set-up-telemetry";
|
||||||
import { DIFF_CACHE_SIZE_MB } from "./consts";
|
|
||||||
import { ServerConfig } from "./services/server-config";
|
import { ServerConfig } from "./services/server-config";
|
||||||
import type { EventListeners } from "./utils/data-structures/event-listeners";
|
import type { EventListeners } from "./utils/data-structures/event-listeners";
|
||||||
import { Lock } from "./utils/data-structures/locks";
|
import { Lock } from "./utils/data-structures/locks";
|
||||||
|
|
@ -174,7 +173,7 @@ export class SyncClient {
|
||||||
fetch
|
fetch
|
||||||
);
|
);
|
||||||
|
|
||||||
const serverConfig = new ServerConfig(syncService);
|
const serverConfig = new ServerConfig(syncService, settings);
|
||||||
|
|
||||||
const expectedFsEvents = new ExpectedFsEvents();
|
const expectedFsEvents = new ExpectedFsEvents();
|
||||||
|
|
||||||
|
|
@ -187,7 +186,7 @@ export class SyncClient {
|
||||||
);
|
);
|
||||||
|
|
||||||
const contentCache = new FixedSizeDocumentCache(
|
const contentCache = new FixedSizeDocumentCache(
|
||||||
1024 * 1024 * DIFF_CACHE_SIZE_MB
|
1024 * 1024 * settings.getSettings().diffCacheSizeMB
|
||||||
);
|
);
|
||||||
|
|
||||||
const webSocketManager = new WebSocketManager(
|
const webSocketManager = new WebSocketManager(
|
||||||
|
|
@ -443,10 +442,21 @@ export class SyncClient {
|
||||||
* without tripping the public `checkIfDestroyed` guard, which exists
|
* without tripping the public `checkIfDestroyed` guard, which exists
|
||||||
* only to keep external callers from continuing to use a disposed
|
* only to keep external callers from continuing to use a disposed
|
||||||
* client.
|
* client.
|
||||||
|
*
|
||||||
|
* Loops because a WebSocket message handler completing is what enqueues
|
||||||
|
* a `RemoteChange` into the syncer; if we awaited the syncer first and
|
||||||
|
* the WS handler second, a message arriving mid-wait would leave a fresh
|
||||||
|
* drain pending while `save()` ran. Each iteration waits for both, then
|
||||||
|
* re-checks; we exit only once both report idle in the same pass.
|
||||||
*/
|
*/
|
||||||
private async waitUntilFinishedInternal(): Promise<void> {
|
private async waitUntilFinishedInternal(): Promise<void> {
|
||||||
await this.syncer.waitUntilFinished();
|
while (
|
||||||
await this.webSocketManager.waitUntilFinished();
|
this.webSocketManager.hasOutstandingWork ||
|
||||||
|
this.syncer.hasPendingWork
|
||||||
|
) {
|
||||||
|
await this.webSocketManager.waitUntilFinished();
|
||||||
|
await this.syncer.waitUntilFinished();
|
||||||
|
}
|
||||||
await this.syncEventQueue.save();
|
await this.syncEventQueue.save();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -93,6 +93,13 @@ export class CursorTracker {
|
||||||
await this.getDocumentsUpToDateness(clientCursor);
|
await this.getDocumentsUpToDateness(clientCursor);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Drop the local-cursor send-cache so the next call re-reads
|
||||||
|
// the file. The first cache key is the editor's input, which
|
||||||
|
// doesn't change when the file content does — without this,
|
||||||
|
// a remote update flipping the file from dirty back to clean
|
||||||
|
// would never re-send the cursor with a fresh `vaultUpdateId`.
|
||||||
|
this.lastLocalCursorStateJson = "";
|
||||||
|
this.lastLocalCursorStateWithoutDirtyDocumentsJson = "";
|
||||||
})
|
})
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -46,6 +46,38 @@ export class ExpectedFsEvents {
|
||||||
this.bump(this.renames, ExpectedFsEvents.renameKey(oldPath, newPath));
|
this.bump(this.renames, ExpectedFsEvents.renameKey(oldPath, newPath));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Cancel a previously-registered expectation when the fs op that registered
|
||||||
|
* it failed before any watcher event could fire. Without this, a leaked
|
||||||
|
* expectation silently swallows the next genuine user event at the same
|
||||||
|
* path (or, for renames, the same `oldPath → newPath` pair).
|
||||||
|
*
|
||||||
|
* Floored at zero: if the watcher *did* fire (op partially completed) and
|
||||||
|
* already consumed the entry, the unexpect is a no-op. The fallback is
|
||||||
|
* acceptable — at worst we re-upload a real edit we'd otherwise filter.
|
||||||
|
*/
|
||||||
|
public unexpectCreate(path: RelativePath): void {
|
||||||
|
this.decrement(this.creates, path);
|
||||||
|
}
|
||||||
|
|
||||||
|
public unexpectUpdate(path: RelativePath): void {
|
||||||
|
this.decrement(this.updates, path);
|
||||||
|
}
|
||||||
|
|
||||||
|
public unexpectDelete(path: RelativePath): void {
|
||||||
|
this.decrement(this.deletes, path);
|
||||||
|
}
|
||||||
|
|
||||||
|
public unexpectRename(
|
||||||
|
oldPath: RelativePath,
|
||||||
|
newPath: RelativePath
|
||||||
|
): void {
|
||||||
|
this.decrement(
|
||||||
|
this.renames,
|
||||||
|
ExpectedFsEvents.renameKey(oldPath, newPath)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
public matchCreate(path: RelativePath): boolean {
|
public matchCreate(path: RelativePath): boolean {
|
||||||
return this.consume(this.creates, path);
|
return this.consume(this.creates, path);
|
||||||
}
|
}
|
||||||
|
|
@ -95,4 +127,10 @@ export class ExpectedFsEvents {
|
||||||
else {map.set(key, count - 1);}
|
else {map.set(key, count - 1);}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private decrement(map: Map<RelativePath, number>, key: RelativePath): void {
|
||||||
|
const count = map.get(key) ?? 0;
|
||||||
|
if (count <= 1) {map.delete(key);}
|
||||||
|
else {map.set(key, count - 1);}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,7 @@ import type { Logger } from "../tracing/logger";
|
||||||
import { globsToRegexes } from "../utils/globs-to-regexes";
|
import { globsToRegexes } from "../utils/globs-to-regexes";
|
||||||
import { CONFLICT_PATH_REGEX } from "./conflict-path";
|
import { CONFLICT_PATH_REGEX } from "./conflict-path";
|
||||||
import { removeFromArray } from "../utils/remove-from-array";
|
import { removeFromArray } from "../utils/remove-from-array";
|
||||||
|
import { EventListeners } from "../utils/data-structures/event-listeners";
|
||||||
import type { DocumentWithPath } from "./types";
|
import type { DocumentWithPath } from "./types";
|
||||||
import {
|
import {
|
||||||
SyncEventType,
|
SyncEventType,
|
||||||
|
|
@ -17,6 +18,14 @@ import {
|
||||||
import { MinCovered } from "../utils/data-structures/min-covered";
|
import { MinCovered } from "../utils/data-structures/min-covered";
|
||||||
|
|
||||||
export class SyncEventQueue {
|
export class SyncEventQueue {
|
||||||
|
// Fires synchronously whenever the events array length changes (push, pop,
|
||||||
|
// remove, bulk-clear). The Syncer mirrors this into its public count
|
||||||
|
// listener; without this hook, listeners only saw deltas at consume time
|
||||||
|
// and missed the "queue grew" / "queue cleared on reset" transitions.
|
||||||
|
public readonly onPendingUpdateCountChanged = new EventListeners<
|
||||||
|
(count: number) => unknown
|
||||||
|
>();
|
||||||
|
|
||||||
private readonly _lastSeenUpdateId: MinCovered;
|
private readonly _lastSeenUpdateId: MinCovered;
|
||||||
|
|
||||||
// Latest state of the filesystem as we know it, excluding
|
// Latest state of the filesystem as we know it, excluding
|
||||||
|
|
@ -123,6 +132,7 @@ export class SyncEventQueue {
|
||||||
|
|
||||||
if (input.type === SyncEventType.RemoteChange) {
|
if (input.type === SyncEventType.RemoteChange) {
|
||||||
this.events.push(input);
|
this.events.push(input);
|
||||||
|
this.notifyPendingUpdateCountChanged();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -154,6 +164,7 @@ export class SyncEventQueue {
|
||||||
originalPath: path,
|
originalPath: path,
|
||||||
resolvers: Promise.withResolvers()
|
resolvers: Promise.withResolvers()
|
||||||
});
|
});
|
||||||
|
this.notifyPendingUpdateCountChanged();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -180,6 +191,7 @@ export class SyncEventQueue {
|
||||||
type: SyncEventType.LocalDelete,
|
type: SyncEventType.LocalDelete,
|
||||||
documentId: (pendingDocumentId ?? documentId)!
|
documentId: (pendingDocumentId ?? documentId)!
|
||||||
});
|
});
|
||||||
|
this.notifyPendingUpdateCountChanged();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -219,6 +231,7 @@ export class SyncEventQueue {
|
||||||
path,
|
path,
|
||||||
originalPath: path
|
originalPath: path
|
||||||
});
|
});
|
||||||
|
this.notifyPendingUpdateCountChanged();
|
||||||
|
|
||||||
if (needsSave) {
|
if (needsSave) {
|
||||||
await this.save();
|
await this.save();
|
||||||
|
|
@ -226,7 +239,11 @@ export class SyncEventQueue {
|
||||||
}
|
}
|
||||||
|
|
||||||
public async next(): Promise<SyncEvent | undefined> {
|
public async next(): Promise<SyncEvent | undefined> {
|
||||||
return this.events.shift();
|
const event = this.events.shift();
|
||||||
|
if (event !== undefined) {
|
||||||
|
this.notifyPendingUpdateCountChanged();
|
||||||
|
}
|
||||||
|
return event;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -250,7 +267,9 @@ export class SyncEventQueue {
|
||||||
* remote-create handler just absorbed).
|
* remote-create handler just absorbed).
|
||||||
*/
|
*/
|
||||||
public consumeEvent(event: SyncEvent): void {
|
public consumeEvent(event: SyncEvent): void {
|
||||||
removeFromArray(this.events, event);
|
if (removeFromArray(this.events, event)) {
|
||||||
|
this.notifyPendingUpdateCountChanged();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -261,7 +280,9 @@ export class SyncEventQueue {
|
||||||
event: Extract<SyncEvent, { type: SyncEventType.LocalCreate }>,
|
event: Extract<SyncEvent, { type: SyncEventType.LocalCreate }>,
|
||||||
record: DocumentRecord
|
record: DocumentRecord
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
removeFromArray(this.events, event); // in case the create event is still pending
|
if (removeFromArray(this.events, event)) {
|
||||||
|
this.notifyPendingUpdateCountChanged();
|
||||||
|
}
|
||||||
await this.setDocument(event.path, record);
|
await this.setDocument(event.path, record);
|
||||||
event.resolvers.resolve(record.documentId);
|
event.resolvers.resolve(record.documentId);
|
||||||
}
|
}
|
||||||
|
|
@ -376,8 +397,16 @@ export class SyncEventQueue {
|
||||||
}
|
}
|
||||||
|
|
||||||
public clearPending(): void {
|
public clearPending(): void {
|
||||||
|
const hadEvents = this.events.length > 0;
|
||||||
this.rejectAllPendingCreates();
|
this.rejectAllPendingCreates();
|
||||||
this.events.length = 0;
|
this.events.length = 0;
|
||||||
|
if (hadEvents) {
|
||||||
|
this.notifyPendingUpdateCountChanged();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private notifyPendingUpdateCountChanged(): void {
|
||||||
|
this.onPendingUpdateCountChanged.trigger(this.events.length);
|
||||||
}
|
}
|
||||||
|
|
||||||
public findLatestCreateForPath(
|
public findLatestCreateForPath(
|
||||||
|
|
|
||||||
|
|
@ -28,7 +28,7 @@ import type { SyncHistory } from "../tracing/sync-history";
|
||||||
import {
|
import {
|
||||||
SyncStatus,
|
SyncStatus,
|
||||||
SyncType,
|
SyncType,
|
||||||
type CommonHistoryEntry
|
type HistoryEntry
|
||||||
} from "../tracing/sync-history";
|
} from "../tracing/sync-history";
|
||||||
import { isBinary } from "../utils/is-binary";
|
import { isBinary } from "../utils/is-binary";
|
||||||
import { isFileTypeMergable } from "../utils/is-file-type-mergable";
|
import { isFileTypeMergable } from "../utils/is-file-type-mergable";
|
||||||
|
|
@ -72,6 +72,12 @@ export class Syncer {
|
||||||
this.webSocketManager.onRemoteVaultUpdateReceived.add(
|
this.webSocketManager.onRemoteVaultUpdateReceived.add(
|
||||||
this.syncRemotelyUpdatedFile.bind(this)
|
this.syncRemotelyUpdatedFile.bind(this)
|
||||||
);
|
);
|
||||||
|
// Funnel every queue mutation (enqueue, consume, clearPending) through
|
||||||
|
// the public count notifier so listeners see grow/shrink transitions
|
||||||
|
// immediately rather than only when a drain consumes an event.
|
||||||
|
this.queue.onPendingUpdateCountChanged.add(() => {
|
||||||
|
this.notifyRemainingOperationsChanged();
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
public syncLocallyCreatedFile(relativePath: RelativePath): void {
|
public syncLocallyCreatedFile(relativePath: RelativePath): void {
|
||||||
|
|
@ -152,9 +158,24 @@ export class Syncer {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* True while there is queued or in-flight work the syncer needs to handle:
|
||||||
|
* a running offline scan, an active drain, or pending events. Used by
|
||||||
|
* `SyncClient.waitUntilFinishedInternal` to detect WebSocket-fed work that
|
||||||
|
* landed in the queue after the syncer's first quiescence point.
|
||||||
|
*/
|
||||||
|
public get hasPendingWork(): boolean {
|
||||||
|
return (
|
||||||
|
this.runningScheduleSyncForOfflineChanges !== undefined ||
|
||||||
|
this.drainPromise !== undefined ||
|
||||||
|
this.queue.pendingUpdateCount > 0
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
public reset(): void {
|
public reset(): void {
|
||||||
this.queue.clearPending();
|
this.queue.clearPending();
|
||||||
this.clearOfflineScanGate();
|
this.clearOfflineScanGate();
|
||||||
|
this.previousRemainingOperationsCount = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -350,13 +371,19 @@ export class Syncer {
|
||||||
event.resolvers.reject(new Error("Create was cancelled"));
|
event.resolvers.reject(new Error("Create was cancelled"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Advance the cursor so the server doesn't replay this update on every
|
||||||
|
// reconnect — the skip is permanent for this version.
|
||||||
|
if (event.type === SyncEventType.RemoteChange) {
|
||||||
|
this.queue.lastSeenUpdateId = event.remoteVersion.vaultUpdateId;
|
||||||
|
}
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
private getHistoryEntryForSkippedOversizedFile(
|
private getHistoryEntryForSkippedOversizedFile(
|
||||||
sizeInBytes: number,
|
sizeInBytes: number,
|
||||||
relativePath: RelativePath
|
relativePath: RelativePath
|
||||||
): CommonHistoryEntry | undefined {
|
): HistoryEntry | undefined {
|
||||||
const sizeInMB = Math.round(sizeInBytes / 1024 / 1024);
|
const sizeInMB = Math.round(sizeInBytes / 1024 / 1024);
|
||||||
const { maxFileSizeMB } = this.settings.getSettings();
|
const { maxFileSizeMB } = this.settings.getSettings();
|
||||||
if (sizeInMB > maxFileSizeMB) {
|
if (sizeInMB > maxFileSizeMB) {
|
||||||
|
|
@ -366,7 +393,8 @@ export class Syncer {
|
||||||
type: SyncType.SKIPPED as const,
|
type: SyncType.SKIPPED as const,
|
||||||
relativePath
|
relativePath
|
||||||
},
|
},
|
||||||
message: `File size of ${sizeInMB} MB exceeds the maximum file size limit of ${maxFileSizeMB} MB`
|
message: `File size of ${sizeInMB} MB exceeds the maximum file size limit of ${maxFileSizeMB} MB`,
|
||||||
|
timestamp: new Date()
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -429,7 +457,6 @@ export class Syncer {
|
||||||
// and history entry. Keeping the entry in the map until then lets
|
// and history entry. Keeping the entry in the map until then lets
|
||||||
// late remote updates be recognised as "file is missing" and
|
// late remote updates be recognised as "file is missing" and
|
||||||
// skipped, instead of resurrecting the doc.
|
// skipped, instead of resurrecting the doc.
|
||||||
//
|
|
||||||
this.history.addHistoryEntry({
|
this.history.addHistoryEntry({
|
||||||
status: SyncStatus.SUCCESS,
|
status: SyncStatus.SUCCESS,
|
||||||
details: {
|
details: {
|
||||||
|
|
@ -437,7 +464,8 @@ export class Syncer {
|
||||||
relativePath: doc.path
|
relativePath: doc.path
|
||||||
},
|
},
|
||||||
message: "Successfully deleted file on the server",
|
message: "Successfully deleted file on the server",
|
||||||
author: response.userId
|
author: response.userId,
|
||||||
|
timestamp: new Date(response.updatedDate)
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -482,8 +510,6 @@ export class Syncer {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
this.queue.lastSeenUpdateId = response.vaultUpdateId;
|
|
||||||
|
|
||||||
await this.handleMaybeMergingResponse({
|
await this.handleMaybeMergingResponse({
|
||||||
path: diskPath,
|
path: diskPath,
|
||||||
response,
|
response,
|
||||||
|
|
|
||||||
|
|
@ -40,12 +40,15 @@ export type SyncDetails =
|
||||||
| SyncMovedDetails
|
| SyncMovedDetails
|
||||||
| SyncSkippedDetails;
|
| SyncSkippedDetails;
|
||||||
|
|
||||||
export interface CommonHistoryEntry {
|
export interface HistoryEntry {
|
||||||
status: SyncStatus;
|
status: SyncStatus;
|
||||||
message: string;
|
message: string;
|
||||||
details: SyncDetails;
|
details: SyncDetails;
|
||||||
|
timestamp: Date;
|
||||||
|
// `author` is the server-side user id and only exists for entries that
|
||||||
|
// round-tripped through the server. Local-only entries (e.g. SKIPPED)
|
||||||
|
// legitimately have no author.
|
||||||
author?: string;
|
author?: string;
|
||||||
timestamp?: Date;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export enum SyncType {
|
export enum SyncType {
|
||||||
|
|
@ -62,7 +65,6 @@ export enum SyncStatus {
|
||||||
SKIPPED = "SKIPPED"
|
SKIPPED = "SKIPPED"
|
||||||
}
|
}
|
||||||
|
|
||||||
export type HistoryEntry = CommonHistoryEntry & { timestamp: Date };
|
|
||||||
|
|
||||||
export interface HistoryStats {
|
export interface HistoryStats {
|
||||||
success: number;
|
success: number;
|
||||||
|
|
@ -81,7 +83,7 @@ export class SyncHistory {
|
||||||
error: 0
|
error: 0
|
||||||
};
|
};
|
||||||
|
|
||||||
public constructor(private readonly logger: Logger) {}
|
public constructor(private readonly logger: Logger) { }
|
||||||
|
|
||||||
public get entries(): readonly HistoryEntry[] {
|
public get entries(): readonly HistoryEntry[] {
|
||||||
return this._entries;
|
return this._entries;
|
||||||
|
|
@ -93,25 +95,20 @@ export class SyncHistory {
|
||||||
*
|
*
|
||||||
* If the entry list is too long, the oldest entry will be removed.
|
* If the entry list is too long, the oldest entry will be removed.
|
||||||
*/
|
*/
|
||||||
public addHistoryEntry(entry: CommonHistoryEntry): void {
|
public addHistoryEntry(entry: HistoryEntry): void {
|
||||||
const historyEntry = {
|
const candidate = this.findSimilarRecentUpdateEntry(entry);
|
||||||
...entry,
|
|
||||||
timestamp: entry.timestamp ?? new Date()
|
|
||||||
};
|
|
||||||
|
|
||||||
const candidate = this.findSimilarRecentUpdateEntry(historyEntry);
|
|
||||||
if (candidate !== undefined) {
|
if (candidate !== undefined) {
|
||||||
removeFromArray(this._entries, candidate);
|
removeFromArray(this._entries, candidate);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Insert the entry at the beginning
|
// Insert the entry at the beginning
|
||||||
this._entries.unshift(historyEntry);
|
this._entries.unshift(entry);
|
||||||
|
|
||||||
if (this._entries.length > MAX_HISTORY_ENTRY_COUNT) {
|
if (this._entries.length > MAX_HISTORY_ENTRY_COUNT) {
|
||||||
this._entries.pop();
|
this._entries.pop();
|
||||||
}
|
}
|
||||||
|
|
||||||
this.updateSuccessCount(historyEntry);
|
this.updateSuccessCount(entry);
|
||||||
}
|
}
|
||||||
|
|
||||||
public reset(): void {
|
public reset(): void {
|
||||||
|
|
@ -139,8 +136,8 @@ export class SyncHistory {
|
||||||
candidate !== undefined &&
|
candidate !== undefined &&
|
||||||
(this._entries[0] === candidate ||
|
(this._entries[0] === candidate ||
|
||||||
candidate.timestamp.getTime() +
|
candidate.timestamp.getTime() +
|
||||||
TIMEOUT_FOR_MERGING_HISTORY_ENTRIES_IN_SECONDS * 1000 >
|
TIMEOUT_FOR_MERGING_HISTORY_ENTRIES_IN_SECONDS * 1000 >
|
||||||
entry.timestamp.getTime())
|
entry.timestamp.getTime())
|
||||||
) {
|
) {
|
||||||
return candidate;
|
return candidate;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -261,11 +261,11 @@ export class MockAgent extends MockClient {
|
||||||
);
|
);
|
||||||
otherAgent.client.logger.info(
|
otherAgent.client.logger.info(
|
||||||
"Other agent's data: " +
|
"Other agent's data: " +
|
||||||
JSON.stringify(otherAgent.data, null, 2)
|
JSON.stringify(otherAgent.data, null, 2)
|
||||||
);
|
);
|
||||||
otherAgent.client.logger.info(
|
otherAgent.client.logger.info(
|
||||||
"Other agent's files: " +
|
"Other agent's files: " +
|
||||||
Array.from(otherAgent.files.keys()).join(", ")
|
Array.from(otherAgent.files.keys()).join(", ")
|
||||||
);
|
);
|
||||||
|
|
||||||
throw e;
|
throw e;
|
||||||
|
|
@ -339,12 +339,7 @@ export class MockAgent extends MockClient {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!this.useSlowFileEvents && !this.doDeletes) {
|
// can't assert(found.length >= 1, ...); because binary files have LWW semantics
|
||||||
assert(
|
|
||||||
found.length >= 1,
|
|
||||||
`[${this.name}] Binary content ${content} not found in any files`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -531,9 +526,9 @@ export class MockAgent extends MockClient {
|
||||||
|
|
||||||
private removeBinaryUuid(file: string): void {
|
private removeBinaryUuid(file: string): void {
|
||||||
const existing = this.files.get(file);
|
const existing = this.files.get(file);
|
||||||
if (existing === undefined) {return;}
|
if (existing === undefined) { return; }
|
||||||
const content = new TextDecoder().decode(existing);
|
const content = new TextDecoder().decode(existing);
|
||||||
if (!content.startsWith("BINARY:")) {return;}
|
if (!content.startsWith("BINARY:")) { return; }
|
||||||
const uuid = content.slice("BINARY:".length);
|
const uuid = content.slice("BINARY:".length);
|
||||||
utils.removeFromArray(this.writtenBinaryContents, uuid);
|
utils.removeFromArray(this.writtenBinaryContents, uuid);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue