Fix syncing when network latency is present (#4)
* WIP * Add debug * Dedupe inserts * Add deterministic ordering * Fix whitespaces * Update insta * Add integration test script * Rename * Add test * Working for non-deletes * omg it mostly works for deletes * Isdeleted fix * remove created dates * update api * Take document id * No max attempt * works * Use string uuids * . * working!!!! (hopefully) * Improve bundling * Add module * lint * . * lint * Fix CI * use toolchain * clean up * Add useSlowFileEvents * Delete fuzz * Fix CI * use docker * fix script * clean up * Clean up * change node version * Build docker image on every commit * fix ci * 1 db per vault * Add scritps folder * Bump versions * Lint * . * Fix tests for real * Style * . * try * Consistent ordering * Fix tests * hmm * . * Clean up diff * Fixes * . * Fix version bump * . * . * .
This commit is contained in:
GitHub
parent
bcf48c428d
commit
8b8f1d91d9
91 changed files with 2252 additions and 1586 deletions
|
|
@ -1,10 +0,0 @@
|
|||
{
|
||||
"id": "vault-link",
|
||||
"name": "VaultLink",
|
||||
"version": "0.0.30",
|
||||
"minAppVersion": "0.0.0",
|
||||
"description": "Self-hosted synchronization and collaboration for your Vault.",
|
||||
"author": "Andras Schmelczer",
|
||||
"authorUrl": "https://schmelczer.dev",
|
||||
"isDesktopOnly": false
|
||||
}
|
||||
|
|
@ -14,7 +14,7 @@
|
|||
"license": "MIT",
|
||||
"devDependencies": {
|
||||
"@types/jest": "^29.5.14",
|
||||
"@types/node": "^22.13.5",
|
||||
"@types/node": "^22.13.10",
|
||||
"css-loader": "^7.1.2",
|
||||
"date-fns": "^4.1.0",
|
||||
"file-loader": "^6.2.0",
|
||||
|
|
@ -23,14 +23,14 @@
|
|||
"mini-css-extract-plugin": "^2.9.2",
|
||||
"obsidian": "1.8.7",
|
||||
"resolve-url-loader": "^5.0.0",
|
||||
"sass": "^1.85.0",
|
||||
"sass": "^1.85.1",
|
||||
"sass-loader": "^16.0.5",
|
||||
"sync-client": "file:../sync-client",
|
||||
"terser-webpack-plugin": "^5.3.11",
|
||||
"terser-webpack-plugin": "^5.3.14",
|
||||
"ts-jest": "^29.2.6",
|
||||
"ts-loader": "^9.5.2",
|
||||
"tslib": "2.8.1",
|
||||
"typescript": "5.7.3",
|
||||
"typescript": "5.8.2",
|
||||
"url": "^0.11.4",
|
||||
"virtual-scroller": "^1.13.1",
|
||||
"webpack": "^5.98.0",
|
||||
|
|
|
|||
|
|
@ -9,10 +9,7 @@ export class ObsidianFileEventHandler {
|
|||
if (file instanceof TFile) {
|
||||
this.client.logger.info(`File created: ${file.path}`);
|
||||
|
||||
await this.client.syncer.syncLocallyCreatedFile(
|
||||
file.path,
|
||||
new Date(file.stat.ctime)
|
||||
);
|
||||
await this.client.syncer.syncLocallyCreatedFile(file.path);
|
||||
} else {
|
||||
this.client.logger.debug(`Folder created: ${file.path}, ignored`);
|
||||
}
|
||||
|
|
@ -34,8 +31,7 @@ export class ObsidianFileEventHandler {
|
|||
|
||||
await this.client.syncer.syncLocallyUpdatedFile({
|
||||
oldPath,
|
||||
relativePath: file.path,
|
||||
updateTime: new Date(file.stat.ctime)
|
||||
relativePath: file.path
|
||||
});
|
||||
} else {
|
||||
this.client.logger.debug(
|
||||
|
|
@ -53,8 +49,7 @@ export class ObsidianFileEventHandler {
|
|||
this.client.logger.info(`File modified: ${file.path}`);
|
||||
|
||||
await this.client.syncer.syncLocallyUpdatedFile({
|
||||
relativePath: file.path,
|
||||
updateTime: new Date(file.stat.ctime)
|
||||
relativePath: file.path
|
||||
});
|
||||
} else {
|
||||
this.client.logger.debug(`Folder modified: ${file.path}, ignored`);
|
||||
|
|
|
|||
|
|
@ -60,7 +60,6 @@ export class HistoryView extends ItemView {
|
|||
}
|
||||
|
||||
element.createEl("span", {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
|
||||
text: entry.relativePath
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import type { WorkspaceLeaf } from "obsidian";
|
||||
import { ItemView } from "obsidian";
|
||||
import type VaultLinkPlugin from "src/vault-link-plugin";
|
||||
import type VaultLinkPlugin from "../vault-link-plugin";
|
||||
import type { SyncClient } from "sync-client";
|
||||
|
||||
export class LogsView extends ItemView {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import type { App } from "obsidian";
|
||||
import { Notice, PluginSettingTab, Setting } from "obsidian";
|
||||
|
||||
import type VaultLinkPlugin from "src/vault-link-plugin";
|
||||
import type VaultLinkPlugin from "../vault-link-plugin";
|
||||
import type { StatusDescription } from "./status-description";
|
||||
import { LogsView } from "./logs-view";
|
||||
import { HistoryView } from "./history-view";
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import type { HistoryStats, SyncClient } from "sync-client";
|
||||
import type VaultLinkPlugin from "src/vault-link-plugin";
|
||||
import type VaultLinkPlugin from "../vault-link-plugin";
|
||||
|
||||
export class StatusBar {
|
||||
private readonly statusBarItem: HTMLElement;
|
||||
|
|
|
|||
|
|
@ -94,9 +94,6 @@ module.exports = (env, argv) => ({
|
|||
alias: {
|
||||
root: __dirname,
|
||||
src: path.resolve(__dirname, "src")
|
||||
},
|
||||
fallback: {
|
||||
url: require.resolve("url")
|
||||
}
|
||||
},
|
||||
output: {
|
||||
|
|
|
|||
272
frontend/package-lock.json
generated
272
frontend/package-lock.json
generated
|
|
@ -12,11 +12,11 @@
|
|||
],
|
||||
"devDependencies": {
|
||||
"concurrently": "^9.1.2",
|
||||
"eslint": "9.21.0",
|
||||
"eslint": "9.22.0",
|
||||
"eslint-plugin-unused-imports": "^4.1.4",
|
||||
"npm-check-updates": "^17.1.14",
|
||||
"prettier": "^3.5.2",
|
||||
"typescript-eslint": "8.24.1"
|
||||
"npm-check-updates": "^17.1.15",
|
||||
"prettier": "^3.5.3",
|
||||
"typescript-eslint": "8.26.1"
|
||||
}
|
||||
},
|
||||
"../backend/sync_lib/pkg": {
|
||||
|
|
@ -39,7 +39,6 @@
|
|||
},
|
||||
"node_modules/@babel/code-frame": {
|
||||
"version": "7.26.2",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@babel/helper-validator-identifier": "^7.25.9",
|
||||
|
|
@ -179,7 +178,6 @@
|
|||
},
|
||||
"node_modules/@babel/helper-validator-identifier": {
|
||||
"version": "7.25.9",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6.9.0"
|
||||
|
|
@ -194,12 +192,14 @@
|
|||
}
|
||||
},
|
||||
"node_modules/@babel/helpers": {
|
||||
"version": "7.26.9",
|
||||
"version": "7.26.10",
|
||||
"resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.26.10.tgz",
|
||||
"integrity": "sha512-UPYc3SauzZ3JGgj87GgZ89JVdC5dj0AoetR5Bw6wj4niittNyFh6+eOGonYvJ1ao6B8lEa3Q3klS7ADZ53bc5g==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@babel/template": "^7.26.9",
|
||||
"@babel/types": "^7.26.9"
|
||||
"@babel/types": "^7.26.10"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6.9.0"
|
||||
|
|
@ -463,7 +463,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/@babel/types": {
|
||||
"version": "7.26.9",
|
||||
"version": "7.26.10",
|
||||
"resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.10.tgz",
|
||||
"integrity": "sha512-emqcG3vHrpxUKTrxcblR36dcrcoRDvKmnL/dCL6ZsHaShW80qxCAcNhzQZrpeM765VzEos+xOi4s+r4IXzTwdQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
|
|
@ -580,6 +582,16 @@
|
|||
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@eslint/config-helpers": {
|
||||
"version": "0.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.1.0.tgz",
|
||||
"integrity": "sha512-kLrdPDJE1ckPo94kmPPf9Hfd0DU0Jw6oKYrhe+pwSC0iTUInmTa+w6fw8sGgcfkFJGNdWOUeOaDM4quW4a7OkA==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@eslint/core": {
|
||||
"version": "0.12.0",
|
||||
"resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.12.0.tgz",
|
||||
|
|
@ -618,9 +630,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/@eslint/js": {
|
||||
"version": "9.21.0",
|
||||
"resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.21.0.tgz",
|
||||
"integrity": "sha512-BqStZ3HX8Yz6LvsF5ByXYrtigrV5AXADWLAGc7PH/1SxOb7/FIYYMszZZWiUou/GB9P2lXWk2SV4d+Z8h0nknw==",
|
||||
"version": "9.22.0",
|
||||
"resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.22.0.tgz",
|
||||
"integrity": "sha512-vLFajx9o8d1/oL2ZkpMYbkLv8nDB6yaIwFNt7nI4+I80U/z03SxmfOMsLbvWr3p7C+Wnoh//aOu2pQW8cS0HCQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
|
|
@ -1130,6 +1142,8 @@
|
|||
},
|
||||
"node_modules/@nodelib/fs.scandir": {
|
||||
"version": "2.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
|
||||
"integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
|
|
@ -1142,6 +1156,8 @@
|
|||
},
|
||||
"node_modules/@nodelib/fs.stat": {
|
||||
"version": "2.0.5",
|
||||
"resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz",
|
||||
"integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
|
|
@ -1150,6 +1166,8 @@
|
|||
},
|
||||
"node_modules/@nodelib/fs.walk": {
|
||||
"version": "1.2.8",
|
||||
"resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz",
|
||||
"integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
|
|
@ -1235,7 +1253,6 @@
|
|||
},
|
||||
"node_modules/@redocly/ajv": {
|
||||
"version": "8.11.2",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"fast-deep-equal": "^3.1.1",
|
||||
|
|
@ -1250,17 +1267,14 @@
|
|||
},
|
||||
"node_modules/@redocly/ajv/node_modules/json-schema-traverse": {
|
||||
"version": "1.0.0",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@redocly/config": {
|
||||
"version": "0.20.3",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@redocly/openapi-core": {
|
||||
"version": "1.29.0",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@redocly/ajv": "^8.11.2",
|
||||
|
|
@ -1280,7 +1294,6 @@
|
|||
},
|
||||
"node_modules/@redocly/openapi-core/node_modules/brace-expansion": {
|
||||
"version": "2.0.1",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"balanced-match": "^1.0.0"
|
||||
|
|
@ -1288,7 +1301,6 @@
|
|||
},
|
||||
"node_modules/@redocly/openapi-core/node_modules/minimatch": {
|
||||
"version": "5.1.6",
|
||||
"dev": true,
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"brace-expansion": "^2.0.1"
|
||||
|
|
@ -1458,9 +1470,9 @@
|
|||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/node": {
|
||||
"version": "22.13.5",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.5.tgz",
|
||||
"integrity": "sha512-+lTU0PxZXn0Dr1NBtC7Y8cR21AJr87dLLU953CWA6pMxxv/UDc7jYAY90upcrie1nRcD6XNG5HOYEDtgW5TxAg==",
|
||||
"version": "22.13.10",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.10.tgz",
|
||||
"integrity": "sha512-I6LPUvlRH+O6VRUqYOcMudhaIdUVWfsjnZavnsraHvpBwaEyMN29ry+0UVJhImYL16xsscu0aske3yA+uPOWfw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
|
|
@ -1494,15 +1506,17 @@
|
|||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@typescript-eslint/eslint-plugin": {
|
||||
"version": "8.24.1",
|
||||
"version": "8.26.1",
|
||||
"resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.26.1.tgz",
|
||||
"integrity": "sha512-2X3mwqsj9Bd3Ciz508ZUtoQQYpOhU/kWoUqIf49H8Z0+Vbh6UF/y0OEYp0Q0axOGzaBGs7QxRwq0knSQ8khQNA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@eslint-community/regexpp": "^4.10.0",
|
||||
"@typescript-eslint/scope-manager": "8.24.1",
|
||||
"@typescript-eslint/type-utils": "8.24.1",
|
||||
"@typescript-eslint/utils": "8.24.1",
|
||||
"@typescript-eslint/visitor-keys": "8.24.1",
|
||||
"@typescript-eslint/scope-manager": "8.26.1",
|
||||
"@typescript-eslint/type-utils": "8.26.1",
|
||||
"@typescript-eslint/utils": "8.26.1",
|
||||
"@typescript-eslint/visitor-keys": "8.26.1",
|
||||
"graphemer": "^1.4.0",
|
||||
"ignore": "^5.3.1",
|
||||
"natural-compare": "^1.4.0",
|
||||
|
|
@ -1518,18 +1532,20 @@
|
|||
"peerDependencies": {
|
||||
"@typescript-eslint/parser": "^8.0.0 || ^8.0.0-alpha.0",
|
||||
"eslint": "^8.57.0 || ^9.0.0",
|
||||
"typescript": ">=4.8.4 <5.8.0"
|
||||
"typescript": ">=4.8.4 <5.9.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@typescript-eslint/parser": {
|
||||
"version": "8.24.1",
|
||||
"version": "8.26.1",
|
||||
"resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.26.1.tgz",
|
||||
"integrity": "sha512-w6HZUV4NWxqd8BdeFf81t07d7/YV9s7TCWrQQbG5uhuvGUAW+fq1usZ1Hmz9UPNLniFnD8GLSsDpjP0hm1S4lQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@typescript-eslint/scope-manager": "8.24.1",
|
||||
"@typescript-eslint/types": "8.24.1",
|
||||
"@typescript-eslint/typescript-estree": "8.24.1",
|
||||
"@typescript-eslint/visitor-keys": "8.24.1",
|
||||
"@typescript-eslint/scope-manager": "8.26.1",
|
||||
"@typescript-eslint/types": "8.26.1",
|
||||
"@typescript-eslint/typescript-estree": "8.26.1",
|
||||
"@typescript-eslint/visitor-keys": "8.26.1",
|
||||
"debug": "^4.3.4"
|
||||
},
|
||||
"engines": {
|
||||
|
|
@ -1541,16 +1557,18 @@
|
|||
},
|
||||
"peerDependencies": {
|
||||
"eslint": "^8.57.0 || ^9.0.0",
|
||||
"typescript": ">=4.8.4 <5.8.0"
|
||||
"typescript": ">=4.8.4 <5.9.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@typescript-eslint/scope-manager": {
|
||||
"version": "8.24.1",
|
||||
"version": "8.26.1",
|
||||
"resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.26.1.tgz",
|
||||
"integrity": "sha512-6EIvbE5cNER8sqBu6V7+KeMZIC1664d2Yjt+B9EWUXrsyWpxx4lEZrmvxgSKRC6gX+efDL/UY9OpPZ267io3mg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@typescript-eslint/types": "8.24.1",
|
||||
"@typescript-eslint/visitor-keys": "8.24.1"
|
||||
"@typescript-eslint/types": "8.26.1",
|
||||
"@typescript-eslint/visitor-keys": "8.26.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
|
||||
|
|
@ -1561,12 +1579,14 @@
|
|||
}
|
||||
},
|
||||
"node_modules/@typescript-eslint/type-utils": {
|
||||
"version": "8.24.1",
|
||||
"version": "8.26.1",
|
||||
"resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.26.1.tgz",
|
||||
"integrity": "sha512-Kcj/TagJLwoY/5w9JGEFV0dclQdyqw9+VMndxOJKtoFSjfZhLXhYjzsQEeyza03rwHx2vFEGvrJWJBXKleRvZg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@typescript-eslint/typescript-estree": "8.24.1",
|
||||
"@typescript-eslint/utils": "8.24.1",
|
||||
"@typescript-eslint/typescript-estree": "8.26.1",
|
||||
"@typescript-eslint/utils": "8.26.1",
|
||||
"debug": "^4.3.4",
|
||||
"ts-api-utils": "^2.0.1"
|
||||
},
|
||||
|
|
@ -1579,11 +1599,13 @@
|
|||
},
|
||||
"peerDependencies": {
|
||||
"eslint": "^8.57.0 || ^9.0.0",
|
||||
"typescript": ">=4.8.4 <5.8.0"
|
||||
"typescript": ">=4.8.4 <5.9.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@typescript-eslint/types": {
|
||||
"version": "8.24.1",
|
||||
"version": "8.26.1",
|
||||
"resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.26.1.tgz",
|
||||
"integrity": "sha512-n4THUQW27VmQMx+3P+B0Yptl7ydfceUj4ON/AQILAASwgYdZ/2dhfymRMh5egRUrvK5lSmaOm77Ry+lmXPOgBQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
|
|
@ -1595,12 +1617,14 @@
|
|||
}
|
||||
},
|
||||
"node_modules/@typescript-eslint/typescript-estree": {
|
||||
"version": "8.24.1",
|
||||
"version": "8.26.1",
|
||||
"resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.26.1.tgz",
|
||||
"integrity": "sha512-yUwPpUHDgdrv1QJ7YQal3cMVBGWfnuCdKbXw1yyjArax3353rEJP1ZA+4F8nOlQ3RfS2hUN/wze3nlY+ZOhvoA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@typescript-eslint/types": "8.24.1",
|
||||
"@typescript-eslint/visitor-keys": "8.24.1",
|
||||
"@typescript-eslint/types": "8.26.1",
|
||||
"@typescript-eslint/visitor-keys": "8.26.1",
|
||||
"debug": "^4.3.4",
|
||||
"fast-glob": "^3.3.2",
|
||||
"is-glob": "^4.0.3",
|
||||
|
|
@ -1616,11 +1640,13 @@
|
|||
"url": "https://opencollective.com/typescript-eslint"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"typescript": ">=4.8.4 <5.8.0"
|
||||
"typescript": ">=4.8.4 <5.9.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
|
||||
"integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
|
|
@ -1629,6 +1655,8 @@
|
|||
},
|
||||
"node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": {
|
||||
"version": "9.0.5",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz",
|
||||
"integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==",
|
||||
"dev": true,
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
|
|
@ -1642,14 +1670,16 @@
|
|||
}
|
||||
},
|
||||
"node_modules/@typescript-eslint/utils": {
|
||||
"version": "8.24.1",
|
||||
"version": "8.26.1",
|
||||
"resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.26.1.tgz",
|
||||
"integrity": "sha512-V4Urxa/XtSUroUrnI7q6yUTD3hDtfJ2jzVfeT3VK0ciizfK2q/zGC0iDh1lFMUZR8cImRrep6/q0xd/1ZGPQpg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@eslint-community/eslint-utils": "^4.4.0",
|
||||
"@typescript-eslint/scope-manager": "8.24.1",
|
||||
"@typescript-eslint/types": "8.24.1",
|
||||
"@typescript-eslint/typescript-estree": "8.24.1"
|
||||
"@typescript-eslint/scope-manager": "8.26.1",
|
||||
"@typescript-eslint/types": "8.26.1",
|
||||
"@typescript-eslint/typescript-estree": "8.26.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
|
||||
|
|
@ -1660,15 +1690,17 @@
|
|||
},
|
||||
"peerDependencies": {
|
||||
"eslint": "^8.57.0 || ^9.0.0",
|
||||
"typescript": ">=4.8.4 <5.8.0"
|
||||
"typescript": ">=4.8.4 <5.9.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@typescript-eslint/visitor-keys": {
|
||||
"version": "8.24.1",
|
||||
"version": "8.26.1",
|
||||
"resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.26.1.tgz",
|
||||
"integrity": "sha512-AjOC3zfnxd6S4Eiy3jwktJPclqhFHNyd8L6Gycf9WUPoKZpgM5PjkxY1X7uSy61xVpiJDhhk7XT2NVsN3ALTWg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@typescript-eslint/types": "8.24.1",
|
||||
"@typescript-eslint/types": "8.26.1",
|
||||
"eslint-visitor-keys": "^4.2.0"
|
||||
},
|
||||
"engines": {
|
||||
|
|
@ -1909,7 +1941,6 @@
|
|||
},
|
||||
"node_modules/agent-base": {
|
||||
"version": "7.1.3",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 14"
|
||||
|
|
@ -1976,7 +2007,6 @@
|
|||
},
|
||||
"node_modules/ansi-colors": {
|
||||
"version": "4.1.3",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
|
|
@ -2039,7 +2069,6 @@
|
|||
},
|
||||
"node_modules/argparse": {
|
||||
"version": "2.0.1",
|
||||
"dev": true,
|
||||
"license": "Python-2.0"
|
||||
},
|
||||
"node_modules/async": {
|
||||
|
|
@ -2161,7 +2190,6 @@
|
|||
},
|
||||
"node_modules/balanced-match": {
|
||||
"version": "1.0.2",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/big.js": {
|
||||
|
|
@ -2249,7 +2277,6 @@
|
|||
},
|
||||
"node_modules/byte-base64": {
|
||||
"version": "1.1.0",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/call-bind-apply-helpers": {
|
||||
|
|
@ -2335,7 +2362,6 @@
|
|||
},
|
||||
"node_modules/change-case": {
|
||||
"version": "5.4.4",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/char-regex": {
|
||||
|
|
@ -2445,7 +2471,6 @@
|
|||
},
|
||||
"node_modules/colorette": {
|
||||
"version": "1.4.0",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/commander": {
|
||||
|
|
@ -2597,7 +2622,6 @@
|
|||
},
|
||||
"node_modules/debug": {
|
||||
"version": "4.4.0",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"ms": "^2.1.3"
|
||||
|
|
@ -2822,18 +2846,19 @@
|
|||
}
|
||||
},
|
||||
"node_modules/eslint": {
|
||||
"version": "9.21.0",
|
||||
"resolved": "https://registry.npmjs.org/eslint/-/eslint-9.21.0.tgz",
|
||||
"integrity": "sha512-KjeihdFqTPhOMXTt7StsDxriV4n66ueuF/jfPNC3j/lduHwr/ijDwJMsF+wyMJethgiKi5wniIE243vi07d3pg==",
|
||||
"version": "9.22.0",
|
||||
"resolved": "https://registry.npmjs.org/eslint/-/eslint-9.22.0.tgz",
|
||||
"integrity": "sha512-9V/QURhsRN40xuHXWjV64yvrzMjcz7ZyNoF2jJFmy9j/SLk0u1OLSZgXi28MrXjymnjEGSR80WCdab3RGMDveQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@eslint-community/eslint-utils": "^4.2.0",
|
||||
"@eslint-community/regexpp": "^4.12.1",
|
||||
"@eslint/config-array": "^0.19.2",
|
||||
"@eslint/config-helpers": "^0.1.0",
|
||||
"@eslint/core": "^0.12.0",
|
||||
"@eslint/eslintrc": "^3.3.0",
|
||||
"@eslint/js": "9.21.0",
|
||||
"@eslint/js": "9.22.0",
|
||||
"@eslint/plugin-kit": "^0.2.7",
|
||||
"@humanfs/node": "^0.16.6",
|
||||
"@humanwhocodes/module-importer": "^1.0.1",
|
||||
|
|
@ -2845,7 +2870,7 @@
|
|||
"cross-spawn": "^7.0.6",
|
||||
"debug": "^4.3.2",
|
||||
"escape-string-regexp": "^4.0.0",
|
||||
"eslint-scope": "^8.2.0",
|
||||
"eslint-scope": "^8.3.0",
|
||||
"eslint-visitor-keys": "^4.2.0",
|
||||
"espree": "^10.3.0",
|
||||
"esquery": "^1.5.0",
|
||||
|
|
@ -2896,7 +2921,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/eslint-scope": {
|
||||
"version": "8.2.0",
|
||||
"version": "8.3.0",
|
||||
"resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.3.0.tgz",
|
||||
"integrity": "sha512-pUNxi75F8MJ/GdeKtVLSbYg4ZI34J6C0C7sbL4YOp2exGwen7ZsuBqKzUhXd0qMQ362yET3z+uPwKeg/0C2XCQ==",
|
||||
"dev": true,
|
||||
"license": "BSD-2-Clause",
|
||||
"dependencies": {
|
||||
|
|
@ -2991,7 +3018,6 @@
|
|||
},
|
||||
"node_modules/eventemitter3": {
|
||||
"version": "5.0.1",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/events": {
|
||||
|
|
@ -3048,11 +3074,12 @@
|
|||
},
|
||||
"node_modules/fast-deep-equal": {
|
||||
"version": "3.1.3",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/fast-glob": {
|
||||
"version": "3.3.3",
|
||||
"resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz",
|
||||
"integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
|
|
@ -3068,6 +3095,8 @@
|
|||
},
|
||||
"node_modules/fast-glob/node_modules/glob-parent": {
|
||||
"version": "5.1.2",
|
||||
"resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
|
||||
"integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==",
|
||||
"dev": true,
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
|
|
@ -3111,7 +3140,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/fastq": {
|
||||
"version": "1.19.0",
|
||||
"version": "1.19.1",
|
||||
"resolved": "https://registry.npmjs.org/fastq/-/fastq-1.19.1.tgz",
|
||||
"integrity": "sha512-GwLTyxkCXjXbxqIhTsMI2Nui8huMPtnxg7krajPJAjnEG/iiOS7i+zCtWGZR9G0NBKbXKh6X9m9UIsYX/N6vvQ==",
|
||||
"dev": true,
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
|
|
@ -3128,7 +3159,6 @@
|
|||
},
|
||||
"node_modules/fetch-retry": {
|
||||
"version": "6.0.0",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/file-entry-cache": {
|
||||
|
|
@ -3449,7 +3479,6 @@
|
|||
},
|
||||
"node_modules/https-proxy-agent": {
|
||||
"version": "7.0.6",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"agent-base": "^7.1.2",
|
||||
|
|
@ -3536,7 +3565,6 @@
|
|||
},
|
||||
"node_modules/index-to-position": {
|
||||
"version": "0.1.2",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
|
|
@ -4271,7 +4299,6 @@
|
|||
},
|
||||
"node_modules/js-levenshtein": {
|
||||
"version": "1.1.6",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
|
|
@ -4279,12 +4306,10 @@
|
|||
},
|
||||
"node_modules/js-tokens": {
|
||||
"version": "4.0.0",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/js-yaml": {
|
||||
"version": "4.1.0",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"argparse": "^2.0.1"
|
||||
|
|
@ -4497,6 +4522,8 @@
|
|||
},
|
||||
"node_modules/merge2": {
|
||||
"version": "1.4.1",
|
||||
"resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz",
|
||||
"integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
|
|
@ -4631,7 +4658,6 @@
|
|||
},
|
||||
"node_modules/ms": {
|
||||
"version": "2.1.3",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/nanoid": {
|
||||
|
|
@ -4686,7 +4712,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/npm-check-updates": {
|
||||
"version": "17.1.14",
|
||||
"version": "17.1.15",
|
||||
"resolved": "https://registry.npmjs.org/npm-check-updates/-/npm-check-updates-17.1.15.tgz",
|
||||
"integrity": "sha512-miATvKu5rjec/1wxc5TGDjpsucgtCHwRVZorZpDkS6NzdWXfnUWlN4abZddWb7XSijAuBNzzYglIdTm9SbgMVg==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"bin": {
|
||||
|
|
@ -4758,8 +4786,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/openapi-fetch": {
|
||||
"version": "0.13.4",
|
||||
"dev": true,
|
||||
"version": "0.13.5",
|
||||
"resolved": "https://registry.npmjs.org/openapi-fetch/-/openapi-fetch-0.13.5.tgz",
|
||||
"integrity": "sha512-AQK8T9GSKFREFlN1DBXTYsLjs7YV2tZcJ7zUWxbjMoQmj8dDSFRrzhLCbHPZWA1TMV3vACqfCxLEZcwf2wxV6Q==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"openapi-typescript-helpers": "^0.0.15"
|
||||
|
|
@ -4767,7 +4796,6 @@
|
|||
},
|
||||
"node_modules/openapi-typescript": {
|
||||
"version": "7.6.1",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@redocly/openapi-core": "^1.28.0",
|
||||
|
|
@ -4786,12 +4814,10 @@
|
|||
},
|
||||
"node_modules/openapi-typescript-helpers": {
|
||||
"version": "0.0.15",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/openapi-typescript/node_modules/parse-json": {
|
||||
"version": "8.1.0",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@babel/code-frame": "^7.22.13",
|
||||
|
|
@ -4807,7 +4833,6 @@
|
|||
},
|
||||
"node_modules/openapi-typescript/node_modules/supports-color": {
|
||||
"version": "9.4.0",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
|
|
@ -4818,7 +4843,6 @@
|
|||
},
|
||||
"node_modules/openapi-typescript/node_modules/type-fest": {
|
||||
"version": "4.35.0",
|
||||
"dev": true,
|
||||
"license": "(MIT OR CC0-1.0)",
|
||||
"engines": {
|
||||
"node": ">=16"
|
||||
|
|
@ -4873,7 +4897,6 @@
|
|||
},
|
||||
"node_modules/p-queue": {
|
||||
"version": "8.1.0",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"eventemitter3": "^5.0.1",
|
||||
|
|
@ -4888,7 +4911,6 @@
|
|||
},
|
||||
"node_modules/p-timeout": {
|
||||
"version": "6.1.4",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=14.16"
|
||||
|
|
@ -4966,7 +4988,6 @@
|
|||
},
|
||||
"node_modules/picocolors": {
|
||||
"version": "1.1.1",
|
||||
"dev": true,
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/picomatch": {
|
||||
|
|
@ -5049,7 +5070,6 @@
|
|||
},
|
||||
"node_modules/pluralize": {
|
||||
"version": "8.0.0",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
|
|
@ -5163,9 +5183,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/prettier": {
|
||||
"version": "3.5.2",
|
||||
"resolved": "https://registry.npmjs.org/prettier/-/prettier-3.5.2.tgz",
|
||||
"integrity": "sha512-lc6npv5PH7hVqozBR7lkBNOGXV9vMwROAPlumdBkX0wTbbzPu/U1hk5yL8p2pt4Xoc+2mkT8t/sow2YrV/M5qg==",
|
||||
"version": "3.5.3",
|
||||
"resolved": "https://registry.npmjs.org/prettier/-/prettier-3.5.3.tgz",
|
||||
"integrity": "sha512-QQtaxnoDJeAkDvDKWCLiwIXkTgRhwYDEQCghU9Z6q03iyek/rxRh/2lC3HB7P8sWT2xC/y5JDctPLBIGzHKbhw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
|
|
@ -5255,6 +5275,8 @@
|
|||
},
|
||||
"node_modules/queue-microtask": {
|
||||
"version": "1.2.3",
|
||||
"resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz",
|
||||
"integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
|
|
@ -5328,7 +5350,6 @@
|
|||
},
|
||||
"node_modules/require-from-string": {
|
||||
"version": "2.0.2",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
|
|
@ -5411,7 +5432,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/reusify": {
|
||||
"version": "1.0.4",
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz",
|
||||
"integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
|
|
@ -5421,6 +5444,8 @@
|
|||
},
|
||||
"node_modules/run-parallel": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz",
|
||||
"integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
|
|
@ -5469,7 +5494,9 @@
|
|||
"license": "MIT"
|
||||
},
|
||||
"node_modules/sass": {
|
||||
"version": "1.85.0",
|
||||
"version": "1.85.1",
|
||||
"resolved": "https://registry.npmjs.org/sass/-/sass-1.85.1.tgz",
|
||||
"integrity": "sha512-Uk8WpxM5v+0cMR0XjX9KfRIacmSG86RH4DCCZjLU2rFh5tyutt9siAXJ7G+YfxQ99Q6wrRMbMlVl6KqUms71ag==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
|
|
@ -5871,7 +5898,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/terser-webpack-plugin": {
|
||||
"version": "5.3.11",
|
||||
"version": "5.3.14",
|
||||
"resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.14.tgz",
|
||||
"integrity": "sha512-vkZjpUjb6OMS7dhV+tILUW6BhpDR7P2L/aQSAv+Uwk+m8KATX9EccViHTJR2qDtACKPIYndLGCyl3FMo+r2LMw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
|
|
@ -6196,8 +6225,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/typescript": {
|
||||
"version": "5.7.3",
|
||||
"dev": true,
|
||||
"version": "5.8.2",
|
||||
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.2.tgz",
|
||||
"integrity": "sha512-aJn6wq13/afZp/jT9QZmwEjDqqvSGp1VT5GVg+f/t6/oVyrgXM6BY1h9BRh/O5p3PlUPAe+WuiEZOmb/49RqoQ==",
|
||||
"license": "Apache-2.0",
|
||||
"bin": {
|
||||
"tsc": "bin/tsc",
|
||||
|
|
@ -6208,13 +6238,15 @@
|
|||
}
|
||||
},
|
||||
"node_modules/typescript-eslint": {
|
||||
"version": "8.24.1",
|
||||
"version": "8.26.1",
|
||||
"resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.26.1.tgz",
|
||||
"integrity": "sha512-t/oIs9mYyrwZGRpDv3g+3K6nZ5uhKEMt2oNmAPwaY4/ye0+EH4nXIPYNtkYFS6QHm+1DFg34DbglYBz5P9Xysg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@typescript-eslint/eslint-plugin": "8.24.1",
|
||||
"@typescript-eslint/parser": "8.24.1",
|
||||
"@typescript-eslint/utils": "8.24.1"
|
||||
"@typescript-eslint/eslint-plugin": "8.26.1",
|
||||
"@typescript-eslint/parser": "8.26.1",
|
||||
"@typescript-eslint/utils": "8.26.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
|
||||
|
|
@ -6225,7 +6257,7 @@
|
|||
},
|
||||
"peerDependencies": {
|
||||
"eslint": "^8.57.0 || ^9.0.0",
|
||||
"typescript": ">=4.8.4 <5.8.0"
|
||||
"typescript": ">=4.8.4 <5.9.0"
|
||||
}
|
||||
},
|
||||
"node_modules/undici-types": {
|
||||
|
|
@ -6280,7 +6312,6 @@
|
|||
},
|
||||
"node_modules/uri-js-replace": {
|
||||
"version": "1.0.1",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/url": {
|
||||
|
|
@ -6311,7 +6342,6 @@
|
|||
},
|
||||
"node_modules/uuid": {
|
||||
"version": "11.1.0",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
"https://github.com/sponsors/broofa",
|
||||
"https://github.com/sponsors/ctavan"
|
||||
|
|
@ -6480,6 +6510,8 @@
|
|||
},
|
||||
"node_modules/webpack-merge": {
|
||||
"version": "6.0.1",
|
||||
"resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-6.0.1.tgz",
|
||||
"integrity": "sha512-hXXvrjtx2PLYx4qruKl+kyRSLc52V+cCvMxRjmKwoA+CBbbF5GfIBtR6kCvl0fYGqTUPKB+1ktVmTHqMOzgCBg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
|
|
@ -6643,7 +6675,6 @@
|
|||
},
|
||||
"node_modules/yaml-ast-parser": {
|
||||
"version": "0.0.43",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/yargs": {
|
||||
|
|
@ -6665,7 +6696,6 @@
|
|||
},
|
||||
"node_modules/yargs-parser": {
|
||||
"version": "21.1.1",
|
||||
"dev": true,
|
||||
"license": "ISC",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
|
|
@ -6698,7 +6728,7 @@
|
|||
"license": "MIT",
|
||||
"devDependencies": {
|
||||
"@types/jest": "^29.5.14",
|
||||
"@types/node": "^22.13.5",
|
||||
"@types/node": "^22.13.10",
|
||||
"css-loader": "^7.1.2",
|
||||
"date-fns": "^4.1.0",
|
||||
"file-loader": "^6.2.0",
|
||||
|
|
@ -6707,14 +6737,14 @@
|
|||
"mini-css-extract-plugin": "^2.9.2",
|
||||
"obsidian": "1.8.7",
|
||||
"resolve-url-loader": "^5.0.0",
|
||||
"sass": "^1.85.0",
|
||||
"sass": "^1.85.1",
|
||||
"sass-loader": "^16.0.5",
|
||||
"sync-client": "file:../sync-client",
|
||||
"terser-webpack-plugin": "^5.3.11",
|
||||
"terser-webpack-plugin": "^5.3.14",
|
||||
"ts-jest": "^29.2.6",
|
||||
"ts-loader": "^9.5.2",
|
||||
"tslib": "2.8.1",
|
||||
"typescript": "5.7.3",
|
||||
"typescript": "5.8.2",
|
||||
"url": "^0.11.4",
|
||||
"virtual-scroller": "^1.13.1",
|
||||
"webpack": "^5.98.0",
|
||||
|
|
@ -6723,22 +6753,26 @@
|
|||
},
|
||||
"sync-client": {
|
||||
"version": "0.0.0",
|
||||
"devDependencies": {
|
||||
"@types/jest": "^29.5.14",
|
||||
"@types/node": "^22.13.5",
|
||||
"dependencies": {
|
||||
"byte-base64": "^1.1.0",
|
||||
"fetch-retry": "^6.0.0",
|
||||
"jest": "^29.7.0",
|
||||
"openapi-fetch": "0.13.4",
|
||||
"openapi-fetch": "0.13.5",
|
||||
"openapi-typescript": "7.6.1",
|
||||
"p-queue": "^8.1.0",
|
||||
"uuid": "^11.1.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/jest": "^29.5.14",
|
||||
"@types/node": "^22.13.10",
|
||||
"jest": "^29.7.0",
|
||||
"sync_lib": "file:../../backend/sync_lib/pkg",
|
||||
"ts-jest": "^29.2.6",
|
||||
"ts-loader": "^9.5.2",
|
||||
"tslib": "2.8.1",
|
||||
"typescript": "5.7.3",
|
||||
"typescript": "5.8.2",
|
||||
"webpack": "^5.98.0",
|
||||
"webpack-cli": "^6.0.1"
|
||||
"webpack-cli": "^6.0.1",
|
||||
"webpack-merge": "^6.0.1"
|
||||
}
|
||||
},
|
||||
"test-client": {
|
||||
|
|
@ -6747,11 +6781,11 @@
|
|||
"test-client": "dist/cli.js"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.13.5",
|
||||
"@types/node": "^22.13.10",
|
||||
"sync-client": "file:../sync-client",
|
||||
"ts-loader": "^9.5.2",
|
||||
"tslib": "2.8.1",
|
||||
"typescript": "5.7.3",
|
||||
"typescript": "5.8.2",
|
||||
"uuid": "^11.1.0",
|
||||
"webpack": "^5.98.0",
|
||||
"webpack-cli": "^6.0.1"
|
||||
|
|
|
|||
|
|
@ -21,10 +21,10 @@
|
|||
},
|
||||
"devDependencies": {
|
||||
"concurrently": "^9.1.2",
|
||||
"eslint": "9.21.0",
|
||||
"eslint": "9.22.0",
|
||||
"eslint-plugin-unused-imports": "^4.1.4",
|
||||
"npm-check-updates": "^17.1.14",
|
||||
"prettier": "^3.5.2",
|
||||
"typescript-eslint": "8.24.1"
|
||||
"npm-check-updates": "^17.1.15",
|
||||
"prettier": "^3.5.3",
|
||||
"typescript-eslint": "8.26.1"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,29 +1,36 @@
|
|||
{
|
||||
"name": "sync-client",
|
||||
"version": "0.0.0",
|
||||
"private": true,
|
||||
"main": "dist/index.js",
|
||||
"version": "0.0.30",
|
||||
"main": "dist/sync-client.node.js",
|
||||
"browser": "dist/sync-client.web.js",
|
||||
"types": "dist/types/index.d.ts",
|
||||
"files": [
|
||||
"dist/**/*"
|
||||
],
|
||||
"scripts": {
|
||||
"dev": "webpack watch --mode development",
|
||||
"build": "webpack --mode production",
|
||||
"test": "NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules\" jest"
|
||||
},
|
||||
"dependencies": {
|
||||
"byte-base64": "^1.1.0",
|
||||
"fetch-retry": "^6.0.0",
|
||||
"openapi-fetch": "0.13.5",
|
||||
"openapi-typescript": "7.6.1",
|
||||
"p-queue": "^8.1.0",
|
||||
"uuid": "^11.1.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"tslib": "2.8.1",
|
||||
"typescript": "5.7.3",
|
||||
"sync_lib": "file:../../backend/sync_lib/pkg",
|
||||
"@types/jest": "^29.5.14",
|
||||
"@types/node": "^22.13.5",
|
||||
"@types/node": "^22.13.10",
|
||||
"jest": "^29.7.0",
|
||||
"ts-jest": "^29.2.6",
|
||||
"p-queue": "^8.1.0",
|
||||
"fetch-retry": "^6.0.0",
|
||||
"byte-base64": "^1.1.0",
|
||||
"openapi-fetch": "0.13.4",
|
||||
"openapi-typescript": "7.6.1",
|
||||
"ts-loader": "^9.5.2",
|
||||
"tslib": "2.8.1",
|
||||
"typescript": "5.8.2",
|
||||
"webpack": "^5.98.0",
|
||||
"webpack-cli": "^6.0.1"
|
||||
"webpack-cli": "^6.0.1",
|
||||
"webpack-merge": "^6.0.1",
|
||||
"sync_lib": "file:../../backend/sync_lib/pkg"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,9 @@
|
|||
import type { Logger } from "../tracing/logger";
|
||||
import type { RelativePath } from "../persistence/database";
|
||||
|
||||
// Manages locks on documents to prevent concurrent modifications
|
||||
// allowing the client's FileOperations implementation to be simpler.
|
||||
// Locks are granted in a first-in-first-out order.
|
||||
export class DocumentLocks {
|
||||
private readonly locked = new Set<RelativePath>();
|
||||
private readonly waiters = new Map<RelativePath, (() => void)[]>();
|
||||
|
|
|
|||
|
|
@ -1,16 +1,27 @@
|
|||
import type { FileSystemOperations } from "sync-client";
|
||||
import type { Database, RelativePath } from "../persistence/database";
|
||||
import type {
|
||||
Database,
|
||||
DocumentRecord,
|
||||
RelativePath
|
||||
} from "../persistence/database";
|
||||
import { FileOperations } from "./file-operations";
|
||||
import { Logger } from "../tracing/logger";
|
||||
import { assertSetContainsExactly } from "../utils/assert-set-contains-exactly";
|
||||
import type { FileSystemOperations } from "./filesystem-operations";
|
||||
|
||||
describe("File operations", () => {
|
||||
class MockDatabase {
|
||||
public async updatePath(
|
||||
class MockDatabase implements Partial<Database> {
|
||||
public getLatestDocumentByRelativePath(
|
||||
_find: RelativePath
|
||||
): DocumentRecord | undefined {
|
||||
// no-op
|
||||
return undefined;
|
||||
}
|
||||
|
||||
public move(
|
||||
_oldRelativePath: RelativePath,
|
||||
_newRelativePath: RelativePath
|
||||
): Promise<void> {
|
||||
// this is called but irrelevant for this mock
|
||||
): void {
|
||||
// no-op
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,6 @@
|
|||
import type { Logger } from "src/tracing/logger";
|
||||
import type { Logger } from "../tracing/logger";
|
||||
import type { FileSystemOperations } from "./filesystem-operations";
|
||||
import type {
|
||||
Database,
|
||||
DocumentId,
|
||||
RelativePath
|
||||
} from "src/persistence/database";
|
||||
import type { Database, RelativePath } from "../persistence/database";
|
||||
import { isBinary, isFileTypeMergable, mergeText } from "sync_lib";
|
||||
import { SafeFileSystemOperations } from "./safe-filesystem-operations";
|
||||
|
||||
|
|
@ -17,7 +13,7 @@ export class FileOperations {
|
|||
private readonly database: Database,
|
||||
fs: FileSystemOperations
|
||||
) {
|
||||
this.fs = new SafeFileSystemOperations(fs);
|
||||
this.fs = new SafeFileSystemOperations(fs, logger);
|
||||
}
|
||||
|
||||
public async listAllFiles(): Promise<RelativePath[]> {
|
||||
|
|
@ -35,7 +31,7 @@ export class FileOperations {
|
|||
|
||||
const decoder = new TextDecoder("utf-8");
|
||||
|
||||
// Normalize line endings to LF on Windows
|
||||
// Normalize line-endings to LF on Windows
|
||||
let text = decoder.decode(content);
|
||||
text = text.replace(/\r\n/g, "\n");
|
||||
|
||||
|
|
@ -46,10 +42,6 @@ export class FileOperations {
|
|||
return this.fs.getFileSize(path);
|
||||
}
|
||||
|
||||
public async getModificationTime(path: RelativePath): Promise<Date> {
|
||||
return this.fs.getModificationTime(path);
|
||||
}
|
||||
|
||||
public async exists(path: RelativePath): Promise<boolean> {
|
||||
return this.fs.exists(path);
|
||||
}
|
||||
|
|
@ -60,18 +52,23 @@ export class FileOperations {
|
|||
path: RelativePath,
|
||||
newContent: Uint8Array
|
||||
): Promise<void> {
|
||||
this.logger.debug(`Creating file: ${path}`);
|
||||
|
||||
await this.fs.write(path, newContent);
|
||||
}
|
||||
|
||||
public async ensureClearPath(path: RelativePath): Promise<void> {
|
||||
if (await this.fs.exists(path)) {
|
||||
const deconflictedPath = await this.deconflictPath(path);
|
||||
this.logger.debug(
|
||||
`Didn't expect ${path} to exist, deconflicting by moving it to '${deconflictedPath}'`
|
||||
);
|
||||
await this.database.updatePath(path, deconflictedPath);
|
||||
|
||||
this.database.move(path, deconflictedPath);
|
||||
await this.fs.rename(path, deconflictedPath);
|
||||
} else {
|
||||
await this.createParentDirectories(path);
|
||||
}
|
||||
|
||||
await this.fs.write(path, newContent);
|
||||
}
|
||||
|
||||
// Update the file at the given path.
|
||||
|
|
@ -126,40 +123,25 @@ export class FileOperations {
|
|||
return new TextEncoder().encode(resultText);
|
||||
}
|
||||
|
||||
public async remove(path: RelativePath): Promise<void> {
|
||||
this.logger.debug(`Deleting file: ${path}`);
|
||||
return this.fs.delete(path);
|
||||
public async delete(path: RelativePath): Promise<void> {
|
||||
if (await this.exists(path)) {
|
||||
this.logger.debug(`Deleting file: ${path}`);
|
||||
return this.fs.delete(path);
|
||||
} else {
|
||||
this.logger.debug(`No need to delete '${path}', it doesn't exist`);
|
||||
}
|
||||
}
|
||||
|
||||
public async move(
|
||||
oldPath: RelativePath,
|
||||
newPath: RelativePath,
|
||||
documentId?: DocumentId
|
||||
newPath: RelativePath
|
||||
): Promise<void> {
|
||||
if (oldPath === newPath) {
|
||||
return;
|
||||
}
|
||||
await this.ensureClearPath(newPath);
|
||||
|
||||
if (await this.fs.exists(newPath)) {
|
||||
const deconflictedPath = await this.deconflictPath(newPath);
|
||||
this.logger.debug(
|
||||
`Conflict when moving '${oldPath}' to '${newPath}', the latter already exists, deconflicting by moving it to '${deconflictedPath}'`
|
||||
);
|
||||
|
||||
const existingMetadata = this.database.getDocument(newPath);
|
||||
if (
|
||||
existingMetadata === undefined ||
|
||||
existingMetadata.documentId !== documentId
|
||||
) {
|
||||
await this.database.updatePath(newPath, deconflictedPath);
|
||||
await this.fs.rename(newPath, deconflictedPath);
|
||||
} else {
|
||||
await this.database.deleteDocument(newPath);
|
||||
}
|
||||
} else {
|
||||
await this.createParentDirectories(newPath);
|
||||
}
|
||||
|
||||
this.database.move(oldPath, newPath);
|
||||
await this.fs.rename(oldPath, newPath);
|
||||
}
|
||||
|
||||
|
|
@ -201,17 +183,12 @@ export class FileOperations {
|
|||
);
|
||||
stem = stem.replace(FileOperations.PARENTHESES_REGEX, "");
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
||||
while (true) {
|
||||
const newName =
|
||||
currentCount === 0
|
||||
? `${directory}${stem}${extension}`
|
||||
: `${directory}${stem} (${currentCount})${extension}`;
|
||||
if (await this.fs.exists(newName)) {
|
||||
currentCount++;
|
||||
} else {
|
||||
return newName;
|
||||
}
|
||||
}
|
||||
let newName = path;
|
||||
do {
|
||||
currentCount++;
|
||||
newName = `${directory}${stem} (${currentCount})${extension}`;
|
||||
} while (await this.fs.exists(newName));
|
||||
|
||||
return newName;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import type { RelativePath } from "src/persistence/database";
|
||||
import type { RelativePath } from "../persistence/database";
|
||||
|
||||
export interface FileSystemOperations {
|
||||
listAllFiles: () => Promise<RelativePath[]>;
|
||||
|
|
@ -9,11 +9,8 @@ export interface FileSystemOperations {
|
|||
updater: (currentContent: string) => string
|
||||
) => Promise<string>;
|
||||
getFileSize: (path: RelativePath) => Promise<number>;
|
||||
getModificationTime: (path: RelativePath) => Promise<Date>;
|
||||
exists: (path: RelativePath) => Promise<boolean>;
|
||||
createDirectory: (path: RelativePath) => Promise<void>;
|
||||
delete: (path: RelativePath) => Promise<void>;
|
||||
|
||||
// Must be able to handle renaming to a file that already exists
|
||||
rename: (oldPath: RelativePath, newPath: RelativePath) => Promise<void>;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
import type { RelativePath } from "src/persistence/database";
|
||||
import type { RelativePath } from "../persistence/database";
|
||||
import type { FileSystemOperations } from "./filesystem-operations";
|
||||
import type { Logger } from "../tracing/logger";
|
||||
import { DocumentLocks } from "./document-locks";
|
||||
|
||||
export class FileNotFoundError extends Error {
|
||||
public constructor(message: string) {
|
||||
|
|
@ -9,71 +11,134 @@ export class FileNotFoundError extends Error {
|
|||
}
|
||||
|
||||
// Decorate FileSystemOperations replacing errors with FileNotFoundError
|
||||
// if the accessed file doesn't exist.
|
||||
// if the accessed file doesn't exist. It also ensures that there's only
|
||||
// ever a single request in-flight for any one file through the use of
|
||||
// DocumentLocks.
|
||||
export class SafeFileSystemOperations implements FileSystemOperations {
|
||||
public constructor(private readonly fs: FileSystemOperations) {}
|
||||
private readonly locks: DocumentLocks;
|
||||
|
||||
public constructor(
|
||||
private readonly fs: FileSystemOperations,
|
||||
private readonly logger: Logger
|
||||
) {
|
||||
this.locks = new DocumentLocks(logger);
|
||||
}
|
||||
|
||||
public async listAllFiles(): Promise<RelativePath[]> {
|
||||
return this.fs.listAllFiles();
|
||||
}
|
||||
|
||||
public async read(path: RelativePath): Promise<Uint8Array> {
|
||||
return this.safeOperation(path, async () => this.fs.read(path));
|
||||
this.logger.debug(`Reading file: ${path}`);
|
||||
return this.safeOperation(
|
||||
path,
|
||||
this.decorateToHoldLock(path, async () => this.fs.read(path)),
|
||||
"read"
|
||||
);
|
||||
}
|
||||
|
||||
public async write(path: RelativePath, content: Uint8Array): Promise<void> {
|
||||
return this.fs.write(path, content);
|
||||
this.logger.debug(`Writing file: ${path}`);
|
||||
return this.decorateToHoldLock(path, async () =>
|
||||
this.fs.write(path, content)
|
||||
)();
|
||||
}
|
||||
|
||||
public async atomicUpdateText(
|
||||
path: RelativePath,
|
||||
updater: (currentContent: string) => string
|
||||
): Promise<string> {
|
||||
return this.safeOperation(path, async () =>
|
||||
this.fs.atomicUpdateText(path, updater)
|
||||
this.logger.debug(`Atomic update of file: ${path}`);
|
||||
return this.safeOperation(
|
||||
path,
|
||||
this.decorateToHoldLock(path, async () =>
|
||||
this.fs.atomicUpdateText(path, updater)
|
||||
),
|
||||
"atomicUpdateText"
|
||||
);
|
||||
}
|
||||
|
||||
public async getFileSize(path: RelativePath): Promise<number> {
|
||||
return this.safeOperation(path, async () => this.fs.getFileSize(path));
|
||||
}
|
||||
|
||||
public async getModificationTime(path: RelativePath): Promise<Date> {
|
||||
return this.safeOperation(path, async () =>
|
||||
this.fs.getModificationTime(path)
|
||||
this.logger.debug(`Getting file size: ${path}`);
|
||||
return this.safeOperation(
|
||||
path,
|
||||
this.decorateToHoldLock(path, async () =>
|
||||
this.fs.getFileSize(path)
|
||||
),
|
||||
"getFileSize"
|
||||
);
|
||||
}
|
||||
|
||||
public async exists(path: RelativePath): Promise<boolean> {
|
||||
return this.fs.exists(path);
|
||||
this.logger.debug(`Checking if file exists: ${path}`);
|
||||
return this.decorateToHoldLock(path, async () =>
|
||||
this.fs.exists(path)
|
||||
)();
|
||||
}
|
||||
|
||||
public async createDirectory(path: RelativePath): Promise<void> {
|
||||
return this.fs.createDirectory(path);
|
||||
this.logger.debug(`Creating directory: ${path}`);
|
||||
return this.decorateToHoldLock(path, async () =>
|
||||
this.fs.createDirectory(path)
|
||||
)();
|
||||
}
|
||||
|
||||
public async delete(path: RelativePath): Promise<void> {
|
||||
return this.fs.delete(path);
|
||||
this.logger.debug(`Deleting file: ${path}`);
|
||||
return this.decorateToHoldLock(path, async () =>
|
||||
this.fs.delete(path)
|
||||
)();
|
||||
}
|
||||
|
||||
public async rename(
|
||||
oldPath: RelativePath,
|
||||
newPath: RelativePath
|
||||
): Promise<void> {
|
||||
return this.safeOperation(oldPath, async () =>
|
||||
this.fs.rename(oldPath, newPath)
|
||||
this.logger.debug(`Renaming file: ${oldPath} to ${newPath}`);
|
||||
return this.safeOperation(
|
||||
oldPath,
|
||||
this.decorateToHoldLock([oldPath, newPath], async () =>
|
||||
this.fs.rename(oldPath, newPath)
|
||||
),
|
||||
"rename"
|
||||
);
|
||||
}
|
||||
|
||||
private decorateToHoldLock<T>(
|
||||
pathOrPaths: RelativePath | RelativePath[],
|
||||
operation: () => Promise<T>
|
||||
): () => Promise<T> {
|
||||
return async () => {
|
||||
const paths = Array.isArray(pathOrPaths)
|
||||
? pathOrPaths
|
||||
: [pathOrPaths];
|
||||
await Promise.all(
|
||||
paths.map(async (path) => this.locks.waitForDocumentLock(path))
|
||||
);
|
||||
try {
|
||||
return await operation();
|
||||
} finally {
|
||||
await Promise.all(
|
||||
paths.map((path) => {
|
||||
this.locks.unlockDocument(path);
|
||||
})
|
||||
);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private async safeOperation<T>(
|
||||
path: RelativePath,
|
||||
operation: () => Promise<T>
|
||||
operation: () => Promise<T>,
|
||||
operationName: string
|
||||
): Promise<T> {
|
||||
// Without locking the file, this isn't atomic, however, it's good enough practicaly.
|
||||
// This will only break if the file exists, gets deleted and then immediately
|
||||
// recreated while `operation` is running.
|
||||
if (!(await this.fs.exists(path))) {
|
||||
throw new FileNotFoundError(path);
|
||||
throw new FileNotFoundError(
|
||||
`File not found: ${path} before trying to ${operationName}`
|
||||
);
|
||||
}
|
||||
try {
|
||||
return await operation();
|
||||
|
|
@ -81,7 +146,9 @@ export class SafeFileSystemOperations implements FileSystemOperations {
|
|||
if (await this.fs.exists(path)) {
|
||||
throw error;
|
||||
} else {
|
||||
throw new FileNotFoundError(path);
|
||||
throw new FileNotFoundError(
|
||||
`File not found: ${path} when trying to ${operationName}`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,23 +1,43 @@
|
|||
import type { Logger } from "../tracing/logger";
|
||||
|
||||
export type VaultUpdateId = number;
|
||||
export type DocumentId = string;
|
||||
export type RelativePath = string;
|
||||
|
||||
export interface DocumentMetadata {
|
||||
parentVersionId: VaultUpdateId;
|
||||
documentId: DocumentId;
|
||||
hash: string;
|
||||
}
|
||||
|
||||
import type { Logger } from "src/tracing/logger";
|
||||
export interface StoredDocumentMetadata {
|
||||
relativePath: RelativePath;
|
||||
documentId: DocumentId;
|
||||
parentVersionId: VaultUpdateId;
|
||||
hash: string;
|
||||
}
|
||||
|
||||
export interface StoredDatabase {
|
||||
documents: Record<RelativePath, DocumentMetadata>;
|
||||
documents: StoredDocumentMetadata[];
|
||||
lastSeenUpdateId: VaultUpdateId | undefined;
|
||||
}
|
||||
|
||||
export class Database {
|
||||
private documents = new Map<RelativePath, DocumentMetadata>();
|
||||
/**
|
||||
* Represents a document in the database.
|
||||
*
|
||||
* It is mutable and its content should always represent the latest
|
||||
* state of the document on disk based on the update events we have seen.
|
||||
*/
|
||||
export interface DocumentRecord {
|
||||
relativePath: RelativePath;
|
||||
documentId: DocumentId;
|
||||
metadata: DocumentMetadata | undefined;
|
||||
isDeleted: boolean;
|
||||
updates: Promise<void>[];
|
||||
parallelVersion: number;
|
||||
}
|
||||
|
||||
export class Database {
|
||||
private documents: DocumentRecord[];
|
||||
private lastSeenUpdateId: VaultUpdateId | undefined;
|
||||
|
||||
public constructor(
|
||||
|
|
@ -26,16 +46,21 @@ export class Database {
|
|||
private readonly saveData: (data: StoredDatabase) => Promise<void>
|
||||
) {
|
||||
initialState ??= {};
|
||||
if (initialState.documents) {
|
||||
for (const [relativePath, metadata] of Object.entries(
|
||||
initialState.documents
|
||||
)) {
|
||||
this.documents.set(relativePath, metadata);
|
||||
}
|
||||
}
|
||||
this.ensureConsistency();
|
||||
|
||||
this.logger.debug(`Loaded ${this.documents.size} documents`);
|
||||
this.documents =
|
||||
initialState.documents?.map(
|
||||
({ relativePath, documentId, ...metadata }) => ({
|
||||
relativePath,
|
||||
documentId,
|
||||
metadata,
|
||||
isDeleted: false,
|
||||
updates: [],
|
||||
parallelVersion: 0
|
||||
})
|
||||
) ?? [];
|
||||
|
||||
this.ensureConsistency();
|
||||
this.logger.debug(`Loaded ${this.documents.length} documents`);
|
||||
|
||||
this.lastSeenUpdateId = initialState.lastSeenUpdateId;
|
||||
this.logger.debug(
|
||||
|
|
@ -43,109 +68,213 @@ export class Database {
|
|||
);
|
||||
}
|
||||
|
||||
public getDocuments(): Map<RelativePath, DocumentMetadata> {
|
||||
return this.documents;
|
||||
public get length(): number {
|
||||
return this.documents.length;
|
||||
}
|
||||
|
||||
public get resolvedDocuments(): DocumentRecord[] {
|
||||
const paths = new Map<string, DocumentRecord[]>();
|
||||
this.documents
|
||||
.filter(({ metadata }) => metadata !== undefined)
|
||||
.forEach((record) =>
|
||||
paths.set(record.relativePath, [
|
||||
record,
|
||||
...(paths.get(record.relativePath) ?? [])
|
||||
])
|
||||
);
|
||||
|
||||
return Array.from(paths.values()).map((records) => {
|
||||
records.sort(
|
||||
(a, b) => b.parallelVersion - a.parallelVersion // descending
|
||||
);
|
||||
|
||||
if (
|
||||
records.length > 1 &&
|
||||
records.some((current, i) =>
|
||||
i === 0
|
||||
? false
|
||||
: records[i - 1].parallelVersion ===
|
||||
current.parallelVersion
|
||||
)
|
||||
) {
|
||||
throw new Error(
|
||||
`Multiple documents with the same parallel version and path at ${records[0].relativePath}`
|
||||
);
|
||||
}
|
||||
return records[0];
|
||||
});
|
||||
}
|
||||
|
||||
public getLastSeenUpdateId(): VaultUpdateId | undefined {
|
||||
return this.lastSeenUpdateId;
|
||||
}
|
||||
|
||||
public async setLastSeenUpdateId(
|
||||
value: VaultUpdateId | undefined
|
||||
): Promise<void> {
|
||||
public setLastSeenUpdateId(value: VaultUpdateId | undefined): void {
|
||||
this.lastSeenUpdateId = value;
|
||||
await this.save();
|
||||
this.save();
|
||||
}
|
||||
|
||||
public async resetSyncState(): Promise<void> {
|
||||
this.documents = new Map();
|
||||
public resetSyncState(): void {
|
||||
this.documents = [];
|
||||
this.lastSeenUpdateId = 0;
|
||||
await this.save();
|
||||
this.save();
|
||||
}
|
||||
|
||||
public updateDocumentMetadata(
|
||||
metadata: {
|
||||
parentVersionId: VaultUpdateId;
|
||||
hash: string;
|
||||
},
|
||||
toUpdate: DocumentRecord
|
||||
): void {
|
||||
if (!this.documents.includes(toUpdate)) {
|
||||
throw new Error("Document not found in database");
|
||||
}
|
||||
|
||||
toUpdate.metadata = metadata;
|
||||
|
||||
this.save();
|
||||
}
|
||||
|
||||
public removeDocumentPromise(promise: Promise<void>): void {
|
||||
const entry = this.documents.find(({ updates }) =>
|
||||
updates.includes(promise)
|
||||
);
|
||||
|
||||
if (entry === undefined) {
|
||||
throw new Error("Document not found by update promise");
|
||||
}
|
||||
|
||||
entry.updates = entry.updates.filter((update) => update !== promise);
|
||||
// No need to save as Promises don't get serialized
|
||||
}
|
||||
|
||||
public getLatestDocumentByRelativePath(
|
||||
find: RelativePath
|
||||
): DocumentRecord | undefined {
|
||||
const candidates = this.documents.filter(
|
||||
({ relativePath }) => relativePath === find
|
||||
);
|
||||
candidates.sort((a, b) => b.parallelVersion - a.parallelVersion); // descending
|
||||
return candidates[0];
|
||||
}
|
||||
|
||||
public async getResolvedDocumentByRelativePath(
|
||||
relativePath: RelativePath,
|
||||
promise: Promise<void>
|
||||
): Promise<DocumentRecord> {
|
||||
const entry = this.getLatestDocumentByRelativePath(relativePath);
|
||||
|
||||
if (entry === undefined) {
|
||||
throw new Error(
|
||||
`Document not found by relative path: ${relativePath}, ${JSON.stringify(
|
||||
this.documents,
|
||||
null,
|
||||
2
|
||||
)}`
|
||||
);
|
||||
}
|
||||
|
||||
const currentPromises = entry.updates;
|
||||
entry.updates = [...currentPromises, promise];
|
||||
await Promise.all(currentPromises);
|
||||
|
||||
return entry;
|
||||
}
|
||||
|
||||
public createNewPendingDocument(
|
||||
documentId: DocumentId,
|
||||
relativePath: RelativePath,
|
||||
promise: Promise<void>
|
||||
): DocumentRecord {
|
||||
const previousEntry =
|
||||
this.getLatestDocumentByRelativePath(relativePath);
|
||||
|
||||
const entry = {
|
||||
relativePath,
|
||||
documentId,
|
||||
metadata: undefined,
|
||||
isDeleted: false,
|
||||
updates: [promise],
|
||||
parallelVersion:
|
||||
previousEntry?.parallelVersion === undefined
|
||||
? 0
|
||||
: previousEntry.parallelVersion + 1
|
||||
};
|
||||
|
||||
this.documents.push(entry);
|
||||
this.save();
|
||||
|
||||
return entry;
|
||||
}
|
||||
|
||||
public getDocumentByDocumentId(
|
||||
documentId: DocumentId
|
||||
): [RelativePath, DocumentMetadata] | undefined {
|
||||
return [...this.documents.entries()].find(
|
||||
([_, metadata]) => metadata.documentId === documentId
|
||||
);
|
||||
find: DocumentId
|
||||
): DocumentRecord | undefined {
|
||||
return this.documents.find(({ documentId }) => documentId === find);
|
||||
}
|
||||
|
||||
public async setDocument({
|
||||
documentId,
|
||||
relativePath,
|
||||
parentVersionId,
|
||||
hash
|
||||
}: {
|
||||
documentId: DocumentId;
|
||||
relativePath: RelativePath;
|
||||
parentVersionId: VaultUpdateId;
|
||||
hash: string;
|
||||
}): Promise<void> {
|
||||
this.documents.set(relativePath, {
|
||||
documentId,
|
||||
parentVersionId,
|
||||
hash
|
||||
});
|
||||
await this.save();
|
||||
}
|
||||
|
||||
public async removeDocument(relativePath: RelativePath): Promise<void> {
|
||||
this.documents.delete(relativePath);
|
||||
await this.save();
|
||||
}
|
||||
|
||||
public getDocument(
|
||||
relativePath: RelativePath
|
||||
): DocumentMetadata | undefined {
|
||||
return this.documents.get(relativePath);
|
||||
}
|
||||
|
||||
public async deleteDocument(relativePath: RelativePath): Promise<void> {
|
||||
this.documents.delete(relativePath);
|
||||
await this.save();
|
||||
}
|
||||
|
||||
public async updatePath(
|
||||
public move(
|
||||
oldRelativePath: RelativePath,
|
||||
newRelativePath: RelativePath
|
||||
): Promise<void> {
|
||||
const document = this.documents.get(oldRelativePath);
|
||||
if (!document) {
|
||||
): void {
|
||||
const oldDocument =
|
||||
this.getLatestDocumentByRelativePath(oldRelativePath);
|
||||
|
||||
if (oldDocument === undefined) {
|
||||
return;
|
||||
}
|
||||
|
||||
const newDocument =
|
||||
this.getLatestDocumentByRelativePath(newRelativePath);
|
||||
if (newDocument?.isDeleted === false) {
|
||||
throw new Error(
|
||||
`Cannot update physical path for document that does not exist: ${oldRelativePath}`
|
||||
`Document already exists at new location: ${newRelativePath}`
|
||||
);
|
||||
}
|
||||
|
||||
if (this.documents.has(newRelativePath)) {
|
||||
throw new Error(
|
||||
`Cannot update physical path to path that is already in use: ${newRelativePath}`
|
||||
);
|
||||
}
|
||||
oldDocument.relativePath = newRelativePath;
|
||||
// We're in a strange state where the target of the move has just got deleted,
|
||||
// however, its metadata might already have a bunch of updates queued up for
|
||||
// the document at the new location. We need to keep these updates.
|
||||
oldDocument.parallelVersion =
|
||||
newDocument !== undefined ? newDocument.parallelVersion + 1 : 0;
|
||||
|
||||
this.documents.delete(oldRelativePath);
|
||||
this.documents.set(newRelativePath, document);
|
||||
|
||||
await this.save();
|
||||
this.save();
|
||||
}
|
||||
|
||||
private async save(): Promise<void> {
|
||||
public delete(relativePath: RelativePath): void {
|
||||
const candidate = this.getLatestDocumentByRelativePath(relativePath);
|
||||
if (candidate === undefined) {
|
||||
throw new Error(
|
||||
`Document not found by relative path: ${relativePath}`
|
||||
);
|
||||
}
|
||||
candidate.isDeleted = true;
|
||||
}
|
||||
|
||||
private save(): void {
|
||||
this.ensureConsistency();
|
||||
await this.saveData({
|
||||
documents: Object.fromEntries(this.documents.entries()),
|
||||
void this.saveData({
|
||||
documents: this.resolvedDocuments.map(
|
||||
({ relativePath, documentId, metadata }) => ({
|
||||
documentId,
|
||||
relativePath,
|
||||
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
||||
...metadata! // resolvedDocuments only returns docs with metadata set
|
||||
})
|
||||
),
|
||||
lastSeenUpdateId: this.lastSeenUpdateId
|
||||
});
|
||||
}
|
||||
|
||||
private ensureConsistency(): void {
|
||||
const allMetadata = Array.from(this.documents.entries());
|
||||
const idToPath = new Map<string, Array<string>>();
|
||||
const idToPath = new Map<string, string[]>();
|
||||
|
||||
allMetadata.forEach(([name, metadata]) => {
|
||||
idToPath.set(metadata.documentId, [
|
||||
...(idToPath.get(metadata.documentId) ?? []),
|
||||
name
|
||||
this.resolvedDocuments.forEach(({ relativePath, documentId }) => {
|
||||
idToPath.set(documentId, [
|
||||
...(idToPath.get(documentId) ?? []),
|
||||
relativePath
|
||||
]);
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import type { Logger } from "src/tracing/logger";
|
||||
import { LogLevel } from "src/tracing/logger";
|
||||
import type { Logger } from "../tracing/logger";
|
||||
import { LogLevel } from "../tracing/logger";
|
||||
|
||||
export interface SyncSettings {
|
||||
remoteUri: string;
|
||||
|
|
|
|||
51
frontend/sync-client/src/services/connected-state.ts
Normal file
51
frontend/sync-client/src/services/connected-state.ts
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
import type { Settings } from "../persistence/settings";
|
||||
import type { Logger } from "../tracing/logger";
|
||||
import { createPromise } from "../utils/create-promise";
|
||||
import { retriedFetchFactory } from "../utils/retried-fetch";
|
||||
|
||||
export class ConnectedState {
|
||||
private resolveIsSyncEnabled: (() => void) | undefined;
|
||||
private syncIsEnabled: Promise<void> | undefined;
|
||||
|
||||
public constructor(
|
||||
settings: Settings,
|
||||
private readonly logger: Logger
|
||||
) {
|
||||
settings.addOnSettingsChangeHandlers((newSettings, oldSettings) => {
|
||||
if (!oldSettings.isSyncEnabled && newSettings.isSyncEnabled) {
|
||||
this.handleComingOnline();
|
||||
} else if (
|
||||
oldSettings.isSyncEnabled &&
|
||||
!newSettings.isSyncEnabled
|
||||
) {
|
||||
this.handleGoingOffline();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public getFetchImplementation(
|
||||
fetch: typeof globalThis.fetch,
|
||||
{ doRetries = true }: { doRetries: boolean } = { doRetries: true }
|
||||
): typeof globalThis.fetch {
|
||||
const retriedFetch = doRetries
|
||||
? retriedFetchFactory(this.logger, fetch)
|
||||
: fetch;
|
||||
|
||||
return async (input: RequestInfo | URL): Promise<Response> => {
|
||||
if (this.syncIsEnabled !== undefined) {
|
||||
await this.syncIsEnabled;
|
||||
}
|
||||
return retriedFetch(input);
|
||||
};
|
||||
}
|
||||
|
||||
private handleComingOnline(): void {
|
||||
this.logger.debug("Sync is enabled");
|
||||
this.resolveIsSyncEnabled?.();
|
||||
}
|
||||
|
||||
private handleGoingOffline(): void {
|
||||
this.logger.debug("Sync is disabled");
|
||||
[this.syncIsEnabled, this.resolveIsSyncEnabled] = createPromise();
|
||||
}
|
||||
}
|
||||
|
|
@ -6,20 +6,22 @@ import type {
|
|||
RelativePath,
|
||||
VaultUpdateId
|
||||
} from "../persistence/database";
|
||||
import type { Logger } from "src/tracing/logger";
|
||||
import { retriedFetchFactory } from "src/utils/retried-fetch";
|
||||
import type { Settings } from "src/persistence/settings";
|
||||
import type { Logger } from "../tracing/logger";
|
||||
import type { Settings } from "../persistence/settings";
|
||||
import type { ConnectedState } from "./connected-state";
|
||||
|
||||
export interface CheckConnectionResult {
|
||||
isSuccessful: boolean;
|
||||
message: string;
|
||||
}
|
||||
|
||||
export class SyncService {
|
||||
private client!: Client<paths>;
|
||||
private clientWithoutRetries!: Client<paths>;
|
||||
private _fetchImplementation: typeof globalThis.fetch = globalThis.fetch;
|
||||
|
||||
public constructor(
|
||||
private readonly connectedState: ConnectedState,
|
||||
private readonly settings: Settings,
|
||||
private readonly logger: Logger
|
||||
) {
|
||||
|
|
@ -52,17 +54,19 @@ export class SyncService {
|
|||
}
|
||||
|
||||
public async create({
|
||||
documentId,
|
||||
relativePath,
|
||||
contentBytes,
|
||||
createdDate
|
||||
contentBytes
|
||||
}: {
|
||||
documentId?: DocumentId;
|
||||
relativePath: RelativePath;
|
||||
contentBytes: Uint8Array;
|
||||
createdDate: Date;
|
||||
}): Promise<components["schemas"]["DocumentUpdateResponse"]> {
|
||||
}): Promise<components["schemas"]["DocumentVersionWithoutContent"]> {
|
||||
const formData = new FormData();
|
||||
if (documentId !== undefined) {
|
||||
formData.append("document_id", documentId);
|
||||
}
|
||||
formData.append("relative_path", relativePath);
|
||||
formData.append("created_date", createdDate.toISOString());
|
||||
formData.append("content", new Blob([contentBytes]));
|
||||
|
||||
const response = await this.client.POST(
|
||||
|
|
@ -100,18 +104,18 @@ export class SyncService {
|
|||
parentVersionId,
|
||||
documentId,
|
||||
relativePath,
|
||||
contentBytes,
|
||||
createdDate
|
||||
contentBytes
|
||||
}: {
|
||||
parentVersionId: VaultUpdateId;
|
||||
documentId: DocumentId;
|
||||
relativePath: RelativePath;
|
||||
contentBytes: Uint8Array;
|
||||
createdDate: Date;
|
||||
}): Promise<components["schemas"]["DocumentUpdateResponse"]> {
|
||||
this.logger.debug(
|
||||
`Updating document ${documentId} with parent version ${parentVersionId} and relative path ${relativePath}`
|
||||
);
|
||||
const formData = new FormData();
|
||||
formData.append("parent_version_id", parentVersionId.toString());
|
||||
formData.append("created_date", createdDate.toISOString());
|
||||
formData.append("relative_path", relativePath);
|
||||
formData.append("content", new Blob([contentBytes]));
|
||||
|
||||
|
|
@ -149,13 +153,11 @@ export class SyncService {
|
|||
|
||||
public async delete({
|
||||
documentId,
|
||||
relativePath,
|
||||
createdDate
|
||||
relativePath
|
||||
}: {
|
||||
documentId: DocumentId;
|
||||
relativePath: RelativePath;
|
||||
createdDate: Date;
|
||||
}): Promise<void> {
|
||||
}): Promise<components["schemas"]["DocumentVersionWithoutContent"]> {
|
||||
const response = await this.client.DELETE(
|
||||
"/vaults/{vault_id}/documents/{document_id}",
|
||||
{
|
||||
|
|
@ -169,7 +171,6 @@ export class SyncService {
|
|||
}
|
||||
},
|
||||
body: {
|
||||
createdDate: createdDate.toISOString(),
|
||||
relativePath
|
||||
}
|
||||
}
|
||||
|
|
@ -295,11 +296,17 @@ export class SyncService {
|
|||
private createClient(remoteUri: string): void {
|
||||
this.client = createClient<paths>({
|
||||
baseUrl: remoteUri,
|
||||
fetch: retriedFetchFactory(this.logger, this._fetchImplementation)
|
||||
fetch: this.connectedState.getFetchImplementation(
|
||||
this._fetchImplementation
|
||||
)
|
||||
});
|
||||
|
||||
this.clientWithoutRetries = createClient<paths>({
|
||||
baseUrl: remoteUri
|
||||
baseUrl: remoteUri,
|
||||
fetch: this.connectedState.getFetchImplementation(
|
||||
this._fetchImplementation,
|
||||
{ doRetries: false }
|
||||
)
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -274,12 +274,13 @@ export interface paths {
|
|||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description no content */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content?: never;
|
||||
content: {
|
||||
"application/json": components["schemas"]["DocumentVersionWithoutContent"];
|
||||
};
|
||||
};
|
||||
default: {
|
||||
headers: {
|
||||
|
|
@ -451,26 +452,25 @@ export interface components {
|
|||
Array_of_uint8: number[];
|
||||
CreateDocumentVersion: {
|
||||
contentBase64: string;
|
||||
/** Format: date-time */
|
||||
createdDate: string;
|
||||
/**
|
||||
* Format: uuid
|
||||
* @description The client can decide the document id (if it wishes to) in order to help with syncing. If the client does not provide a document id, the server will generate one. If the client provides a document id it must not already exist in the database.
|
||||
*/
|
||||
documentId?: string | null;
|
||||
relativePath: string;
|
||||
};
|
||||
CreateDocumentVersionMultipart: {
|
||||
content: components["schemas"]["Array_of_uint8"];
|
||||
/** Format: date-time */
|
||||
created_date: string;
|
||||
/** Format: uuid */
|
||||
document_id?: string | null;
|
||||
relative_path: string;
|
||||
};
|
||||
DeleteDocumentVersion: {
|
||||
/** Format: date-time */
|
||||
createdDate: string;
|
||||
relativePath: string;
|
||||
};
|
||||
/** @description Response to a update document request. */
|
||||
/** @description Response to an update document request. */
|
||||
DocumentUpdateResponse:
|
||||
| {
|
||||
/** Format: date-time */
|
||||
createdDate: string;
|
||||
/** Format: uuid */
|
||||
documentId: string;
|
||||
isDeleted: boolean;
|
||||
|
|
@ -479,14 +479,11 @@ export interface components {
|
|||
type: "FastForwardUpdate";
|
||||
/** Format: date-time */
|
||||
updatedDate: string;
|
||||
vaultId: string;
|
||||
/** Format: int64 */
|
||||
vaultUpdateId: number;
|
||||
}
|
||||
| {
|
||||
contentBase64: string;
|
||||
/** Format: date-time */
|
||||
createdDate: string;
|
||||
/** Format: uuid */
|
||||
documentId: string;
|
||||
isDeleted: boolean;
|
||||
|
|
@ -495,34 +492,27 @@ export interface components {
|
|||
type: "MergingUpdate";
|
||||
/** Format: date-time */
|
||||
updatedDate: string;
|
||||
vaultId: string;
|
||||
/** Format: int64 */
|
||||
vaultUpdateId: number;
|
||||
};
|
||||
DocumentVersion: {
|
||||
contentBase64: string;
|
||||
/** Format: date-time */
|
||||
createdDate: string;
|
||||
/** Format: uuid */
|
||||
documentId: string;
|
||||
isDeleted: boolean;
|
||||
relativePath: string;
|
||||
/** Format: date-time */
|
||||
updatedDate: string;
|
||||
vaultId: string;
|
||||
/** Format: int64 */
|
||||
vaultUpdateId: number;
|
||||
};
|
||||
DocumentVersionWithoutContent: {
|
||||
/** Format: date-time */
|
||||
createdDate: string;
|
||||
/** Format: uuid */
|
||||
documentId: string;
|
||||
isDeleted: boolean;
|
||||
relativePath: string;
|
||||
/** Format: date-time */
|
||||
updatedDate: string;
|
||||
vaultId: string;
|
||||
/** Format: int64 */
|
||||
vaultUpdateId: number;
|
||||
};
|
||||
|
|
@ -587,16 +577,12 @@ export interface components {
|
|||
};
|
||||
UpdateDocumentVersion: {
|
||||
contentBase64: string;
|
||||
/** Format: date-time */
|
||||
createdDate: string;
|
||||
/** Format: int64 */
|
||||
parentVersionId: number;
|
||||
relativePath: string;
|
||||
};
|
||||
UpdateDocumentVersionMultipart: {
|
||||
content: components["schemas"]["Array_of_uint8"];
|
||||
/** Format: date-time */
|
||||
createdDate: string;
|
||||
/** Format: int64 */
|
||||
parentVersionId: number;
|
||||
relativePath: string;
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import { SyncService } from "./services/sync-service";
|
|||
import { Syncer } from "./sync-operations/syncer";
|
||||
import type { FileSystemOperations } from "./file-operations/filesystem-operations";
|
||||
import { FileOperations } from "./file-operations/file-operations";
|
||||
import { ConnectedState } from "./services/connected-state";
|
||||
|
||||
export class SyncClient {
|
||||
private remoteListenerIntervalId: NodeJS.Timeout | null = null;
|
||||
|
|
@ -42,7 +43,7 @@ export class SyncClient {
|
|||
}
|
||||
|
||||
public get documentCount(): number {
|
||||
return this._database.getDocuments().size;
|
||||
return this._database.length;
|
||||
}
|
||||
|
||||
public set fetchImplementation(fetch: typeof globalThis.fetch) {
|
||||
|
|
@ -90,7 +91,9 @@ export class SyncClient {
|
|||
}
|
||||
);
|
||||
|
||||
const syncService = new SyncService(settings, logger);
|
||||
const connectedState = new ConnectedState(settings, logger);
|
||||
|
||||
const syncService = new SyncService(connectedState, settings, logger);
|
||||
|
||||
const syncer = new Syncer(
|
||||
logger,
|
||||
|
|
@ -117,18 +120,13 @@ export class SyncClient {
|
|||
);
|
||||
|
||||
settings.addOnSettingsChangeHandlers((newSettings, oldSettings) => {
|
||||
client.registerRemoteEventListener(
|
||||
newSettings.fetchChangesUpdateIntervalMs
|
||||
);
|
||||
|
||||
if (!oldSettings.isSyncEnabled && newSettings.isSyncEnabled) {
|
||||
syncer
|
||||
.scheduleSyncForOfflineChanges()
|
||||
.catch((_error: unknown) => {
|
||||
logger.error(
|
||||
"Failed to schedule sync for offline changes"
|
||||
);
|
||||
});
|
||||
if (
|
||||
newSettings.fetchChangesUpdateIntervalMs !==
|
||||
oldSettings.fetchChangesUpdateIntervalMs
|
||||
) {
|
||||
client.registerRemoteEventListener(
|
||||
newSettings.fetchChangesUpdateIntervalMs
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
|
|
@ -148,7 +146,7 @@ export class SyncClient {
|
|||
this.stop();
|
||||
await this._syncer.reset();
|
||||
this._history.reset();
|
||||
await this._database.resetSyncState();
|
||||
this._database.resetSyncState();
|
||||
this.logger.reset();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,15 +1,17 @@
|
|||
import type { Database, RelativePath } from "../persistence/database";
|
||||
|
||||
import type { SyncService } from "src/services/sync-service";
|
||||
import type { Logger } from "src/tracing/logger";
|
||||
import type { SyncHistory } from "src/tracing/sync-history";
|
||||
import type { SyncService } from "../services/sync-service";
|
||||
import type { Logger } from "../tracing/logger";
|
||||
import type { SyncHistory } from "../tracing/sync-history";
|
||||
import PQueue from "p-queue";
|
||||
import { hash } from "src/utils/hash";
|
||||
import type { components } from "src/services/types";
|
||||
import type { Settings } from "src/persistence/settings";
|
||||
import type { FileOperations } from "src/file-operations/file-operations";
|
||||
import { findMatchingFileBasedOnHash } from "src/utils/find-matching-file-based-on-hash";
|
||||
import { hash } from "../utils/hash";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import type { components } from "../services/types";
|
||||
import type { Settings } from "../persistence/settings";
|
||||
import type { FileOperations } from "../file-operations/file-operations";
|
||||
import { findMatchingFile } from "../utils/find-matching-file";
|
||||
import { UnrestrictedSyncer } from "./unrestricted-syncer";
|
||||
import { FileNotFoundError } from "../file-operations/safe-filesystem-operations";
|
||||
import { createPromise } from "../utils/create-promise";
|
||||
|
||||
export class Syncer {
|
||||
private readonly remainingOperationsListeners: ((
|
||||
|
|
@ -18,17 +20,15 @@ export class Syncer {
|
|||
|
||||
private readonly syncQueue: PQueue;
|
||||
|
||||
private runningScheduleSyncForOfflineChanges: Promise<void> | undefined =
|
||||
undefined;
|
||||
private runningApplyRemoteChangesLocally: Promise<void> | undefined =
|
||||
undefined;
|
||||
private runningScheduleSyncForOfflineChanges: Promise<void> | undefined;
|
||||
private runningApplyRemoteChangesLocally: Promise<void> | undefined;
|
||||
|
||||
private readonly internalSyncer: UnrestrictedSyncer;
|
||||
|
||||
public constructor(
|
||||
private readonly logger: Logger,
|
||||
private readonly database: Database,
|
||||
private readonly settings: Settings,
|
||||
settings: Settings,
|
||||
private readonly syncService: SyncService,
|
||||
private readonly operations: FileOperations,
|
||||
history: SyncHistory
|
||||
|
|
@ -45,7 +45,9 @@ export class Syncer {
|
|||
});
|
||||
|
||||
this.syncQueue.on("active", () => {
|
||||
this.emitRemainingOperationsChange(this.syncQueue.size);
|
||||
this.remainingOperationsListeners.forEach((listener) => {
|
||||
listener(this.syncQueue.size);
|
||||
});
|
||||
});
|
||||
|
||||
this.internalSyncer = new UnrestrictedSyncer(
|
||||
|
|
@ -65,48 +67,131 @@ export class Syncer {
|
|||
}
|
||||
|
||||
public async syncLocallyCreatedFile(
|
||||
relativePath: RelativePath,
|
||||
updateTime: Date
|
||||
relativePath: RelativePath
|
||||
): Promise<void> {
|
||||
await this.syncQueue.add(async () =>
|
||||
this.internalSyncer.unrestrictedSyncLocallyCreatedFile(
|
||||
relativePath,
|
||||
updateTime
|
||||
)
|
||||
);
|
||||
}
|
||||
if (
|
||||
this.database.getLatestDocumentByRelativePath(relativePath)
|
||||
?.isDeleted === false
|
||||
) {
|
||||
this.logger.debug(
|
||||
`Document ${relativePath} already exists in the database, skipping`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
public async syncLocallyUpdatedFile(args: {
|
||||
oldPath?: RelativePath;
|
||||
relativePath: RelativePath;
|
||||
updateTime: Date;
|
||||
}): Promise<void> {
|
||||
await this.syncQueue.add(async () =>
|
||||
this.internalSyncer.unrestrictedSyncLocallyUpdatedFile(args)
|
||||
);
|
||||
}
|
||||
const [promise, resolve, reject] = createPromise();
|
||||
|
||||
public async waitForSyncQueue(): Promise<void> {
|
||||
return this.syncQueue.onEmpty();
|
||||
const document = this.database.createNewPendingDocument(
|
||||
uuidv4(),
|
||||
relativePath,
|
||||
promise
|
||||
);
|
||||
|
||||
try {
|
||||
await this.syncQueue.add(async () =>
|
||||
this.internalSyncer.unrestrictedSyncLocallyCreatedFile(document)
|
||||
);
|
||||
|
||||
resolve();
|
||||
} catch (e) {
|
||||
reject(e);
|
||||
} finally {
|
||||
this.database.removeDocumentPromise(promise);
|
||||
}
|
||||
}
|
||||
|
||||
public async syncLocallyDeletedFile(
|
||||
relativePath: RelativePath
|
||||
): Promise<void> {
|
||||
await this.syncQueue.add(async () =>
|
||||
this.internalSyncer.unrestrictedSyncLocallyDeletedFile(relativePath)
|
||||
// We have to have a record of the delete in case there's an in-flight update for the same
|
||||
// document which finishes after the delete has succeeded and would introduce a phantom metadata record.
|
||||
this.database.delete(relativePath);
|
||||
|
||||
const [promise, resolve, reject] = createPromise();
|
||||
|
||||
const document = await this.database.getResolvedDocumentByRelativePath(
|
||||
relativePath,
|
||||
promise
|
||||
);
|
||||
|
||||
try {
|
||||
await this.syncQueue.add(async () =>
|
||||
this.internalSyncer.unrestrictedSyncLocallyDeletedFile(document)
|
||||
);
|
||||
|
||||
resolve();
|
||||
} catch (e) {
|
||||
reject(e);
|
||||
} finally {
|
||||
this.database.removeDocumentPromise(promise);
|
||||
}
|
||||
}
|
||||
|
||||
public async scheduleSyncForOfflineChanges(): Promise<void> {
|
||||
if (!this.settings.getSettings().isSyncEnabled) {
|
||||
public async syncLocallyUpdatedFile({
|
||||
oldPath,
|
||||
relativePath
|
||||
}: {
|
||||
oldPath?: RelativePath;
|
||||
relativePath: RelativePath;
|
||||
}): Promise<void> {
|
||||
if (
|
||||
oldPath !== undefined &&
|
||||
(this.database.getLatestDocumentByRelativePath(relativePath) ===
|
||||
undefined ||
|
||||
this.database.getLatestDocumentByRelativePath(relativePath)
|
||||
?.isDeleted === true)
|
||||
) {
|
||||
if (oldPath === relativePath) {
|
||||
throw new Error(
|
||||
`Old path and new path are the same: ${oldPath}`
|
||||
);
|
||||
}
|
||||
|
||||
this.database.move(oldPath, relativePath);
|
||||
}
|
||||
|
||||
let document =
|
||||
this.database.getLatestDocumentByRelativePath(relativePath);
|
||||
|
||||
if (document === undefined) {
|
||||
this.logger.debug(
|
||||
`Syncing is disabled, not uploading local changes`
|
||||
`Cannot find document ${relativePath} in the database, skipping`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.runningScheduleSyncForOfflineChanges != null) {
|
||||
if (document.isDeleted) {
|
||||
this.logger.debug(
|
||||
`Document ${relativePath} has been deleted locally, skipping`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const [promise, resolve, reject] = createPromise();
|
||||
|
||||
document = await this.database.getResolvedDocumentByRelativePath(
|
||||
relativePath,
|
||||
promise
|
||||
);
|
||||
|
||||
try {
|
||||
await this.syncQueue.add(async () =>
|
||||
this.internalSyncer.unrestrictedSyncLocallyUpdatedFile({
|
||||
oldPath,
|
||||
document
|
||||
})
|
||||
);
|
||||
|
||||
resolve();
|
||||
} catch (e) {
|
||||
reject(e);
|
||||
} finally {
|
||||
this.database.removeDocumentPromise(promise);
|
||||
}
|
||||
}
|
||||
|
||||
public async scheduleSyncForOfflineChanges(): Promise<void> {
|
||||
if (this.runningScheduleSyncForOfflineChanges !== undefined) {
|
||||
this.logger.debug("Uploading local changes is already in progress");
|
||||
return this.runningScheduleSyncForOfflineChanges;
|
||||
}
|
||||
|
|
@ -127,13 +212,6 @@ export class Syncer {
|
|||
}
|
||||
|
||||
public async applyRemoteChangesLocally(): Promise<void> {
|
||||
if (!this.settings.getSettings().isSyncEnabled) {
|
||||
this.logger.debug(
|
||||
`Syncing is disabled, not fetching remote changes`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.runningApplyRemoteChangesLocally != null) {
|
||||
this.logger.debug(
|
||||
"Applying remote changes locally is already in progress"
|
||||
|
|
@ -154,6 +232,10 @@ export class Syncer {
|
|||
}
|
||||
}
|
||||
|
||||
public async waitForSyncQueue(): Promise<void> {
|
||||
return this.syncQueue.onEmpty();
|
||||
}
|
||||
|
||||
public async reset(): Promise<void> {
|
||||
this.syncQueue.clear();
|
||||
await this.syncQueue.onEmpty();
|
||||
|
|
@ -163,115 +245,15 @@ export class Syncer {
|
|||
this.internalSyncer.reset();
|
||||
}
|
||||
|
||||
private async syncRemotelyUpdatedFile(
|
||||
remoteVersion: components["schemas"]["DocumentVersionWithoutContent"]
|
||||
): Promise<void> {
|
||||
await this.syncQueue.add(async () =>
|
||||
this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
|
||||
remoteVersion
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
private async internalScheduleSyncForOfflineChanges(): Promise<void> {
|
||||
const allLocalFiles = await this.operations.listAllFiles();
|
||||
|
||||
// This includes renamed files for now
|
||||
let locallyPossiblyDeletedFiles = [
|
||||
...this.database.getDocuments().entries()
|
||||
].filter(([path, _]) => !allLocalFiles.includes(path));
|
||||
|
||||
await Promise.all(
|
||||
allLocalFiles.map(async (relativePath) =>
|
||||
this.syncQueue.add(async () => {
|
||||
const metadata = this.database.getDocument(relativePath);
|
||||
|
||||
if (metadata) {
|
||||
this.logger.debug(
|
||||
`Document ${relativePath} might have been updated locally, scheduling sync to validate and update it`
|
||||
);
|
||||
return this.internalSyncer.unrestrictedSyncLocallyUpdatedFile(
|
||||
{
|
||||
relativePath,
|
||||
updateTime:
|
||||
await this.operations.getModificationTime(
|
||||
relativePath
|
||||
)
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
// Perhaps the file has been moved. Let's check by looking at the deleted files
|
||||
const contentBytes =
|
||||
await this.operations.read(relativePath);
|
||||
const contentHash = hash(contentBytes);
|
||||
|
||||
// todo: make this smarter so that offline files can be renamed & edited at the same time
|
||||
const originalFile = findMatchingFileBasedOnHash(
|
||||
contentHash,
|
||||
locallyPossiblyDeletedFiles
|
||||
);
|
||||
if (originalFile !== undefined) {
|
||||
// `originalFile` hasn't been deleted but it got moved instead
|
||||
locallyPossiblyDeletedFiles =
|
||||
locallyPossiblyDeletedFiles.filter(
|
||||
(item) => item[0] !== originalFile[0]
|
||||
);
|
||||
|
||||
this.logger.debug(
|
||||
`Document '${originalFile[0]}' was not found under its current path in the database but was found under a different path (${relativePath}), scheduling sync to move it`
|
||||
);
|
||||
return this.internalSyncer.unrestrictedSyncLocallyUpdatedFile(
|
||||
{
|
||||
oldPath: originalFile[0],
|
||||
relativePath: relativePath,
|
||||
updateTime:
|
||||
await this.operations.getModificationTime(
|
||||
relativePath
|
||||
),
|
||||
optimisations: {
|
||||
contentBytes,
|
||||
contentHash
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
this.logger.debug(
|
||||
`Document ${relativePath} not found in database, scheduling sync to create it`
|
||||
);
|
||||
return this.internalSyncer.unrestrictedSyncLocallyCreatedFile(
|
||||
relativePath,
|
||||
await this.operations.getModificationTime(relativePath)
|
||||
);
|
||||
})
|
||||
)
|
||||
);
|
||||
|
||||
await Promise.all(
|
||||
locallyPossiblyDeletedFiles.map(async ([relativePath, _]) => {
|
||||
this.logger.debug(
|
||||
`Document ${relativePath} has been deleted locally, scheduling sync to delete it`
|
||||
);
|
||||
|
||||
if (await this.operations.exists(relativePath)) {
|
||||
this.logger.debug(
|
||||
`Document ${relativePath} actually exists locally, skipping`
|
||||
);
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
// We're outside of the pqueue, so we need to call the public wrapper
|
||||
return this.syncLocallyDeletedFile(relativePath);
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
private async internalApplyRemoteChangesLocally(): Promise<void> {
|
||||
const remote = await this.syncService.getAll(
|
||||
this.database.getLastSeenUpdateId()
|
||||
const remote = await this.syncQueue.add(async () =>
|
||||
this.syncService.getAll(this.database.getLastSeenUpdateId())
|
||||
);
|
||||
|
||||
if (!remote) {
|
||||
throw new Error("Failed to fetch remote changes");
|
||||
}
|
||||
|
||||
if (remote.latestDocuments.length === 0) {
|
||||
this.logger.debug("No remote changes to apply");
|
||||
return;
|
||||
|
|
@ -280,9 +262,7 @@ export class Syncer {
|
|||
this.logger.info("Applying remote changes locally");
|
||||
|
||||
await Promise.all(
|
||||
remote.latestDocuments.map(async (remoteDocument) =>
|
||||
this.syncRemotelyUpdatedFile(remoteDocument)
|
||||
)
|
||||
remote.latestDocuments.map(this.syncRemotelyUpdatedFile.bind(this))
|
||||
);
|
||||
|
||||
const lastSeenUpdateId = this.database.getLastSeenUpdateId();
|
||||
|
|
@ -290,13 +270,124 @@ export class Syncer {
|
|||
lastSeenUpdateId === undefined ||
|
||||
remote.lastUpdateId > lastSeenUpdateId
|
||||
) {
|
||||
await this.database.setLastSeenUpdateId(remote.lastUpdateId);
|
||||
this.database.setLastSeenUpdateId(remote.lastUpdateId);
|
||||
}
|
||||
}
|
||||
|
||||
private emitRemainingOperationsChange(remainingOperations: number): void {
|
||||
this.remainingOperationsListeners.forEach((listener) => {
|
||||
listener(remainingOperations);
|
||||
});
|
||||
private async syncRemotelyUpdatedFile(
|
||||
remoteVersion: components["schemas"]["DocumentVersionWithoutContent"]
|
||||
): Promise<void> {
|
||||
let document = this.database.getDocumentByDocumentId(
|
||||
remoteVersion.documentId
|
||||
);
|
||||
|
||||
const [promise, resolve, reject] = createPromise();
|
||||
|
||||
if (document === undefined) {
|
||||
await this.syncQueue.add(async () =>
|
||||
this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
|
||||
remoteVersion
|
||||
)
|
||||
);
|
||||
} else {
|
||||
document = await this.database.getResolvedDocumentByRelativePath(
|
||||
document.relativePath,
|
||||
promise
|
||||
);
|
||||
|
||||
try {
|
||||
await this.syncQueue.add(async () =>
|
||||
this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
|
||||
remoteVersion,
|
||||
document
|
||||
)
|
||||
);
|
||||
|
||||
resolve();
|
||||
} catch (e) {
|
||||
reject(e);
|
||||
} finally {
|
||||
this.database.removeDocumentPromise(promise);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async internalScheduleSyncForOfflineChanges(): Promise<void> {
|
||||
const allLocalFiles = await this.operations.listAllFiles();
|
||||
|
||||
let locallyPossiblyDeletedFiles = [
|
||||
...this.database.resolvedDocuments
|
||||
].filter(({ relativePath }) => !allLocalFiles.includes(relativePath));
|
||||
|
||||
const updates = Promise.all(
|
||||
allLocalFiles.map(async (relativePath) => {
|
||||
if (
|
||||
this.database.getLatestDocumentByRelativePath(relativePath)
|
||||
?.metadata !== undefined
|
||||
) {
|
||||
this.logger.debug(
|
||||
`Document ${relativePath} might have been updated locally, scheduling sync to validate and update it`
|
||||
);
|
||||
|
||||
return this.syncLocallyUpdatedFile({
|
||||
relativePath
|
||||
});
|
||||
}
|
||||
|
||||
// Perhaps the file has been moved; let's check by looking at the deleted files
|
||||
const contentHash = await this.syncQueue.add(async () => {
|
||||
const contentBytes =
|
||||
await this.operations.read(relativePath); // this can throw FileNotFoundError
|
||||
return hash(contentBytes);
|
||||
});
|
||||
|
||||
if (contentHash == undefined) {
|
||||
// The file was deleted before we had a chance to read it, no need to sync it here
|
||||
return;
|
||||
}
|
||||
|
||||
const originalFile = findMatchingFile(
|
||||
contentHash,
|
||||
locallyPossiblyDeletedFiles
|
||||
);
|
||||
if (originalFile !== undefined) {
|
||||
// `originalFile` hasn't been deleted but it got moved instead
|
||||
locallyPossiblyDeletedFiles =
|
||||
locallyPossiblyDeletedFiles.filter(
|
||||
(item) =>
|
||||
item.relativePath !== originalFile.relativePath
|
||||
);
|
||||
|
||||
this.logger.debug(
|
||||
`Document '${originalFile.relativePath}' was not found under its current path in the database but was found under a different path (${relativePath}), scheduling sync to move it`
|
||||
);
|
||||
|
||||
// We're outside of the pqueue, so we need to call the public wrapper
|
||||
return this.syncLocallyUpdatedFile({
|
||||
oldPath: originalFile.relativePath,
|
||||
relativePath
|
||||
});
|
||||
}
|
||||
|
||||
this.logger.debug(
|
||||
`Document ${relativePath} not found in database, scheduling sync to create it`
|
||||
);
|
||||
// We're outside of the pqueue, so we need to call the public wrapper
|
||||
return this.syncLocallyCreatedFile(relativePath);
|
||||
})
|
||||
);
|
||||
|
||||
const deletes = Promise.all(
|
||||
locallyPossiblyDeletedFiles.map(async ({ relativePath }) => {
|
||||
this.logger.debug(
|
||||
`Document ${relativePath} has been deleted locally, scheduling sync to delete it`
|
||||
);
|
||||
|
||||
// We're outside of the pqueue, so we need to call the public wrapper
|
||||
return this.syncLocallyDeletedFile(relativePath);
|
||||
})
|
||||
);
|
||||
|
||||
await Promise.all([updates, deletes]);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,19 +1,24 @@
|
|||
import type { Database, RelativePath } from "../persistence/database";
|
||||
import type {
|
||||
Database,
|
||||
DocumentRecord,
|
||||
RelativePath
|
||||
} from "../persistence/database";
|
||||
|
||||
import type { SyncService } from "src/services/sync-service";
|
||||
import type { Logger } from "src/tracing/logger";
|
||||
import type { SyncHistory } from "src/tracing/sync-history";
|
||||
import { SyncSource, SyncStatus, SyncType } from "src/tracing/sync-history";
|
||||
import { hash } from "src/utils/hash";
|
||||
import type { components } from "src/services/types";
|
||||
import { deserialize } from "src/utils/deserialize";
|
||||
import type { Settings } from "src/persistence/settings";
|
||||
import type { FileOperations } from "src/file-operations/file-operations";
|
||||
import { FileNotFoundError } from "src/file-operations/safe-filesystem-operations";
|
||||
import { DocumentLocks } from "./document-locks";
|
||||
import type { SyncService } from "../services/sync-service";
|
||||
import type { Logger } from "../tracing/logger";
|
||||
import type { SyncHistory } from "../tracing/sync-history";
|
||||
import { SyncSource, SyncStatus, SyncType } from "../tracing/sync-history";
|
||||
import { EMPTY_HASH, hash } from "../utils/hash";
|
||||
import type { components } from "../services/types";
|
||||
import { deserialize } from "../utils/deserialize";
|
||||
import type { Settings } from "../persistence/settings";
|
||||
import type { FileOperations } from "../file-operations/file-operations";
|
||||
import { FileNotFoundError } from "../file-operations/safe-filesystem-operations";
|
||||
import { DocumentLocks } from "../file-operations/document-locks";
|
||||
import { createPromise } from "../utils/create-promise";
|
||||
|
||||
export class UnrestrictedSyncer {
|
||||
private readonly locks = new DocumentLocks();
|
||||
private readonly locks: DocumentLocks;
|
||||
|
||||
public constructor(
|
||||
private readonly logger: Logger,
|
||||
|
|
@ -22,507 +27,375 @@ export class UnrestrictedSyncer {
|
|||
private readonly syncService: SyncService,
|
||||
private readonly operations: FileOperations,
|
||||
private readonly history: SyncHistory
|
||||
) {}
|
||||
) {
|
||||
this.locks = new DocumentLocks(logger);
|
||||
}
|
||||
|
||||
public async unrestrictedSyncLocallyCreatedFile(
|
||||
relativePath: RelativePath,
|
||||
updateTime: Date,
|
||||
optimisations?: {
|
||||
contentBytes?: Uint8Array;
|
||||
contentHash?: string;
|
||||
}
|
||||
document: DocumentRecord
|
||||
): Promise<void> {
|
||||
await this.executeWhileHoldingFileLock(
|
||||
[relativePath],
|
||||
return this.executeSync(
|
||||
document.relativePath,
|
||||
SyncType.CREATE,
|
||||
SyncSource.PUSH,
|
||||
async () => {
|
||||
if (
|
||||
(await this.operations.getFileSize(relativePath)) / // this can throw FileNotFoundError
|
||||
1024 /
|
||||
1024 >
|
||||
this.settings.getSettings().maxFileSizeMB
|
||||
) {
|
||||
this.history.addHistoryEntry({
|
||||
status: SyncStatus.ERROR,
|
||||
relativePath,
|
||||
message: `File size exceeds the maximum file size limit of ${
|
||||
this.settings.getSettings().maxFileSizeMB
|
||||
}MB`,
|
||||
type: SyncType.CREATE
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const contentBytes =
|
||||
optimisations?.contentBytes ??
|
||||
(await this.operations.read(relativePath)); // this can throw FileNotFoundError
|
||||
let contentHash =
|
||||
optimisations?.contentHash ?? hash(contentBytes);
|
||||
|
||||
const localMetadata = this.database.getDocument(relativePath);
|
||||
if (localMetadata) {
|
||||
this.logger.debug(
|
||||
`Document metadata already exists for ${relativePath}, it must have been downloaded from the server`
|
||||
);
|
||||
|
||||
if (localMetadata.hash === contentHash) {
|
||||
this.history.addHistoryEntry({
|
||||
status: SyncStatus.NO_OP,
|
||||
relativePath,
|
||||
message: `File hash matches with last synced version, no need to sync`,
|
||||
type: SyncType.UPDATE
|
||||
});
|
||||
return;
|
||||
}
|
||||
}
|
||||
const contentBytes = await this.operations.read(
|
||||
document.relativePath
|
||||
); // this can throw FileNotFoundError
|
||||
const contentHash = hash(contentBytes);
|
||||
|
||||
const response = await this.syncService.create({
|
||||
relativePath,
|
||||
contentBytes,
|
||||
createdDate: updateTime
|
||||
documentId: document.documentId,
|
||||
relativePath: document.relativePath,
|
||||
contentBytes
|
||||
});
|
||||
|
||||
this.history.addHistoryEntry({
|
||||
status: SyncStatus.SUCCESS,
|
||||
source: SyncSource.PUSH,
|
||||
relativePath,
|
||||
relativePath: document.relativePath,
|
||||
message: `Successfully uploaded locally created file`,
|
||||
type: SyncType.CREATE
|
||||
});
|
||||
|
||||
// The response can't have a different relative path than the one we sent
|
||||
// because the relative path is the key when finding existing documents
|
||||
// when a create request is sent.
|
||||
this.database.updateDocumentMetadata(
|
||||
{
|
||||
parentVersionId: response.vaultUpdateId,
|
||||
hash: contentHash
|
||||
},
|
||||
document
|
||||
);
|
||||
|
||||
if (response.type === "MergingUpdate") {
|
||||
const responseBytes = deserialize(response.contentBase64);
|
||||
contentHash = hash(responseBytes);
|
||||
this.tryIncrementVaultUpdateId(response.vaultUpdateId);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
await this.operations.write(
|
||||
relativePath,
|
||||
contentBytes,
|
||||
responseBytes
|
||||
);
|
||||
this.history.addHistoryEntry({
|
||||
status: SyncStatus.SUCCESS,
|
||||
source: SyncSource.PULL,
|
||||
relativePath,
|
||||
message: `The file we created locally has already existed remotely, so we have merged them`,
|
||||
type: SyncType.UPDATE
|
||||
});
|
||||
}
|
||||
|
||||
await this.database.setDocument({
|
||||
documentId: response.documentId,
|
||||
relativePath: response.relativePath,
|
||||
parentVersionId: response.vaultUpdateId,
|
||||
hash: contentHash
|
||||
public async unrestrictedSyncLocallyDeletedFile(
|
||||
document: DocumentRecord
|
||||
): Promise<void> {
|
||||
await this.executeSync(
|
||||
document.relativePath,
|
||||
SyncType.DELETE,
|
||||
SyncSource.PUSH,
|
||||
async () => {
|
||||
const response = await this.syncService.delete({
|
||||
documentId: document.documentId,
|
||||
relativePath: document.relativePath
|
||||
});
|
||||
|
||||
await this.tryIncrementVaultUpdateId(response.vaultUpdateId);
|
||||
this.history.addHistoryEntry({
|
||||
status: SyncStatus.SUCCESS,
|
||||
source: SyncSource.PUSH,
|
||||
relativePath: document.relativePath,
|
||||
message: `Successfully deleted locally deleted file on the remote server`,
|
||||
type: SyncType.DELETE
|
||||
});
|
||||
|
||||
this.database.updateDocumentMetadata(
|
||||
{
|
||||
parentVersionId: response.vaultUpdateId,
|
||||
hash: EMPTY_HASH
|
||||
},
|
||||
document
|
||||
);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
public async unrestrictedSyncLocallyUpdatedFile({
|
||||
oldPath,
|
||||
relativePath,
|
||||
updateTime,
|
||||
optimisations
|
||||
document,
|
||||
force = false
|
||||
}: {
|
||||
oldPath?: RelativePath;
|
||||
relativePath: RelativePath;
|
||||
updateTime: Date;
|
||||
optimisations?: {
|
||||
contentBytes?: Uint8Array;
|
||||
contentHash?: string;
|
||||
};
|
||||
force?: boolean;
|
||||
document: DocumentRecord;
|
||||
}): Promise<void> {
|
||||
await this.executeWhileHoldingFileLock(
|
||||
[oldPath, relativePath].filter((path) => path !== undefined),
|
||||
await this.executeSync(
|
||||
document.relativePath,
|
||||
SyncType.UPDATE,
|
||||
SyncSource.PUSH,
|
||||
async () => {
|
||||
// Check the new path first in case the metadata has been already moved
|
||||
let localMetadata = this.database.getDocument(relativePath);
|
||||
let metadataPath = relativePath;
|
||||
const originalRelativePath = document.relativePath;
|
||||
|
||||
if (localMetadata === undefined && oldPath !== undefined) {
|
||||
localMetadata = this.database.getDocument(oldPath);
|
||||
metadataPath = oldPath;
|
||||
}
|
||||
|
||||
if (!localMetadata) {
|
||||
// It's fine, a subsequent sync operation must have dealt with this
|
||||
if (document.metadata === undefined || document.isDeleted) {
|
||||
this.logger.debug(
|
||||
`Document ${document.relativePath} has been already deleted, no need to update it`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
(await this.operations.getFileSize(relativePath)) / // this can throw FileNotFoundError
|
||||
1024 /
|
||||
1024 >
|
||||
this.settings.getSettings().maxFileSizeMB
|
||||
) {
|
||||
this.history.addHistoryEntry({
|
||||
status: SyncStatus.ERROR,
|
||||
relativePath,
|
||||
message: `File size exceeds the maximum file size limit of ${
|
||||
this.settings.getSettings().maxFileSizeMB
|
||||
}MB`,
|
||||
type: SyncType.CREATE
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const contentBytes =
|
||||
optimisations?.contentBytes ??
|
||||
(await this.operations.read(relativePath)); // this can throw FileNotFoundError
|
||||
|
||||
let contentHash =
|
||||
optimisations?.contentHash ?? hash(contentBytes);
|
||||
const contentBytes = await this.operations.read(
|
||||
document.relativePath
|
||||
); // this can throw FileNotFoundError
|
||||
let contentHash = hash(contentBytes);
|
||||
|
||||
if (
|
||||
localMetadata.hash === contentHash &&
|
||||
oldPath === undefined
|
||||
document.metadata.hash === contentHash &&
|
||||
oldPath === undefined &&
|
||||
!force
|
||||
) {
|
||||
this.history.addHistoryEntry({
|
||||
status: SyncStatus.NO_OP,
|
||||
relativePath,
|
||||
message: `File hash matches with last synced version, no need to sync`,
|
||||
type: SyncType.UPDATE
|
||||
});
|
||||
this.logger.debug(
|
||||
`File hash of ${document.relativePath} matches with last synced version and the path hasn't changed; no need to sync`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const response = await this.syncService.put({
|
||||
documentId: localMetadata.documentId,
|
||||
parentVersionId: localMetadata.parentVersionId,
|
||||
relativePath,
|
||||
contentBytes,
|
||||
createdDate: updateTime
|
||||
documentId: document.documentId,
|
||||
parentVersionId: document.metadata.parentVersionId,
|
||||
relativePath: document.relativePath,
|
||||
contentBytes
|
||||
});
|
||||
|
||||
// `document` is mutable and reflects the latest state in the local database
|
||||
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
||||
if (document.isDeleted) {
|
||||
this.logger.info(
|
||||
`Document ${document.relativePath} has been deleted before we could finish updating it`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
||||
if (document.metadata === undefined) {
|
||||
throw new Error(
|
||||
`Document ${document.relativePath} no longer has metadata after updating it, this cannot happen`
|
||||
);
|
||||
}
|
||||
|
||||
if (
|
||||
document.metadata.parentVersionId >= response.vaultUpdateId
|
||||
) {
|
||||
this.logger.debug(
|
||||
`Document ${document.relativePath} is already more up to date than the fetched version`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
this.history.addHistoryEntry({
|
||||
status: SyncStatus.SUCCESS,
|
||||
source: SyncSource.PUSH,
|
||||
relativePath,
|
||||
relativePath: document.relativePath,
|
||||
message: `Successfully uploaded locally updated file to the remote server`,
|
||||
type: SyncType.UPDATE
|
||||
});
|
||||
|
||||
if (response.isDeleted) {
|
||||
await this.operations.remove(oldPath ?? relativePath);
|
||||
await this.database.removeDocument(oldPath ?? relativePath);
|
||||
await this.tryIncrementVaultUpdateId(
|
||||
response.vaultUpdateId
|
||||
);
|
||||
|
||||
this.history.addHistoryEntry({
|
||||
status: SyncStatus.SUCCESS,
|
||||
source: SyncSource.PULL,
|
||||
relativePath,
|
||||
relativePath: document.relativePath,
|
||||
message:
|
||||
"The file we tried to update had been deleted remotely, therefore, we have deleted it locally",
|
||||
type: SyncType.DELETE
|
||||
});
|
||||
|
||||
this.database.delete(document.relativePath);
|
||||
this.database.updateDocumentMetadata(
|
||||
{
|
||||
parentVersionId: response.vaultUpdateId,
|
||||
hash: EMPTY_HASH
|
||||
},
|
||||
document
|
||||
);
|
||||
|
||||
await this.operations.delete(document.relativePath);
|
||||
|
||||
this.tryIncrementVaultUpdateId(response.vaultUpdateId);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
response.relativePath != relativePath &&
|
||||
response.relativePath != oldPath
|
||||
) {
|
||||
await this.locks.waitForDocumentLock(response.relativePath);
|
||||
let actualPath = document.relativePath;
|
||||
|
||||
if (response.relativePath != originalRelativePath) {
|
||||
actualPath = response.relativePath;
|
||||
await this.operations.move(
|
||||
document.relativePath,
|
||||
response.relativePath
|
||||
); // this can throw FileNotFoundError
|
||||
}
|
||||
|
||||
try {
|
||||
if (response.relativePath != relativePath) {
|
||||
// TODO: this can fail, that's bad
|
||||
await this.operations.move(
|
||||
// this can throw FileNotFoundError
|
||||
relativePath,
|
||||
response.relativePath,
|
||||
response.documentId
|
||||
);
|
||||
}
|
||||
|
||||
if (response.type === "MergingUpdate") {
|
||||
const responseBytes = deserialize(
|
||||
response.contentBase64
|
||||
);
|
||||
contentHash = hash(responseBytes);
|
||||
|
||||
await this.operations.write(
|
||||
response.relativePath,
|
||||
contentBytes,
|
||||
responseBytes
|
||||
);
|
||||
|
||||
this.history.addHistoryEntry({
|
||||
status: SyncStatus.SUCCESS,
|
||||
source: SyncSource.PULL,
|
||||
relativePath,
|
||||
message: `The file we updated had been updated remotely, so we downloaded the merged version`,
|
||||
type: SyncType.UPDATE
|
||||
});
|
||||
}
|
||||
|
||||
if (metadataPath !== response.relativePath) {
|
||||
await this.database.updatePath(
|
||||
metadataPath,
|
||||
response.relativePath
|
||||
);
|
||||
}
|
||||
await this.database.setDocument({
|
||||
documentId: localMetadata.documentId,
|
||||
relativePath: response.relativePath,
|
||||
this.database.updateDocumentMetadata(
|
||||
{
|
||||
parentVersionId: response.vaultUpdateId,
|
||||
hash: contentHash
|
||||
});
|
||||
},
|
||||
document
|
||||
);
|
||||
|
||||
await this.tryIncrementVaultUpdateId(
|
||||
response.vaultUpdateId
|
||||
if (response.type === "MergingUpdate") {
|
||||
const responseBytes = deserialize(response.contentBase64);
|
||||
contentHash = hash(responseBytes);
|
||||
|
||||
await this.operations.write(
|
||||
actualPath,
|
||||
contentBytes,
|
||||
responseBytes
|
||||
);
|
||||
} finally {
|
||||
if (
|
||||
response.relativePath != relativePath &&
|
||||
response.relativePath != oldPath
|
||||
) {
|
||||
this.locks.unlockDocument(response.relativePath);
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
public async unrestrictedSyncLocallyDeletedFile(
|
||||
relativePath: RelativePath
|
||||
): Promise<void> {
|
||||
await this.executeWhileHoldingFileLock(
|
||||
[relativePath],
|
||||
SyncType.DELETE,
|
||||
SyncSource.PUSH,
|
||||
async () => {
|
||||
const localMetadata = this.database.getDocument(relativePath);
|
||||
if (!localMetadata) {
|
||||
this.history.addHistoryEntry({
|
||||
status: SyncStatus.NO_OP,
|
||||
relativePath,
|
||||
message: `Locally deleted file hasn't been uploaded yet, so there's no need to delete it on the remote server`,
|
||||
type: SyncType.DELETE
|
||||
status: SyncStatus.SUCCESS,
|
||||
source: SyncSource.PULL,
|
||||
relativePath: document.relativePath,
|
||||
message: `The file we updated had been updated remotely, so we downloaded the merged version`,
|
||||
type: SyncType.UPDATE
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
await this.syncService.delete({
|
||||
documentId: localMetadata.documentId,
|
||||
relativePath,
|
||||
createdDate: new Date() // We got the event now, so it must have been deleted just now
|
||||
});
|
||||
|
||||
this.history.addHistoryEntry({
|
||||
status: SyncStatus.SUCCESS,
|
||||
source: SyncSource.PUSH,
|
||||
relativePath,
|
||||
message: `Successfully deleted locally deleted file on the remote server`,
|
||||
type: SyncType.DELETE
|
||||
});
|
||||
|
||||
await this.database.removeDocument(relativePath);
|
||||
this.tryIncrementVaultUpdateId(response.vaultUpdateId);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
public async unrestrictedSyncRemotelyUpdatedFile(
|
||||
remoteVersion: components["schemas"]["DocumentVersionWithoutContent"]
|
||||
remoteVersion: components["schemas"]["DocumentVersionWithoutContent"],
|
||||
document?: DocumentRecord
|
||||
): Promise<void> {
|
||||
await this.executeWhileHoldingFileLock(
|
||||
[remoteVersion.relativePath],
|
||||
await this.executeSync(
|
||||
remoteVersion.relativePath,
|
||||
SyncType.UPDATE,
|
||||
SyncSource.PULL,
|
||||
async () => {
|
||||
let localMetadata = this.database.getDocumentByDocumentId(
|
||||
remoteVersion.documentId
|
||||
);
|
||||
|
||||
if (
|
||||
localMetadata &&
|
||||
localMetadata[0] !== remoteVersion.relativePath
|
||||
) {
|
||||
await this.locks.waitForDocumentLock(localMetadata[0]);
|
||||
}
|
||||
// Waiting for the new lock might take a while so we need to fetch the database
|
||||
// entry again in case it's changed.
|
||||
localMetadata = this.database.getDocumentByDocumentId(
|
||||
remoteVersion.documentId
|
||||
);
|
||||
|
||||
if (!localMetadata) {
|
||||
if (remoteVersion.isDeleted) {
|
||||
this.history.addHistoryEntry({
|
||||
status: SyncStatus.NO_OP,
|
||||
source: SyncSource.PULL,
|
||||
relativePath: remoteVersion.relativePath,
|
||||
message: `Remotely deleted file hasn't been synced yet, so there's no need to delete it locally`,
|
||||
type: SyncType.DELETE
|
||||
});
|
||||
if (document?.metadata !== undefined) {
|
||||
// If the file exists locally, let's pretend the user has updated it
|
||||
// and deal with remote update/deletion within `unrestrictedSyncLocallyUpdatedFile`
|
||||
if (
|
||||
document.metadata.parentVersionId >=
|
||||
remoteVersion.vaultUpdateId
|
||||
) {
|
||||
this.logger.debug(
|
||||
`Document ${remoteVersion.relativePath} is already more up to date than the fetched version`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const content = (
|
||||
await this.syncService.get({
|
||||
documentId: remoteVersion.documentId
|
||||
})
|
||||
).contentBase64;
|
||||
const contentBytes = deserialize(content);
|
||||
|
||||
await this.operations.create(
|
||||
remoteVersion.relativePath,
|
||||
contentBytes
|
||||
return this.unrestrictedSyncLocallyUpdatedFile({
|
||||
document,
|
||||
force: true
|
||||
});
|
||||
} else if (remoteVersion.isDeleted) {
|
||||
// Either the doc hasn't made it to us before and therefore we don't need to delete it,
|
||||
// or we already have it, in which case the preceeding if will deal with it
|
||||
this.logger.debug(
|
||||
`Document ${remoteVersion.relativePath} has been deleted remotely, no need to sync`
|
||||
);
|
||||
await this.database.setDocument({
|
||||
documentId: remoteVersion.documentId,
|
||||
relativePath: remoteVersion.relativePath,
|
||||
return;
|
||||
}
|
||||
|
||||
const content = (
|
||||
await this.syncService.get({
|
||||
documentId: remoteVersion.documentId
|
||||
})
|
||||
).contentBase64;
|
||||
|
||||
document = this.database.getDocumentByDocumentId(
|
||||
remoteVersion.documentId
|
||||
);
|
||||
|
||||
if (document?.isDeleted === true) {
|
||||
this.logger.info(
|
||||
`Document ${remoteVersion.relativePath} has been deleted locally before we could finish updating it`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
(document?.metadata?.parentVersionId ?? -1) >=
|
||||
remoteVersion.vaultUpdateId
|
||||
) {
|
||||
this.logger.debug(
|
||||
`Document ${remoteVersion.relativePath} is already more up to date than the fetched version`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const contentBytes = deserialize(content);
|
||||
|
||||
await this.operations.ensureClearPath(
|
||||
remoteVersion.relativePath
|
||||
);
|
||||
|
||||
const [promise, resolve] = createPromise();
|
||||
this.database.updateDocumentMetadata(
|
||||
{
|
||||
parentVersionId: remoteVersion.vaultUpdateId,
|
||||
hash: hash(contentBytes)
|
||||
});
|
||||
this.history.addHistoryEntry({
|
||||
status: SyncStatus.SUCCESS,
|
||||
source: SyncSource.PULL,
|
||||
relativePath: remoteVersion.relativePath,
|
||||
message: `Successfully downloaded remote file which hadn't existed locally`,
|
||||
type: SyncType.CREATE
|
||||
});
|
||||
return;
|
||||
}
|
||||
},
|
||||
this.database.createNewPendingDocument(
|
||||
remoteVersion.documentId,
|
||||
remoteVersion.relativePath,
|
||||
promise
|
||||
)
|
||||
);
|
||||
|
||||
const [relativePath, metadata] = localMetadata;
|
||||
await this.operations.create(
|
||||
remoteVersion.relativePath,
|
||||
contentBytes
|
||||
);
|
||||
|
||||
if (remoteVersion.vaultUpdateId <= metadata.parentVersionId) {
|
||||
this.logger.debug(
|
||||
`Document ${relativePath} is already up to date`
|
||||
);
|
||||
return;
|
||||
}
|
||||
resolve();
|
||||
this.database.removeDocumentPromise(promise);
|
||||
|
||||
try {
|
||||
if (remoteVersion.isDeleted) {
|
||||
await this.operations.remove(relativePath);
|
||||
await this.database.removeDocument(relativePath);
|
||||
|
||||
this.history.addHistoryEntry({
|
||||
status: SyncStatus.SUCCESS,
|
||||
source: SyncSource.PULL,
|
||||
relativePath: remoteVersion.relativePath,
|
||||
message: `Successfully deleted remotely deleted file locally`,
|
||||
type: SyncType.DELETE
|
||||
});
|
||||
} else {
|
||||
// TODO: this can fail, that's bad
|
||||
const currentContent =
|
||||
await this.operations.read(relativePath); // this can throw FileNotFoundError
|
||||
const currentHash = hash(currentContent);
|
||||
|
||||
if (currentHash !== metadata.hash) {
|
||||
this.logger.info(
|
||||
`Document ${relativePath} has been updated both remotely and locally, letting the local file update event handle it`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const content = (
|
||||
await this.syncService.get({
|
||||
documentId: remoteVersion.documentId
|
||||
})
|
||||
).contentBase64;
|
||||
const contentBytes = deserialize(content);
|
||||
const contentHash = hash(contentBytes);
|
||||
|
||||
if (relativePath !== remoteVersion.relativePath) {
|
||||
// TODO: this can fail, that's bad
|
||||
await this.operations.move(
|
||||
// this can throw FileNotFoundError
|
||||
relativePath,
|
||||
remoteVersion.relativePath,
|
||||
remoteVersion.documentId
|
||||
);
|
||||
|
||||
await this.database.updatePath(
|
||||
relativePath,
|
||||
remoteVersion.relativePath
|
||||
);
|
||||
}
|
||||
|
||||
await this.operations.write(
|
||||
remoteVersion.relativePath,
|
||||
currentContent,
|
||||
contentBytes
|
||||
);
|
||||
await this.database.setDocument({
|
||||
documentId: remoteVersion.documentId,
|
||||
relativePath: remoteVersion.relativePath,
|
||||
parentVersionId: remoteVersion.vaultUpdateId,
|
||||
hash: contentHash
|
||||
});
|
||||
|
||||
this.history.addHistoryEntry({
|
||||
status: SyncStatus.SUCCESS,
|
||||
source: SyncSource.PULL,
|
||||
relativePath: remoteVersion.relativePath,
|
||||
message: `Successfully updated remotely updated file locally`,
|
||||
type: SyncType.UPDATE
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
if (relativePath !== remoteVersion.relativePath) {
|
||||
this.locks.unlockDocument(relativePath);
|
||||
}
|
||||
}
|
||||
this.history.addHistoryEntry({
|
||||
status: SyncStatus.SUCCESS,
|
||||
source: SyncSource.PULL,
|
||||
relativePath: remoteVersion.relativePath,
|
||||
message: `Successfully downloaded remote file which hadn't existed locally`,
|
||||
type: SyncType.CREATE
|
||||
});
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
public async executeWhileHoldingFileLock(
|
||||
lockedPaths: RelativePath[],
|
||||
public async executeSync<T>(
|
||||
relativePath: RelativePath,
|
||||
syncType: SyncType,
|
||||
syncSource: SyncSource,
|
||||
fn: () => Promise<void>
|
||||
): Promise<void> {
|
||||
const relativePath = lockedPaths[lockedPaths.length - 1];
|
||||
|
||||
if (!this.settings.getSettings().isSyncEnabled) {
|
||||
this.logger.info(
|
||||
`Syncing is disabled, not syncing ${relativePath}`
|
||||
);
|
||||
return;
|
||||
}
|
||||
fn: () => Promise<T>
|
||||
): Promise<T | undefined> {
|
||||
if (!this.operations.isFileEligibleForSync(relativePath)) {
|
||||
this.logger.info(
|
||||
`File ${relativePath} is not eligible for syncing`
|
||||
);
|
||||
this.history.addHistoryEntry({
|
||||
status: SyncStatus.ERROR,
|
||||
relativePath,
|
||||
message: `File ${relativePath} is not eligible for syncing`,
|
||||
type: syncType
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
this.logger.debug(
|
||||
`Syncing ${relativePath} (${syncSource} - ${syncType})`
|
||||
);
|
||||
|
||||
await Promise.all(
|
||||
lockedPaths.map(this.locks.waitForDocumentLock.bind(this.locks))
|
||||
);
|
||||
try {
|
||||
await fn();
|
||||
if (
|
||||
(await this.operations.exists(relativePath)) &&
|
||||
(await this.operations.getFileSize(relativePath)) / // this can throw FileNotFoundError
|
||||
1024 /
|
||||
1024 >
|
||||
this.settings.getSettings().maxFileSizeMB
|
||||
) {
|
||||
this.history.addHistoryEntry({
|
||||
status: SyncStatus.ERROR,
|
||||
relativePath,
|
||||
message: `File size exceeds the maximum file size limit of ${
|
||||
this.settings.getSettings().maxFileSizeMB
|
||||
}MB`,
|
||||
type: syncType
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
return await fn();
|
||||
} catch (e) {
|
||||
if (e instanceof FileNotFoundError) {
|
||||
// A subsequent sync operation must have been creating to deal with this
|
||||
this.history.addHistoryEntry({
|
||||
status: SyncStatus.NO_OP,
|
||||
relativePath,
|
||||
message: `Skip ${syncSource.toLocaleLowerCase()} file because it no longer exists when trying to ${syncType.toLocaleLowerCase()} it`,
|
||||
type: syncType,
|
||||
source: syncSource
|
||||
});
|
||||
this.logger.info(
|
||||
`Skip ${syncSource.toLocaleLowerCase()} file because it no longer exists when trying to ${syncType.toLocaleLowerCase()} it`
|
||||
);
|
||||
} else {
|
||||
this.history.addHistoryEntry({
|
||||
status: SyncStatus.ERROR,
|
||||
|
|
@ -533,8 +406,6 @@ export class UnrestrictedSyncer {
|
|||
});
|
||||
throw e;
|
||||
}
|
||||
} finally {
|
||||
lockedPaths.forEach(this.locks.unlockDocument.bind(this.locks));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -542,11 +413,9 @@ export class UnrestrictedSyncer {
|
|||
this.locks.reset();
|
||||
}
|
||||
|
||||
private async tryIncrementVaultUpdateId(
|
||||
responseVaultUpdateId: number
|
||||
): Promise<void> {
|
||||
private tryIncrementVaultUpdateId(responseVaultUpdateId: number): void {
|
||||
if (this.database.getLastSeenUpdateId() === responseVaultUpdateId - 1) {
|
||||
await this.database.setLastSeenUpdateId(responseVaultUpdateId);
|
||||
this.database.setLastSeenUpdateId(responseVaultUpdateId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import type { RelativePath } from "src/persistence/database";
|
||||
import type { RelativePath } from "../persistence/database";
|
||||
import type { Logger } from "./logger";
|
||||
|
||||
export interface CommonHistoryEntry {
|
||||
|
|
|
|||
15
frontend/sync-client/src/utils/create-promise.ts
Normal file
15
frontend/sync-client/src/utils/create-promise.ts
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
export function createPromise<T = void>(): [
|
||||
Promise<T>,
|
||||
(value: T) => void,
|
||||
(error: unknown) => void
|
||||
] {
|
||||
let resolve: undefined | ((resolved: T) => void) = undefined;
|
||||
let reject: undefined | ((error: unknown) => void) = undefined;
|
||||
|
||||
const creationPromise = new Promise<T>(
|
||||
(resolve_, reject_) => ((resolve = resolve_), (reject = reject_))
|
||||
);
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
||||
return [creationPromise, resolve!, reject!];
|
||||
}
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
import type { DocumentMetadata, RelativePath } from "src/persistence/database";
|
||||
import { EMPTY_HASH } from "./hash";
|
||||
|
||||
export function findMatchingFileBasedOnHash(
|
||||
contentHash: string,
|
||||
candidates: [RelativePath, DocumentMetadata][]
|
||||
): [RelativePath, DocumentMetadata] | undefined {
|
||||
if (contentHash === EMPTY_HASH) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return candidates.find(([_, metadata]) => metadata.hash === contentHash);
|
||||
}
|
||||
14
frontend/sync-client/src/utils/find-matching-file.ts
Normal file
14
frontend/sync-client/src/utils/find-matching-file.ts
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
import type { DocumentRecord } from "../persistence/database";
|
||||
import { EMPTY_HASH } from "./hash";
|
||||
|
||||
// TODO: make this smarter so that offline files can be renamed & edited at the same time
|
||||
export function findMatchingFile(
|
||||
contentHash: string,
|
||||
candidates: DocumentRecord[]
|
||||
): DocumentRecord | undefined {
|
||||
if (contentHash === EMPTY_HASH) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return candidates.find(({ metadata }) => metadata?.hash === contentHash);
|
||||
}
|
||||
|
|
@ -6,7 +6,7 @@ export function hash(content: Uint8Array): string {
|
|||
result = (result << 5) - result + content[i];
|
||||
result |= 0; // Convert to 32bit integer
|
||||
}
|
||||
return Math.abs(result).toString(16);
|
||||
return Math.abs(result).toString(16).padStart(8, "0");
|
||||
}
|
||||
|
||||
export const EMPTY_HASH = hash(new Uint8Array(0));
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import * as fetchRetryFactory from "fetch-retry";
|
||||
import type { RequestInitRetryParams } from "fetch-retry";
|
||||
import type { Logger } from "src/tracing/logger";
|
||||
import type { Logger } from "../tracing/logger";
|
||||
|
||||
function getUrlFromInput(input: RequestInfo | URL): string {
|
||||
if (input instanceof URL) {
|
||||
|
|
@ -31,7 +31,6 @@ export function retriedFetchFactory(
|
|||
}
|
||||
return false;
|
||||
},
|
||||
retries: 6,
|
||||
retryDelay: (attempt) => Math.pow(1.5, attempt) * 500,
|
||||
...init
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,12 +1,15 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"baseUrl": ".",
|
||||
"module": "ESNext",
|
||||
"target": "ESNext",
|
||||
"strict": true,
|
||||
"moduleResolution": "bundler",
|
||||
"allowSyntheticDefaultImports": true,
|
||||
"lib": ["DOM", "ESNext"]
|
||||
"moduleResolution": "bundler",
|
||||
"lib": [
|
||||
"DOM" // to get "fetch"
|
||||
],
|
||||
"declaration": true,
|
||||
"declarationDir": "./dist/types"
|
||||
},
|
||||
"exclude": ["./dist"]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,25 +1,13 @@
|
|||
const path = require("path");
|
||||
const { merge } = require("webpack-merge");
|
||||
|
||||
module.exports = (_env, _argv) => ({
|
||||
const common = {
|
||||
entry: "./src/index.ts",
|
||||
devtool: "source-map",
|
||||
target: "node",
|
||||
module: {
|
||||
rules: [
|
||||
{
|
||||
test: /\.ts$/,
|
||||
use: [
|
||||
{
|
||||
loader: "ts-loader",
|
||||
options: {
|
||||
compilerOptions: {
|
||||
declaration: true,
|
||||
declarationDir: "./dist/types"
|
||||
},
|
||||
transpileOnly: false
|
||||
}
|
||||
}
|
||||
]
|
||||
use: ["ts-loader"]
|
||||
},
|
||||
{
|
||||
test: /\.wasm$/,
|
||||
|
|
@ -28,22 +16,40 @@ module.exports = (_env, _argv) => ({
|
|||
]
|
||||
},
|
||||
optimization: {
|
||||
// the consuming project should take care of minification
|
||||
minimize: false
|
||||
},
|
||||
resolve: {
|
||||
extensions: [".ts", ".js"],
|
||||
extensions: [".ts"],
|
||||
alias: {
|
||||
root: __dirname,
|
||||
src: path.resolve(__dirname, "src")
|
||||
}
|
||||
},
|
||||
output: {
|
||||
clean: true,
|
||||
filename: "index.js",
|
||||
library: {
|
||||
name: "SyncClient",
|
||||
type: "umd"
|
||||
},
|
||||
path: path.resolve(__dirname, "dist")
|
||||
performance: {
|
||||
hints: false // it's a library, no need to warn about its size
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
module.exports = [
|
||||
merge(common, {
|
||||
target: "web",
|
||||
output: {
|
||||
path: path.resolve(__dirname, "dist"),
|
||||
filename: "sync-client.web.js",
|
||||
library: {
|
||||
name: "SyncClient",
|
||||
type: "umd"
|
||||
},
|
||||
globalObject: "this"
|
||||
}
|
||||
}),
|
||||
merge(common, {
|
||||
target: "node",
|
||||
output: {
|
||||
path: path.resolve(__dirname, "dist"),
|
||||
filename: "sync-client.node.js",
|
||||
libraryTarget: "commonjs2"
|
||||
}
|
||||
})
|
||||
];
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "test-client",
|
||||
"version": "0.0.0",
|
||||
"version": "0.0.30",
|
||||
"private": true,
|
||||
"bin": {
|
||||
"test-client": "./dist/cli.js"
|
||||
|
|
@ -11,13 +11,13 @@
|
|||
"test": "jest --passWithNoTests"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.13.5",
|
||||
"@types/node": "^22.13.10",
|
||||
"sync-client": "file:../sync-client",
|
||||
"ts-loader": "^9.5.2",
|
||||
"tslib": "2.8.1",
|
||||
"typescript": "5.7.3",
|
||||
"typescript": "5.8.2",
|
||||
"uuid": "^11.1.0",
|
||||
"webpack": "^5.98.0",
|
||||
"webpack-cli": "^6.0.1"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -18,9 +18,10 @@ export class MockAgent extends MockClient {
|
|||
initialSettings: Partial<SyncSettings>,
|
||||
public readonly name: string,
|
||||
private readonly doDeletes: boolean,
|
||||
useSlowFileEvents: boolean,
|
||||
private readonly jitterScaleInSeconds: number
|
||||
) {
|
||||
super(initialSettings);
|
||||
super(initialSettings, useSlowFileEvents);
|
||||
}
|
||||
|
||||
public async init(): Promise<void> {
|
||||
|
|
@ -46,27 +47,33 @@ export class MockAgent extends MockClient {
|
|||
? "(online) "
|
||||
: "(offline)";
|
||||
const formatted = `[${this.name} ${state}] ${logLine.timestamp.toISOString()} ${logLine.level} ${logLine.message}`;
|
||||
|
||||
// HACK: we have to ensure the file has been synced if we want to change it offline without data loss
|
||||
const historyEntry = /.*History entry: (.*.md).*/.exec(
|
||||
logLine.message
|
||||
);
|
||||
|
||||
if (historyEntry) {
|
||||
this.doNotTouchWhileOffline =
|
||||
this.doNotTouchWhileOffline.filter(
|
||||
(file) => file !== historyEntry[1]
|
||||
);
|
||||
}
|
||||
switch (logLine.level) {
|
||||
case LogLevel.ERROR:
|
||||
console.error(formatted);
|
||||
// Let's not ignore errors
|
||||
process.exit(1);
|
||||
|
||||
if (!this.useSlowFileEvents) {
|
||||
// Let's not ignore errors
|
||||
// eslint-disable-next-line @typescript-eslint/no-floating-promises
|
||||
sleep(100).then(() => process.exit(1));
|
||||
}
|
||||
|
||||
break;
|
||||
case LogLevel.WARNING:
|
||||
console.warn(formatted);
|
||||
break;
|
||||
case LogLevel.INFO:
|
||||
// HACK: we have to ensure the file has been synced if we want to change it offline without data loss
|
||||
const result = /.*History entry: (.*.md).*/.exec(
|
||||
logLine.message
|
||||
);
|
||||
if (result) {
|
||||
this.doNotTouchWhileOffline =
|
||||
this.doNotTouchWhileOffline.filter(
|
||||
(file) => file !== result[1]
|
||||
);
|
||||
}
|
||||
|
||||
console.info(formatted);
|
||||
break;
|
||||
case LogLevel.DEBUG:
|
||||
|
|
@ -84,11 +91,10 @@ export class MockAgent extends MockClient {
|
|||
this.changeFetchChangesUpdateIntervalMsAction.bind(this)
|
||||
];
|
||||
|
||||
if (
|
||||
this.client.settings.getSettings().isSyncEnabled &&
|
||||
this.doNotTouchWhileOffline.length === 0
|
||||
) {
|
||||
options.push(this.disableSyncAction.bind(this));
|
||||
if (this.client.settings.getSettings().isSyncEnabled) {
|
||||
if (this.doNotTouchWhileOffline.length === 0) {
|
||||
options.push(this.disableSyncAction.bind(this));
|
||||
}
|
||||
} else {
|
||||
options.push(this.enableSyncAction.bind(this));
|
||||
}
|
||||
|
|
@ -186,6 +192,14 @@ export class MockAgent extends MockClient {
|
|||
}
|
||||
|
||||
public assertAllContentIsPresentOnce(): void {
|
||||
if (this.useSlowFileEvents) {
|
||||
this.client.logger.info(
|
||||
// We can't ensure that we have seen every single update
|
||||
`Skipping content check for ${this.name} because slow file events are enabled`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
for (const content of this.writtenContents) {
|
||||
const found = Array.from(this.localFiles.keys()).filter((key) => {
|
||||
return new TextDecoder()
|
||||
|
|
@ -215,7 +229,7 @@ export class MockAgent extends MockClient {
|
|||
);
|
||||
assert(
|
||||
fileContent.split(content).length == 2,
|
||||
`Content ${content} (of ${this.name}) found more than once in file ${file}. File content:\n${fileContent}`
|
||||
`Content ${content} (of ${this.name}) found more than once in '${file}'. File content:\n${fileContent}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -237,10 +251,7 @@ export class MockAgent extends MockClient {
|
|||
`Decided to create file ${file} with content ${content}`
|
||||
);
|
||||
|
||||
return this.create(
|
||||
file,
|
||||
new TextEncoder().encode(` |${content}| `)
|
||||
);
|
||||
return this.create(file, new TextEncoder().encode(` ${content} `));
|
||||
}
|
||||
|
||||
private async changeFetchChangesUpdateIntervalMsAction(): Promise<void> {
|
||||
|
|
@ -314,7 +325,7 @@ export class MockAgent extends MockClient {
|
|||
`Decided to update file ${file} with ${content}`
|
||||
);
|
||||
this.doNotTouchWhileOffline.push(file);
|
||||
await this.atomicUpdateText(file, (old) => old + ` |${content}| `);
|
||||
await this.atomicUpdateText(file, (old) => old + ` ${content} `);
|
||||
}
|
||||
|
||||
private async deleteFileAction(files: RelativePath[]): Promise<void> {
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
import type {
|
||||
RelativePath,
|
||||
FileSystemOperations,
|
||||
SyncSettings
|
||||
import { assert } from "../utils/assert";
|
||||
import {
|
||||
type RelativePath,
|
||||
type FileSystemOperations,
|
||||
type SyncSettings,
|
||||
SyncClient
|
||||
} from "sync-client";
|
||||
import { SyncClient } from "sync-client";
|
||||
|
||||
export class MockClient implements FileSystemOperations {
|
||||
protected readonly localFiles = new Map<string, Uint8Array>();
|
||||
|
|
@ -11,7 +12,8 @@ export class MockClient implements FileSystemOperations {
|
|||
protected data: object | undefined = undefined;
|
||||
|
||||
public constructor(
|
||||
private readonly initialSettings: Partial<SyncSettings>
|
||||
private readonly initialSettings: Partial<SyncSettings>,
|
||||
protected readonly useSlowFileEvents: boolean
|
||||
) {}
|
||||
|
||||
public async init(): Promise<void> {
|
||||
|
|
@ -22,9 +24,10 @@ export class MockClient implements FileSystemOperations {
|
|||
|
||||
await Promise.all(
|
||||
Object.keys(this.initialSettings).map(async (key) => {
|
||||
const settingKey = key as keyof SyncSettings; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
|
||||
return this.client.settings.setSetting(
|
||||
key as keyof SyncSettings, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
|
||||
this.initialSettings[key as keyof SyncSettings] // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
|
||||
settingKey,
|
||||
this.initialSettings[settingKey]! // eslint-disable-line @typescript-eslint/no-non-null-assertion
|
||||
);
|
||||
})
|
||||
);
|
||||
|
|
@ -46,13 +49,6 @@ export class MockClient implements FileSystemOperations {
|
|||
return (await this.read(path)).length;
|
||||
}
|
||||
|
||||
public async getModificationTime(path: RelativePath): Promise<Date> {
|
||||
if (!this.localFiles.has(path)) {
|
||||
throw new Error(`File ${path} does not exist`);
|
||||
}
|
||||
return new Date();
|
||||
}
|
||||
|
||||
public async exists(path: RelativePath): Promise<boolean> {
|
||||
return this.localFiles.has(path);
|
||||
}
|
||||
|
|
@ -68,7 +64,10 @@ export class MockClient implements FileSystemOperations {
|
|||
`Creating file ${path} with content ${new TextDecoder().decode(newContent)}`
|
||||
);
|
||||
this.localFiles.set(path, newContent);
|
||||
void this.client.syncer.syncLocallyCreatedFile(path, new Date());
|
||||
|
||||
this.runCallback(() => {
|
||||
void this.client.syncer.syncLocallyCreatedFile(path);
|
||||
});
|
||||
}
|
||||
|
||||
public async createDirectory(_path: RelativePath): Promise<void> {
|
||||
|
|
@ -88,28 +87,51 @@ export class MockClient implements FileSystemOperations {
|
|||
const newContentUint8Array = new TextEncoder().encode(newContent);
|
||||
this.localFiles.set(path, newContentUint8Array);
|
||||
|
||||
if (!this.useSlowFileEvents) {
|
||||
const existingParts = currentContent
|
||||
.split(" ")
|
||||
.map((part) => part.trim());
|
||||
const newParts = newContent.split(" ").map((part) => part.trim());
|
||||
existingParts.forEach((part) =>
|
||||
// all changes should be additive
|
||||
{
|
||||
assert(
|
||||
newParts.includes(part),
|
||||
`Part ${part} not found in new content`
|
||||
);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
this.client.logger.info(
|
||||
`Updated file ${path} with:\n current content: ${currentContent}\n new content: ${newContent}`
|
||||
);
|
||||
|
||||
void this.client.syncer.syncLocallyUpdatedFile({
|
||||
relativePath: path,
|
||||
updateTime: new Date()
|
||||
this.runCallback(() => {
|
||||
void this.client.syncer.syncLocallyUpdatedFile({
|
||||
relativePath: path
|
||||
});
|
||||
});
|
||||
|
||||
return newContent;
|
||||
}
|
||||
|
||||
public async write(path: RelativePath, content: Uint8Array): Promise<void> {
|
||||
const hasExisted = this.localFiles.has(path);
|
||||
this.localFiles.set(path, content);
|
||||
|
||||
this.client.logger.info(
|
||||
`Updated file ${path} with:\n new content: ${new TextDecoder().decode(content)}`
|
||||
);
|
||||
|
||||
void this.client.syncer.syncLocallyUpdatedFile({
|
||||
relativePath: path,
|
||||
updateTime: new Date()
|
||||
this.runCallback(() => {
|
||||
if (hasExisted) {
|
||||
void this.client.syncer.syncLocallyUpdatedFile({
|
||||
relativePath: path
|
||||
});
|
||||
} else {
|
||||
void this.client.syncer.syncLocallyCreatedFile(path);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -118,7 +140,10 @@ export class MockClient implements FileSystemOperations {
|
|||
`Deleting file: ${path} with:\n content ${new TextDecoder().decode(this.localFiles.get(path))}`
|
||||
);
|
||||
this.localFiles.delete(path);
|
||||
void this.client.syncer.syncLocallyDeletedFile(path);
|
||||
|
||||
this.runCallback(() => {
|
||||
void this.client.syncer.syncLocallyDeletedFile(path);
|
||||
});
|
||||
}
|
||||
|
||||
public async rename(
|
||||
|
|
@ -138,10 +163,20 @@ export class MockClient implements FileSystemOperations {
|
|||
`Renamed file: ${oldPath} -> ${newPath} with:\n content ${new TextDecoder().decode(file)}`
|
||||
);
|
||||
|
||||
void this.client.syncer.syncLocallyUpdatedFile({
|
||||
oldPath,
|
||||
relativePath: newPath,
|
||||
updateTime: new Date()
|
||||
this.runCallback(() => {
|
||||
void this.client.syncer.syncLocallyUpdatedFile({
|
||||
oldPath,
|
||||
relativePath: newPath
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private runCallback(callback: () => void): void {
|
||||
if (this.useSlowFileEvents) {
|
||||
// we aren't the best client and it takes some time to notice changes
|
||||
setTimeout(callback, 100);
|
||||
} else {
|
||||
callback();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,20 +3,26 @@ import { MockAgent } from "./agent/mock-agent";
|
|||
import { sleep } from "./utils/sleep";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
|
||||
let slowFileEvents = false;
|
||||
|
||||
async function runTest({
|
||||
agentCount,
|
||||
concurrency,
|
||||
iterations,
|
||||
doDeletes,
|
||||
useSlowFileEvents,
|
||||
jitterScaleInSeconds
|
||||
}: {
|
||||
agentCount: number;
|
||||
concurrency: number;
|
||||
iterations: number;
|
||||
doDeletes: boolean;
|
||||
useSlowFileEvents: boolean;
|
||||
jitterScaleInSeconds: number;
|
||||
}): Promise<void> {
|
||||
const settings = `with ${agentCount} agents, concurrency ${concurrency}, iterations ${iterations}, doDeletes ${doDeletes}, jitterScaleInSeconds ${jitterScaleInSeconds}`;
|
||||
slowFileEvents = useSlowFileEvents;
|
||||
|
||||
const settings = `with ${agentCount} agents, concurrency ${concurrency}, iterations ${iterations}, doDeletes ${doDeletes}, jitterScaleInSeconds ${jitterScaleInSeconds}, useSlowFileEvents ${useSlowFileEvents}`;
|
||||
console.info(`Running test ${settings}`);
|
||||
|
||||
const initialSettings: Partial<SyncSettings> = {
|
||||
|
|
@ -34,6 +40,7 @@ async function runTest({
|
|||
initialSettings,
|
||||
`agent-${i}`,
|
||||
doDeletes,
|
||||
useSlowFileEvents,
|
||||
jitterScaleInSeconds
|
||||
)
|
||||
);
|
||||
|
|
@ -52,12 +59,24 @@ async function runTest({
|
|||
|
||||
// Each agent can have unpushed changes which might conflict with eachother so each has to resolve the conflicts & push, and
|
||||
for (const client of clients) {
|
||||
await client.finish();
|
||||
try {
|
||||
await client.finish();
|
||||
} catch (err) {
|
||||
if (!slowFileEvents) {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// then we need a second pass to ensure that all agents pull the same state.
|
||||
for (const client of clients) {
|
||||
await client.finish();
|
||||
try {
|
||||
await client.finish();
|
||||
} catch (err) {
|
||||
if (!slowFileEvents) {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.info("Agents finished successfully");
|
||||
|
|
@ -78,41 +97,49 @@ async function runTest({
|
|||
console.info(`Content check for ${client.name} passed`);
|
||||
});
|
||||
|
||||
console.info(`Test passed with ${settings}`);
|
||||
console.info(`Test passed ${settings}`);
|
||||
} catch (err) {
|
||||
console.error(`Test failed with ${settings}`);
|
||||
console.error(`Test failed ${settings}`);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async function runTests(): Promise<void> {
|
||||
const agentCounts = [2, 10];
|
||||
const jitterScaleInSeconds = [0.5, 3, 0];
|
||||
const concurrencies = [1, 16];
|
||||
const iterations = [50, 300];
|
||||
const doDeletes = [false, true];
|
||||
|
||||
for (const agentCount of agentCounts) {
|
||||
for (const concurrency of concurrencies) {
|
||||
for (const jitter of jitterScaleInSeconds) {
|
||||
for (const iteration of iterations) {
|
||||
for (const deleteFiles of doDeletes) {
|
||||
while (true) {
|
||||
await runTest({
|
||||
agentCount,
|
||||
concurrency,
|
||||
iterations: iteration,
|
||||
doDeletes: deleteFiles,
|
||||
jitterScaleInSeconds: jitter
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const useSlowFileEvents of [false, true]) {
|
||||
for (const concurrency of [
|
||||
16,
|
||||
1 // test with concurrency 1 to check for deadlocks
|
||||
]) {
|
||||
for (const doDeletes of [true, false]) {
|
||||
await runTest({
|
||||
agentCount: 3,
|
||||
concurrency,
|
||||
iterations: 100,
|
||||
doDeletes,
|
||||
useSlowFileEvents,
|
||||
jitterScaleInSeconds: 0.75
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
process.on("uncaughtException", (error) => {
|
||||
if (slowFileEvents) {
|
||||
return;
|
||||
}
|
||||
console.error("Uncaught Exception:", error);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
process.on("unhandledRejection", (reason, _promise) => {
|
||||
if (slowFileEvents) {
|
||||
return;
|
||||
}
|
||||
console.error("Unhandled Rejection:", reason);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
runTests()
|
||||
.then(() => {
|
||||
process.exit(0);
|
||||
|
|
|
|||
|
|
@ -12,8 +12,7 @@ module.exports = {
|
|||
rules: [
|
||||
{
|
||||
test: /\.ts$/,
|
||||
use: "ts-loader",
|
||||
exclude: /node_modules/
|
||||
use: "ts-loader"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue