andras/reconcile
andras/reconcile
1
0
Fork 0
Code Issues Pull requests 6 Projects Releases Packages Wiki Activity Actions

Add line tokenizer

Browse source
This commit is contained in:
Andras Schmelczer 2025-07-06 13:03:25 +01:00
commit 469e62106c
No known key found for this signature in database
GPG key ID: FC8F2C3D3D1A718C
13 changed files with 324 additions and 50 deletions
Download patch file Download diff file Expand all files Collapse all files

15
reconcile-js/src/index.ts
View file

@ -39,13 +39,18 @@ export interface SpanWithHistory {
history: History;
}
export type Tokenizer = "word" | "character";
export type Tokenizer = "Line" | "Word" | "Character";
const TOKENIZERS = ["Line", "Word", "Character"];
let isInitialised = false;
const UNINITIALISED_MODULE_ERROR =
"Reconcile module has not been initialized. Please call init() before using any other functions.";
const UNSUPPORTED_TOKENIZER_ERROR = `Unsupported tokenizer. Only ${TOKENIZERS.join(
", "
)} are supported.`;
/**
* Initializes the WASM module for text reconciliation.
* Must be called before using any other functions.
@ -84,6 +89,10 @@ export function reconcile(
throw new Error(UNINITIALISED_MODULE_ERROR);
}
if (!TOKENIZERS.includes(tokenizer)) {
throw new Error(UNSUPPORTED_TOKENIZER_ERROR);
}
const leftCursor = toWasmTextWithCursors(left);
const rightCursor = toWasmTextWithCursors(right);
@ -119,6 +128,10 @@ export function reconcileWithHistory(
throw new Error(UNINITIALISED_MODULE_ERROR);
}
if (!TOKENIZERS.includes(tokenizer)) {
throw new Error(UNSUPPORTED_TOKENIZER_ERROR);
}
const leftCursor = toWasmTextWithCursors(left);
const rightCursor = toWasmTextWithCursors(right);