andras/reconcile
1
0
Fork 0
Code Issues Pull requests 5 Projects Releases Packages Wiki Activity Actions

Don't depend on serde for wasm

Browse source
This commit is contained in:
Andras Schmelczer 2025-11-16 12:29:30 +00:00
parent d0ce566118
commit 4c77a0360e
14 changed files with 251 additions and 314 deletions
Download patch file Download diff file Expand all files Collapse all files

20
Cargo.lock generated
View file

@ -124,12 +124,6 @@ version = "0.4.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
[[package]]
name = "memchr"
version = "2.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273"
[[package]]
name = "memory_units"
version = "0.4.0"
@ -188,7 +182,6 @@ dependencies = [
"insta",
"pretty_assertions",
"serde",
"serde_json",
"serde_yaml",
"test-case",
"wasm-bindgen",
@ -247,19 +240,6 @@ dependencies = [
"syn",
]
[[package]]
name = "serde_json"
version = "1.0.145"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c"
dependencies = [
"itoa",
"memchr",
"ryu",
"serde",
"serde_core",
]
[[package]]
name = "serde_yaml"
version = "0.9.34+deprecated"

5
Cargo.toml
View file

@ -24,7 +24,6 @@ path = "examples/merge-file.rs"
serde = { version = "1.0.219", optional = true, features = ["derive"] }
wasm-bindgen = { version = "0.2.99", optional = true }
serde_json = { version = "1.0.145", optional = true }
# The `console_error_panic_hook` crate provides better debugging of panics by
# logging them with `console.error`. This is great for development, but requires
@ -37,9 +36,9 @@ wee_alloc = { version = "0.4.2", optional = true }
[features]
default = []
serde = [ "dep:serde" ]
wasm = [ "dep:wasm-bindgen", "dep:wee_alloc", "dep:serde_json", "serde" ]
wasm = [ "dep:wasm-bindgen", "dep:wee_alloc" ]
console_error_panic_hook = [ "dep:console_error_panic_hook" ]
all = [ "wasm", "console_error_panic_hook" ]
all = [ "wasm", "console_error_panic_hook", "serde" ]
[dev-dependencies]
insta = "1.43.2"

9
reconcile-js/src/index.ts
View file

@ -182,22 +182,21 @@ export function reconcile(
/**
* Generates a compact diff representation between an original and changed text.
*
* These can be parsed and unpacked using Rust crate's EditedText::from_change_set.
* These can be parsed and unpacked using Rust crate's EditedText::from_changes.
*
* This function computes the differences between two versions of text and returns
* a compact string representation of those changes. The returned format is
* serialised JSON.
* a compact representation of those changes.
*
* @param original - The original/base version of the text
* @param changed - The modified version of the text (either string or TextWithCursors with cursor positions)
* @param tokenizer - The tokenisation strategy, which is the same as used in `reconcile`.
* @returns A compact string representation of the diff between original and changed text
* @returns An array representing the compact diff, with inserts as strings and deletes as negative integers.
*/
export function getCompactDiff(
original: string,
changed: string | TextWithOptionalCursors,
tokenizer: BuiltinTokenizer = 'Word'
): string {
): Array<number | string> {
init();
if (!BUILTIN_TOKENIZERS.includes(tokenizer)) {

24
src/lib.rs
View file

@ -157,6 +157,8 @@
//! original text, making the size only depends on the changes made.
//!
//! ```rust
//! # #[cfg(feature = "serde")]
//! # {
//! use reconcile_text::{EditedText, BuiltinTokenizer};
//! use serde_yaml;
//! use pretty_assertions::assert_eq;
@ -170,20 +172,18 @@
//! &changes.into()
//! );
//!
//! let serialized = serde_yaml::to_string(&result.to_change_set()).unwrap();
//! let serialized = serde_yaml::to_string(&result.to_changes()).unwrap();
//! assert_eq!(
//! serialized,
//! concat!(
//! "operations:\n",
//! "- 15\n",
//! "- -6\n",
//! "- ' easy with reconcile!'\n",
//! "cursors: []\n"
//! "- ' easy with reconcile!'\n"
//! )
//! );
//!
//! let deserialized = serde_yaml::from_str(&serialized).unwrap();
//! let reconstructed = EditedText::from_change_set(
//! let reconstructed = EditedText::from_changes(
//! original,
//! deserialized,
//! &*BuiltinTokenizer::Word
@ -192,13 +192,17 @@
//! reconstructed.apply().text(),
//! "Merging text is easy with reconcile!"
//! );
//! # }
//! ```
//!
//! ## Error handling
//!
//! The library is designed to be robust and will always produce a result, even
//! in edge cases. However, be aware that extremely large diffs may have
//! performance implications.
//! for edge cases.
//!
//! ## Performance
//!
//! Be aware that extremely large diffs may have performance implications.
//!
//! ## Algorithm overview
//!
@ -211,11 +215,11 @@ mod tokenizer;
mod types;
mod utils;
pub use operation_transformation::{ChangeSet, EditedText, reconcile};
pub use operation_transformation::{EditedText, reconcile};
pub use tokenizer::{BuiltinTokenizer, Tokenizer, token::Token};
pub use types::{
cursor_position::CursorPosition, history::History, side::Side,
span_with_history::SpanWithHistory, text_with_cursors::TextWithCursors,
cursor_position::CursorPosition, history::History, number_or_string::NumberOrString,
side::Side, span_with_history::SpanWithHistory, text_with_cursors::TextWithCursors,
};
#[cfg(feature = "wasm")]

1
src/operation_transformation.rs
View file

@ -5,7 +5,6 @@ use std::fmt::Debug;
pub use edited_text::EditedText;
pub use operation::Operation;
pub use transport::ChangeSet;
use crate::{Tokenizer, types::text_with_cursors::TextWithCursors};

136
src/operation_transformation/edited_text.rs
View file

@ -4,15 +4,17 @@ use std::{fmt::Debug, vec};
use serde::{Deserialize, Serialize};
use crate::{
BuiltinTokenizer, ChangeSet, CursorPosition, TextWithCursors,
BuiltinTokenizer, CursorPosition, TextWithCursors,
operation_transformation::{
Operation,
transport::SimpleOperation,
utils::{cook_operations::cook_operations, elongate_operations::elongate_operations},
},
raw_operation::RawOperation,
tokenizer::Tokenizer,
types::{history::History, side::Side, span_with_history::SpanWithHistory},
types::{
history::History, number_or_string::NumberOrString, side::Side,
span_with_history::SpanWithHistory,
},
utils::string_builder::StringBuilder,
};
@ -345,34 +347,105 @@ where
history
}
/// Serialize the `EditedText` as a `ChangeSet`, which contains only
/// the operations and cursor positions, but without the original text.
/// This is useful for sending changes over the network if there's
/// a clear consensus on the original text.
/// Convert the `EditedText` into a terse representation ready for
/// serialization. The result omits cursor positions and the original text.
/// This is useful for sending text diffs over the network if there's a
/// clear consensus on the original text.
///
/// Inserts are represented as strings, deletes as negative integers,
/// and equal spans as positive integers.
#[must_use]
pub fn to_change_set(&self) -> ChangeSet {
ChangeSet::new(
SimpleOperation::from_operations(&self.operations),
self.cursors.clone(),
)
pub fn to_changes(&self) -> Vec<NumberOrString> {
let mut result: Vec<NumberOrString> = Vec::with_capacity(self.operations.len());
let mut previous_equal: Option<usize> = None;
for operation in &self.operations {
match operation {
Operation::Equal { length, .. } => {
if let Some(prev_length) = previous_equal {
previous_equal = Some(prev_length + *length);
} else {
previous_equal = Some(*length);
}
}
Operation::Insert { text, .. } => {
if let Some(prev_length) = previous_equal {
result.push(NumberOrString::Number(prev_length as i64));
previous_equal = None;
}
let text: String = text
.iter()
.map(super::super::tokenizer::token::Token::original)
.collect();
result.push(NumberOrString::Text(text));
}
Operation::Delete {
deleted_character_count,
..
} => {
if let Some(prev_length) = previous_equal {
result.push(NumberOrString::Number(prev_length as i64));
previous_equal = None;
}
result.push(NumberOrString::Number(-(*deleted_character_count as i64)));
}
}
}
if let Some(prev_length) = previous_equal {
result.push(NumberOrString::Number(prev_length as i64));
}
result
}
/// Deserialize an `EditedText` from a `ChangeSet` and the original text.
/// This is useful for reconstructing the `EditedText` on the receiving
/// end after sending only the `ChangeSet` over the network.
/// Deserialize an `EditedText` from a change list and the original text.
#[must_use]
pub fn from_change_set(
text: &'a str,
change_set: ChangeSet,
pub fn from_changes(
original_text: &'a str,
simple_operations: Vec<NumberOrString>,
tokenizer: &Tokenizer<T>,
) -> EditedText<'a, T> {
let operations = SimpleOperation::to_operations(change_set.operations, text, tokenizer);
let mut operations: Vec<Operation<T>> = Vec::with_capacity(simple_operations.len());
let mut order = 0;
for simple_operation in simple_operations {
match simple_operation {
NumberOrString::Number(length) => {
if length >= 0 {
let length = length as usize;
let original_characters: String =
original_text.chars().skip(order).take(length).collect();
let original_tokens = tokenizer(&original_characters);
for token in original_tokens {
operations
.push(Operation::create_equal(order, token.get_original_length()));
order += token.get_original_length();
}
} else {
let length = -length as usize;
operations.push(Operation::create_delete(order, length));
order += length;
}
}
NumberOrString::Text(text) => {
let tokens = tokenizer(&text);
operations.push(Operation::create_insert(order, tokens));
}
}
}
let operation_count = operations.len();
EditedText::new(
text,
original_text,
operations,
vec![Side::Left; operation_count],
change_set.cursors,
vec![],
)
}
}
@ -423,34 +496,29 @@ mod tests {
assert_eq!(operations.apply().text(), expected);
}
#[cfg(feature = "serde")]
#[test]
fn test_change_set_deserialisation() {
fn test_changes_deserialisation() {
let original = "Merging text is hard!";
let changes = "Merging text is easy with reconcile!";
let result = EditedText::from_strings(original, &changes.into());
let serialized = serde_yaml::to_string(&result.to_change_set()).unwrap();
let expected = concat!(
"operations:\n",
"- 15\n",
"- -6\n",
"- ' easy with reconcile!'\n",
"cursors: []\n"
);
let serialized = serde_yaml::to_string(&result.to_changes()).unwrap();
let expected = concat!("- 15\n", "- -6\n", "- ' easy with reconcile!'\n",);
assert_eq!(serialized, expected);
}
#[cfg(feature = "serde")]
#[test]
fn test_change_set_serialization() {
fn test_changes_serialization() {
let original = "The quick brown fox jumps over the lazy dog.";
let updated = "The quick red fox jumped over the very lazy dog!";
let edited_text = EditedText::from_strings(original, &updated.into());
let change_set = edited_text.to_change_set();
let changes = edited_text.to_changes();
let deserialized_edited_text =
EditedText::from_change_set(original, change_set, &*BuiltinTokenizer::Word);
EditedText::from_changes(original, changes, &*BuiltinTokenizer::Word);
assert_eq!(deserialized_edited_text.apply().text(), updated);
}

2
src/types.rs
View file

@ -1,6 +1,6 @@
pub mod change_set;
pub mod cursor_position;
pub mod history;
pub mod number_or_string;
pub mod side;
pub mod span_with_history;
pub mod text_with_cursors;

189
src/types/change_set.rs
View file

@ -1,189 +0,0 @@
use std::fmt::Debug;
#[cfg(feature = "serde")]
use serde::{
Deserialize, Serialize,
de::{self, Deserializer, Visitor},
ser::Serializer,
};
#[cfg(feature = "wasm")]
use wasm_bindgen::prelude::*;
use crate::{Tokenizer, operation_transformation::Operation};
/// A serializable representation of the changes made to a text document
/// without the original text.
#[derive(Clone, PartialEq, Eq, Debug)]
enum SimpleOperation {
Equal { length: usize },
Insert { text: String },
Delete { length: usize },
}
impl SimpleOperation {
pub fn from_operations<T>(operation: &Vec<Operation<T>>) -> Vec<Self>
where
T: PartialEq + Clone + Debug,
{
let mut result: Vec<Self> = Vec::with_capacity(operation.len());
let mut previous_equal: Option<usize> = None;
for operation in operation {
match operation {
Operation::Equal { length, .. } => {
if let Some(prev_length) = previous_equal {
previous_equal = Some(prev_length + *length);
} else {
previous_equal = Some(*length);
}
}
Operation::Insert { text, .. } => {
if let Some(prev_length) = previous_equal {
result.push(SimpleOperation::Equal {
length: prev_length,
});
previous_equal = None;
}
let text: String = text
.iter()
.map(super::super::tokenizer::token::Token::original)
.collect();
result.push(SimpleOperation::Insert { text });
}
Operation::Delete {
deleted_character_count,
..
} => {
if let Some(prev_length) = previous_equal {
result.push(SimpleOperation::Equal {
length: prev_length,
});
previous_equal = None;
}
result.push(SimpleOperation::Delete {
length: *deleted_character_count,
});
}
}
}
if let Some(prev_length) = previous_equal {
result.push(SimpleOperation::Equal {
length: prev_length,
});
}
result
}
// This is similar to `crate::operation_transformation::utils::cook_operations`
pub fn to_operations<T>(
simple_operations: Vec<Self>,
original_text: &str,
tokenizer: &Tokenizer<T>,
) -> Vec<Operation<T>>
where
T: PartialEq + Clone + Debug,
{
let mut operations: Vec<Operation<T>> = Vec::with_capacity(simple_operations.len());
let mut order = 0;
for simple_operation in simple_operations {
match simple_operation {
SimpleOperation::Equal { length } => {
let original_characters: String =
original_text.chars().skip(order).take(length).collect();
let original_tokens = tokenizer(&original_characters);
for token in original_tokens {
operations
.push(Operation::create_equal(order, token.get_original_length()));
order += token.get_original_length();
}
}
SimpleOperation::Insert { text } => {
let tokens = tokenizer(&text);
operations.push(Operation::create_insert(order, tokens));
}
SimpleOperation::Delete { length } => {
operations.push(Operation::create_delete(order, length));
order += length;
}
}
}
operations
}
}
#[cfg(feature = "serde")]
impl Serialize for SimpleOperation {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
// neat idea from https://github.com/spebern/operational-transform-rs/blob/9faa17f0a2b282ac2e09dbb2d29fdaf2ae0bbb4a/operational-transform/src/serde.rs#L14
match self {
SimpleOperation::Equal { length } => serializer.serialize_u64(*length as u64),
SimpleOperation::Insert { text } => serializer.serialize_str(text),
SimpleOperation::Delete { length } => {
serializer.serialize_i64(-(i64::try_from(*length).unwrap_or(i64::MAX)))
}
}
}
}
#[cfg(feature = "serde")]
impl<'de> Deserialize<'de> for SimpleOperation {
fn deserialize<D>(deserializer: D) -> Result<SimpleOperation, D::Error>
where
D: Deserializer<'de>,
{
use std::fmt;
struct OperationVisitor;
impl Visitor<'_> for OperationVisitor {
type Value = SimpleOperation;
fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("an integer between -2^63 and 2^64-1 or a string")
}
fn visit_u64<E>(self, value: u64) -> Result<Self::Value, E>
where
E: de::Error,
{
Ok(SimpleOperation::Equal {
length: usize::try_from(value).unwrap_or(usize::MAX),
})
}
fn visit_i64<E>(self, value: i64) -> Result<Self::Value, E>
where
E: de::Error,
{
Ok(SimpleOperation::Delete {
length: usize::try_from(-value).unwrap_or(usize::MAX),
})
}
fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
where
E: de::Error,
{
Ok(SimpleOperation::Insert {
text: value.to_owned(),
})
}
}
deserializer.deserialize_any(OperationVisitor)
}
}

74
src/types/number_or_string.rs Normal file
View file

@ -0,0 +1,74 @@
use std::fmt::Debug;
#[cfg(feature = "serde")]
use serde::{Deserialize, Serialize};
#[cfg(feature = "wasm")]
use wasm_bindgen::prelude::*;
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[cfg_attr(feature = "serde", serde(untagged))]
#[derive(Debug, Clone, PartialEq)]
pub enum NumberOrString {
Number(i64),
Text(String),
}
#[cfg(feature = "wasm")]
impl TryFrom<JsValue> for NumberOrString {
type Error = DeserialisationError;
fn try_from(value: JsValue) -> Result<Self, Self::Error> {
if let Ok(num) = value.clone().try_into() {
return Ok(NumberOrString::Number(num));
}
if let Ok(text) = value.try_into() {
return Ok(NumberOrString::Text(text));
}
Err(DeserialisationError::new(
"Could not parse JsValue as either number or string",
))
}
}
#[cfg(feature = "wasm")]
impl From<NumberOrString> for JsValue {
fn from(value: NumberOrString) -> Self {
match value {
NumberOrString::Number(num) => JsValue::from(num),
NumberOrString::Text(text) => JsValue::from(text),
}
}
}
/// Error type for deserialisation failures
#[cfg(feature = "wasm")]
#[derive(Debug, Clone)]
pub struct DeserialisationError {
pub message: String,
}
#[cfg(feature = "wasm")]
impl DeserialisationError {
pub fn new(message: impl Into<String>) -> Self {
Self {
message: message.into(),
}
}
}
#[cfg(feature = "wasm")]
impl std::fmt::Display for DeserialisationError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "Deserialisation error: {}", self.message)
}
}
#[cfg(feature = "wasm")]
impl std::error::Error for DeserialisationError {}
#[cfg(feature = "wasm")]
impl From<DeserialisationError> for JsValue {
fn from(error: DeserialisationError) -> Self { JsValue::from_str(&error.message) }
}

1
src/utils.rs
View file

@ -3,4 +3,3 @@ pub mod common_suffix_len;
pub mod find_longest_prefix_contained_within;
pub mod myers_diff;
pub mod string_builder;
pub mod string_or_nothing;

26
src/utils/string_or_nothing.rs
View file

@ -1,26 +0,0 @@
/// Determine if the given data is a binary or a text file's content.
///
/// Returns the UTF8 parsed string if it's a text, or `None` if it's likely
/// binary.
#[must_use]
pub fn string_or_nothing(data: &[u8]) -> Option<String> {
if data.contains(&0) {
// Even though the NUL character is valid in UTF-8, it's highly suspicious in
// human-readable text.
return None;
}
std::str::from_utf8(data).map(|s| s.to_string()).ok()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_string_or_nothing() {
assert_eq!(string_or_nothing(&[0, 159, 146, 150]), None);
assert_eq!(string_or_nothing(&[0, 12]), None);
assert_eq!(string_or_nothing(b"hello"), Some("hello".into()));
}
}

49
src/wasm.rs
View file

@ -3,10 +3,7 @@ use core::str;
use wasm_bindgen::prelude::*;
use crate::{
BuiltinTokenizer, CursorPosition, SpanWithHistory, TextWithCursors,
utils::string_or_nothing::string_or_nothing,
};
use crate::{BuiltinTokenizer, CursorPosition, SpanWithHistory, TextWithCursors};
#[global_allocator]
static ALLOC: wee_alloc::WeeAlloc<'_> = wee_alloc::WeeAlloc::INIT;
@ -81,23 +78,22 @@ pub fn generic_reconcile(
}
}
/// WASM wrapper around getting a compact diff representation as a JSON string
///
/// # Panics
///
/// If serialization to JSON fails which should not happen
/// WASM wrapper around getting a compact diff representation of two texts as a
/// list of numbers and strings.
#[wasm_bindgen(js_name = getCompactDiff)]
#[must_use]
pub fn get_compact_diff(
parent: &str,
changed: &TextWithCursors,
tokenizer: BuiltinTokenizer,
) -> String {
) -> Vec<JsValue> {
set_panic_hook();
let edited_text = crate::EditedText::from_strings_with_tokenizer(parent, changed, &*tokenizer);
let change_set = edited_text.to_change_set();
serde_json::to_string(&change_set).expect("Failed to serialize change set")
edited_text
.to_changes()
.into_iter()
.map(std::convert::Into::into)
.collect()
}
fn set_panic_hook() {
@ -125,3 +121,30 @@ impl TextWithCursorsAndHistory {
#[must_use]
pub fn history(&self) -> Vec<SpanWithHistory> { self.history.clone() }
}
/// Returns the UTF8 parsed string if it's a text, or `None` if it's likely
/// binary.
#[must_use]
pub fn string_or_nothing(data: &[u8]) -> Option<String> {
if data.contains(&0) {
// Even though the NUL character is valid in UTF-8, it's highly suspicious in
// human-readable text.
return None;
}
std::str::from_utf8(data)
.map(std::borrow::ToOwned::to_owned)
.ok()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_string_or_nothing() {
assert_eq!(string_or_nothing(&[0, 159, 146, 150]), None);
assert_eq!(string_or_nothing(&[0, 12]), None);
assert_eq!(string_or_nothing(b"hello"), Some("hello".into()));
}
}

23
tests/test.rs
View file

@ -34,8 +34,9 @@ fn test_document_one_way_with_cursors() {
}
}
#[cfg(feature = "serde")]
#[test]
fn test_document_one_way_with_cursors_and_serialisation() {
fn test_document_one_way_with_serialisation() {
for doc in &get_all_documents() {
let parent = doc.parent();
let left_operations =
@ -47,19 +48,23 @@ fn test_document_one_way_with_cursors_and_serialisation() {
);
let serialised_left =
serde_yaml::from_str(&serde_yaml::to_string(&left_operations.to_change_set()).unwrap())
serde_yaml::from_str(&serde_yaml::to_string(&left_operations.to_changes()).unwrap())
.unwrap();
let serialised_right =
serde_yaml::from_str(&serde_yaml::to_string(&right_operations.to_changes()).unwrap())
.unwrap();
let serialised_right = serde_yaml::from_str(
&serde_yaml::to_string(&right_operations.to_change_set()).unwrap(),
)
.unwrap();
let restored_left_operations =
EditedText::from_change_set(&parent, serialised_left, &*BuiltinTokenizer::Word);
EditedText::from_changes(&parent, serialised_left, &*BuiltinTokenizer::Word);
let restored_right_operations =
EditedText::from_change_set(&parent, serialised_right, &*BuiltinTokenizer::Word);
EditedText::from_changes(&parent, serialised_right, &*BuiltinTokenizer::Word);
doc.assert_eq(&restored_left_operations.merge(restored_right_operations));
doc.assert_eq_without_cursors(
&restored_left_operations
.merge(restored_right_operations)
.apply()
.text(),
);
}
}

6
tests/wasm.rs
View file

@ -55,10 +55,12 @@ fn test_merge_binary() {
);
}
#[wasm_bindgen_test(unsupported = test)]
#[wasm_bindgen_test] // JsValue isn't supported outside of wasm
fn test_get_compact_diff() {
let parent = "hello ";
let changed = "world";
let result = get_compact_diff(parent, &changed.into(), BuiltinTokenizer::Word);
assert_eq!(result, "{\"operations\":[-6,\"world\"],\"cursors\":[]}");
assert_eq!(result.len(), 2);
assert_eq!(result[0].as_f64().unwrap(), -6.0);
assert_eq!(result[1].as_string().unwrap(), "world");
}