diff --git a/src/tokenizer/token.rs b/src/tokenizer/token.rs index 5d82eb5..0f45d41 100644 --- a/src/tokenizer/token.rs +++ b/src/tokenizer/token.rs @@ -1,4 +1,7 @@ -use std::fmt::Debug; +use std::{ + fmt::Debug, + hash::{Hash, Hasher}, +}; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; @@ -78,3 +81,14 @@ where self.normalized == other.normalized } } + +/// Hashes based on the `normalized` field only, consistent with the +/// [`PartialEq`] implementation. +impl Hash for Token +where + T: PartialEq + Clone + Debug + Hash, +{ + fn hash(&self, state: &mut H) { + self.normalized.hash(state); + } +}