From fc0d17837d97b64b88d13d248b51f4ba9c22dc7e Mon Sep 17 00:00:00 2001
From: Andras Schmelczer <andras@schmelczer.dev>
Date: Sat, 14 Mar 2026 11:50:54 +0000
Subject: [PATCH] Implement hash

---
 src/tokenizer/token.rs | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/src/tokenizer/token.rs b/src/tokenizer/token.rs
index 5d82eb5..0f45d41 100644
--- a/src/tokenizer/token.rs
+++ b/src/tokenizer/token.rs
@@ -1,4 +1,7 @@
-use std::fmt::Debug;
+use std::{
+    fmt::Debug,
+    hash::{Hash, Hasher},
+};
 
 #[cfg(feature = "serde")]
 use serde::{Deserialize, Serialize};
@@ -78,3 +81,14 @@ where
         self.normalized == other.normalized
     }
 }
+
+/// Hashes based on the `normalized` field only, consistent with the
+/// [`PartialEq`] implementation.
+impl<T> Hash for Token<T>
+where
+    T: PartialEq + Clone + Debug + Hash,
+{
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.normalized.hash(state);
+    }
+}