Add files

Signed-off-by: András Schmelczer <andras@schmelczer.dev>

example/helper/__init__.py

@@ -0,0 +1 @@
+from .preprocess import preprocess

example/helper/preprocess.py

@@ -0,0 +1,12 @@
+from sus.clean import clean
+from sus.lemmatize_text import lemmatize_text
+import re
+
+
+def preprocess(text: str) -> str:
+    cleaned = clean(text, convert_to_ascii=True)
+    lemmas = [
+        re.sub(r'\d+', 'NUM', lemma)
+        for lemma in lemmatize_text(cleaned)
+    ]
+    return " ".join(lemmas)