Experiment with framework

Signed-off-by: András Schmelczer <andras@schmelczer.dev>
This commit is contained in:
Andras Schmelczer 2022-04-02 21:39:58 +02:00
parent 60cd55c0cd
commit 5890bbc3d5
26 changed files with 442 additions and 256 deletions

9
example/preprocess.py Normal file
View file

@ -0,0 +1,9 @@
import re
from sus.clean import clean
from sus.lemmatize_text import lemmatize_text
def preprocess(text: str) -> str:
lemmas = [re.sub(r"\d+", "NUM", lemma) for lemma in lemmatize_text(text)]
return " ".join(lemmas)