Experiment with framework
Signed-off-by: András Schmelczer <andras@schmelczer.dev>
This commit is contained in:
parent
60cd55c0cd
commit
5890bbc3d5
26 changed files with 442 additions and 256 deletions
9
example/preprocess.py
Normal file
9
example/preprocess.py
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
import re
|
||||
|
||||
from sus.clean import clean
|
||||
from sus.lemmatize_text import lemmatize_text
|
||||
|
||||
|
||||
def preprocess(text: str) -> str:
|
||||
lemmas = [re.sub(r"\d+", "NUM", lemma) for lemma in lemmatize_text(text)]
|
||||
return " ".join(lemmas)
|
||||
Loading…
Add table
Add a link
Reference in a new issue