16 lines
446 B
Python
16 lines
446 B
Python
import re
|
|
|
|
from great_ai.utilities.clean import clean
|
|
from great_ai.utilities.lemmatize_text import lemmatize_text
|
|
|
|
|
|
def preprocess(text: str) -> str:
|
|
text = clean(text, convert_to_ascii=True)
|
|
text = re.sub(r"[^a-zA-Z0-9]", " ", text)
|
|
return text
|
|
|
|
|
|
def lemmatize(text: str) -> str:
|
|
lemmatized = lemmatize_text(text)
|
|
clean_lemmas = [re.sub(r"\d[\d.,]*", "NUM", lemma) for lemma in lemmatized]
|
|
return " ".join(clean_lemmas)
|