andras/great-ai
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases 4 Packages Wiki Activity Actions

Add exception logging

Browse source
This commit is contained in:
Andras Schmelczer 2022-05-26 21:00:14 +02:00
parent 47e8d8b33a
commit cdc43f75ac
20 changed files with 144 additions and 182 deletions
Download patch file Download diff file Expand all files Collapse all files

16
examples/simple/helpers.py
View file

@ -1,16 +0,0 @@
import re
from great_ai.utilities.clean import clean
from great_ai.utilities.lemmatize_text import lemmatize_text
def preprocess(text: str) -> str:
text = clean(text, convert_to_ascii=True)
text = re.sub(r"[^a-zA-Z0-9]", " ", text)
return text
def lemmatize(text: str) -> str:
lemmatized = lemmatize_text(text)
clean_lemmas = [re.sub(r"\d[\d.,]*", "NUM", lemma) for lemma in lemmatized]
return " ".join(clean_lemmas)