andras/great-ai
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases 4 Packages Wiki Activity Actions

Format

Browse source 
Signed-off-by: András Schmelczer <andras@schmelczer.dev>
This commit is contained in:
Andras Schmelczer 2022-04-02 13:57:16 +02:00
parent d79a75d352
commit ea54cd9c8c
13 changed files with 168 additions and 116 deletions
Download patch file Download diff file Expand all files Collapse all files

8
example/helper/preprocess.py
View file

@ -1,12 +1,10 @@
import re
from sus.clean import clean
from sus.lemmatize_text import lemmatize_text
import re
def preprocess(text: str) -> str:
cleaned = clean(text, convert_to_ascii=True)
lemmas = [
re.sub(r'\d+', 'NUM', lemma)
for lemma in lemmatize_text(cleaned)
]
lemmas = [re.sub(r"\d+", "NUM", lemma) for lemma in lemmatize_text(cleaned)]
return " ".join(lemmas)