andras/great-ai
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases 4 Packages Wiki Activity Actions

Remove Spacy and Publication TEI

Browse source
This commit is contained in:
Andras Schmelczer 2022-06-26 10:46:40 +02:00
parent e16db17db0
commit 46c99a06f9
30 changed files with 5 additions and 2412 deletions
Download patch file Download diff file Expand all files Collapse all files

139
tests/utilities/test_lemmatize_text.py
View file

@ -1,139 +0,0 @@
import unittest
from src.great_ai.utilities import lemmatize_text
class TestLemmatizeText(unittest.TestCase):
def test_simple(self) -> None:
text = "The state-of-the-art could not be improved, however we managed to create a less resource-intensive implementation of it."
lemmatized = [
"the",
"state",
"-",
"of",
"-",
"the",
"-",
"art",
"could",
"not",
"be",
"improve",
",",
"however",
"we",
"manage",
"to",
"create",
"a",
"less",
"resource",
"-",
"intensive",
"implementation",
"of",
"it",
".",
]
lemmatized_pos = [
"the_DET",
"state_NOUN",
"-_PUNCT",
"of_ADP",
"-_PUNCT",
"the_DET",
"-_PUNCT",
"art_NOUN",
"could_AUX",
"not_PART",
"be_AUX",
"improve_VERB",
",_PUNCT",
"however_ADV",
"we_PRON",
"manage_VERB",
"to_PART",
"create_VERB",
"a_DET",
"less_ADV",
"resource_NOUN",
"-_PUNCT",
"intensive_ADJ",
"implementation_NOUN",
"of_ADP",
"it_PRON",
"._PUNCT",
]
lemmatized_neg = [
"the",
"state",
"-",
"of",
"-",
"the",
"-",
"art",
"could",
"not",
"NOT_be",
"NOT_improve",
"NOT_,",
"however",
"we",
"manage",
"to",
"create",
"a",
"less",
"resource",
"-",
"intensive",
"implementation",
"of",
"it",
".",
]
lemmatized_pos_neg = [
"the_DET",
"state_NOUN",
"-_PUNCT",
"of_ADP",
"-_PUNCT",
"the_DET",
"-_PUNCT",
"art_NOUN",
"could_AUX",
"not_PART",
"NOT_be_AUX",
"NOT_improve_VERB",
"NOT_,_PUNCT",
"however_ADV",
"we_PRON",
"manage_VERB",
"to_PART",
"create_VERB",
"a_DET",
"less_ADV",
"resource_NOUN",
"-_PUNCT",
"intensive_ADJ",
"implementation_NOUN",
"of_ADP",
"it_PRON",
"._PUNCT",
]
assert lemmatize_text(text) == lemmatized
assert lemmatize_text(text, add_part_of_speech=True) == lemmatized_pos
assert lemmatize_text(text, add_negation=True) == lemmatized_neg
self.assertEqual(
lemmatize_text(text, add_negation=True, add_part_of_speech=True),
lemmatized_pos_neg,
)
def test_empty(self) -> None:
assert lemmatize_text("") == []

19
tests/utilities/test_lemmatize_token.py
View file

@ -1,19 +0,0 @@
import unittest
from src.great_ai.utilities import lemmatize_token, nlp
class TestLemmatizeToken(unittest.TestCase):
def test_simple(self) -> None:
token = nlp("Center")[0]
assert lemmatize_token(token) == "centre"
assert lemmatize_token(token, add_negation=True) == "centre"
assert lemmatize_token(token, add_part_of_speech=True) == "centre_NOUN"
def test_punctuation(self) -> None:
token = nlp("This.")[1]
assert lemmatize_token(token) == "."
assert lemmatize_token(token, add_negation=True) == "."
assert lemmatize_token(token, add_part_of_speech=True) == "._PUNCT"

67
tests/utilities/test_publication_tei.py
View file

@ -1,67 +0,0 @@
import unittest
from pathlib import Path
from src.great_ai.utilities import PublicationTEI
from .data.parsed import (
abstract,
authors,
bookmarks,
conclusion,
content,
introduction,
metadata,
sentences,
)
DATA_PATH = Path(__file__).parent.resolve() / "data"
class TestPublicationTEI(unittest.TestCase):
test_xml: str
@classmethod
def setUpClass(cls) -> None:
with open(
DATA_PATH / "10.1136_bmjspcare-2021-003026.pdf.tei.xml", encoding="utf-8"
) as f:
cls.test_xml = f.read()
def test_metadata_extraction(self) -> None:
assert PublicationTEI(self.test_xml).publication_metadata == metadata
def test_authors(self) -> None:
assert PublicationTEI(self.test_xml).authors == authors
def test_content(self) -> None:
assert PublicationTEI(self.test_xml).content == content
def test_sentences(self) -> None:
assert PublicationTEI(self.test_xml).sentences == sentences
def test_bookmarks(self) -> None:
assert PublicationTEI(self.test_xml).bookmarks == bookmarks
def test_abstract(self) -> None:
assert PublicationTEI(self.test_xml).abstract_sentences == abstract
def test_introduction(self) -> None:
self.assertEqual(
PublicationTEI(self.test_xml).introduction_sentences, introduction
)
def test_conclusion(self) -> None:
assert PublicationTEI(self.test_xml).conclusion_sentences == conclusion
def test_empty1(self) -> None:
tei = PublicationTEI("<TEI/>")
tei.publication_metadata, tei.publication_metadata, tei.authors, tei.content, tei.sentences
def test_empty2(self) -> None:
tei = PublicationTEI("")
tei.publication_metadata, tei.publication_metadata, tei.authors, tei.content, tei.sentences
def test_missing_fields(self) -> None:
with open(DATA_PATH / "bad.tei.xml", encoding="utf-8") as f:
tei = PublicationTEI(f.read())
tei.publication_metadata, tei.publication_metadata, tei.authors, tei.content, tei.sentences