diff --git a/Dockerfile b/Dockerfile index 66c8ada..0579296 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,9 +19,6 @@ RUN python3 -m pip --no-cache-dir install --upgrade pip &&\ pip install --no-cache-dir ./great_ai &&\ rm -rf great_ai -# great_ai.utilities.nlp depends on this -RUN pip3 install --no-cache-dir en-core-web-sm@https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl - HEALTHCHECK \ --interval=60s \ --timeout=60s \ diff --git a/docs/README.md b/docs/README.md index 20de423..44b0b3d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,32 +1,14 @@ # **S**coutinScience **U**tilitie**S** for text processing [![Lint and test ScoutinScience utilities](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml/badge.svg)](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml) -> amogus - ## Exports - [clean](src/sus/clean.py) - [unique](src/sus/unique.py) - [parallel_map](src/sus/parallel_map.py) -- [match_names](src/sus/match_names/match_names.py) +- [lemmatize](src/sus/lemmatize.py) - [evaluate_ranking](src/sus/evaluate_ranking/evaluate_ranking.py) - [get_sentences](src/sus/get_sentences.py) -### Requires loading spacy model - -> This is automatic but will require some time. - -> Add this to the Dockerfile for caching the spaCy model: -> -> ```docker -> RUN pip install --no-cache-dir en-core-web-sm@https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl -> ``` - -- [publication TEI](src/sus/publication_tei/publication_tei.py) -- [lemmatize_text](src/sus/lemmatize_text.py) -- [lemmatize_token](src/sus/lemmatize_token.py) -- [spacy model (nlp)](src/sus/nlp.py) -- [filter_sentences](src/sus/matcher/filter_sentences.py) - ## Development - Optional booleans must have a default value of `False`. diff --git a/setup.cfg b/setup.cfg index 2a14c10..265c1ad 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,9 +23,6 @@ install_requires = unidecode >= 1.3.0 multiprocess >= 0.70.0.0 tqdm >= 4.0.0 - beautifulsoup4 >= 4.10.0 - lxml >= 4.6.0 - spacy >= 3.3.0 scikit-learn == 1.1.1 matplotlib >= 3.5.0 numpy >= 1.22.0 @@ -34,7 +31,6 @@ install_requires = langdetect >= 1.0.9 tinydb >= 4.7.0 pandas >= 1.4.0 - pyaml >= 21.0.0 boto3 >= 1.23.0 fastapi >= 0.70.0 plotly >= 5.8.0 diff --git a/src/great_ai/utilities/__init__.py b/src/great_ai/utilities/__init__.py index 22ed3b1..984d698 100644 --- a/src/great_ai/utilities/__init__.py +++ b/src/great_ai/utilities/__init__.py @@ -1,14 +1,10 @@ from .clean import clean +from .parallel_map import parallel_map +from .unique import unique from .config_file import ConfigFile, ParseError -from .evaluate_ranking import evaluate_ranking from .get_sentences import get_sentences from .language import english_name_of_language, is_english, predict_language -from .lemmatize_text import lemmatize_text -from .lemmatize_token import lemmatize_token +from .lemmatize import lemmatize from .logger import get_logger +from .evaluate_ranking import evaluate_ranking from .match_names import match_names -from .matcher import fast_tokenize, filter_sentences, normalize -from .nlp import nlp -from .parallel_map import parallel_map -from .publication_tei import PublicationTEI -from .unique import unique diff --git a/src/great_ai/utilities/external/negspacy/README.md b/src/great_ai/utilities/external/negspacy/README.md deleted file mode 100644 index 213da8d..0000000 --- a/src/great_ai/utilities/external/negspacy/README.md +++ /dev/null @@ -1 +0,0 @@ -https://github.com/jenojp/negspacy diff --git a/src/great_ai/utilities/external/negspacy/__init__.py b/src/great_ai/utilities/external/negspacy/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/great_ai/utilities/external/negspacy/negation.py b/src/great_ai/utilities/external/negspacy/negation.py deleted file mode 100644 index ff9115a..0000000 --- a/src/great_ai/utilities/external/negspacy/negation.py +++ /dev/null @@ -1,222 +0,0 @@ -import logging - -from spacy.language import Language -from spacy.matcher import PhraseMatcher -from spacy.tokens import Token - -from .termsets import termset - -default_ts = termset("en").get_patterns() - - -@Language.factory( - "negex", - default_config={ - "neg_termset": default_ts, - "extension_name": "negex", - "chunk_prefix": list(), - }, -) -class Negex: - """ - A spaCy pipeline component which identifies negated tokens in text. - - Based on: NegEx - A Simple Algorithm for Identifying Negated Findings and Diseasesin Discharge Summaries - Chapman, Bridewell, Hanbury, Cooper, Buchanan - - Parameters - ---------- - nlp: object - spaCy language object - termset_lang: str - language code, if using default termsets (e.g. "en" for english) - extension_name: str - defaults to "negex"; whether entity is negated is then available as ent._.negex - pseudo_negations: list - list of phrases that cancel out a negation, if empty, defaults are used - preceding_negations: list - negations that appear before an entity, if empty, defaults are used - following_negations: list - negations that appear after an entity, if empty, defaults are used - termination: list - phrases that "terminate" a sentence for processing purposes such as "but". If empty, defaults are used - - """ - - def __init__( - self, - nlp: Language, - name: str, - neg_termset: dict, - extension_name: str, - chunk_prefix: list, - ): - if not Token.has_extension(extension_name): - Token.set_extension(extension_name, default=False, force=True) - - ts = neg_termset - expected_keys = [ - "pseudo_negations", - "preceding_negations", - "following_negations", - "termination", - ] - if not set(ts.keys()) == set(expected_keys): - raise KeyError( - f"Unexpected or missing keys in 'neg_termset', expected: {expected_keys}, instead got: {list(ts.keys())}" - ) - - self.pseudo_negations = ts["pseudo_negations"] - self.preceding_negations = ts["preceding_negations"] - self.following_negations = ts["following_negations"] - self.termination = ts["termination"] - - self.nlp = nlp - self.extension_name = extension_name - self.build_patterns() - self.chunk_prefix = list(nlp.tokenizer.pipe(chunk_prefix)) - - def build_patterns(self): - # efficiently build spaCy matcher patterns - self.matcher = PhraseMatcher(self.nlp.vocab, attr="LOWER") - - self.pseudo_patterns = list(self.nlp.tokenizer.pipe(self.pseudo_negations)) - self.matcher.add("pseudo", None, *self.pseudo_patterns) - - self.preceding_patterns = list( - self.nlp.tokenizer.pipe(self.preceding_negations) - ) - self.matcher.add("Preceding", None, *self.preceding_patterns) - - self.following_patterns = list( - self.nlp.tokenizer.pipe(self.following_negations) - ) - self.matcher.add("Following", None, *self.following_patterns) - - self.termination_patterns = list(self.nlp.tokenizer.pipe(self.termination)) - self.matcher.add("Termination", None, *self.termination_patterns) - - def process_negations(self, doc): - """ - Find negations in doc and clean candidate negations to remove pseudo negations - - Parameters - ---------- - doc: object - spaCy Doc object - - Returns - ------- - preceding: list - list of tuples for preceding negations - following: list - list of tuples for following negations - terminating: list - list of tuples of terminating phrases - - """ - ### - # does not work properly in spacy 2.1.8. Will incorporate after 2.2. - # Relying on user to use NER in meantime - # see https://github.com/jenojp/negspacy/issues/7 - ### - # if not doc.is_nered: - # raise ValueError( - # "Negations are evaluated for Named Entities found in text. " - # "Your SpaCy pipeline does not included Named Entity resolution. " - # "Please ensure it is enabled or choose a different language model that includes it." - # ) - preceding = list() - following = list() - terminating = list() - - matches = self.matcher(doc) - pseudo = [ - (match_id, start, end) - for match_id, start, end in matches - if self.nlp.vocab.strings[match_id] == "pseudo" - ] - - for match_id, start, end in matches: - if self.nlp.vocab.strings[match_id] == "pseudo": - continue - pseudo_flag = False - for p in pseudo: - if start >= p[1] and start <= p[2]: - pseudo_flag = True - continue - if not pseudo_flag: - if self.nlp.vocab.strings[match_id] == "Preceding": - preceding.append((match_id, start, end)) - elif self.nlp.vocab.strings[match_id] == "Following": - following.append((match_id, start, end)) - elif self.nlp.vocab.strings[match_id] == "Termination": - terminating.append((match_id, start, end)) - else: - logging.warnings( - f"phrase {doc[start:end].text} not in one of the expected matcher types." - ) - return preceding, following, terminating - - def termination_boundaries(self, doc, terminating): - """ - Create sub sentences based on terminations found in text. - - Parameters - ---------- - doc: object - spaCy Doc object - terminating: list - list of tuples with (match_id, start, end) - - returns - ------- - boundaries: list - list of tuples with (start, end) of spans - - """ - sent_starts = [sent.start for sent in doc.sents] - terminating_starts = [t[1] for t in terminating] - starts = sent_starts + terminating_starts + [len(doc)] - starts.sort() - boundaries = list() - index = 0 - for i, start in enumerate(starts): - if not i == 0: - boundaries.append((index, start)) - index = start - return boundaries - - def negex(self, doc): - """ - Negates entities of interest - - Parameters - ---------- - doc: object - spaCy Doc object - - """ - preceding, following, terminating = self.process_negations(doc) - boundaries = self.termination_boundaries(doc, terminating) - for b in boundaries: - sub_preceding = [i for i in preceding if b[0] <= i[1] < b[1]] - sub_following = [i for i in following if b[0] <= i[1] < b[1]] - - for e in doc[b[0] : b[1]]: - if any(pre < e.i for pre in [i[1] for i in sub_preceding]): - e._.set(self.extension_name, True) - continue - if any(fol > e.i for fol in [i[2] for i in sub_following]): - e._.set(self.extension_name, True) - continue - if self.chunk_prefix: - if any( - e.text.lower().startswith(c.text.lower()) - for c in self.chunk_prefix - ): - e._.set(self.extension_name, True) - return doc - - def __call__(self, doc): - return self.negex(doc) diff --git a/src/great_ai/utilities/external/negspacy/termsets.py b/src/great_ai/utilities/external/negspacy/termsets.py deleted file mode 100644 index a226aba..0000000 --- a/src/great_ai/utilities/external/negspacy/termsets.py +++ /dev/null @@ -1,229 +0,0 @@ -""" -Default termsets for various languages -""" - -LANGUAGES = dict() - -# english termset dictionary -en = dict() -pseudo = [ - "no further", - "not able to be", - "not certain if", - "not certain whether", - "not necessarily", - "without any further", - "without difficulty", - "without further", - "might not", - "not only", - "no increase", - "no significant change", - "no change", - "no definite change", - "not extend", - "not cause", -] -en["pseudo_negations"] = pseudo - -preceding = [ - "absence of", - "declined", - "denied", - "denies", - "denying", - "no sign of", - "no signs of", - "not", - "not demonstrate", - "symptoms atypical", - "doubt", - "negative for", - "no", - "versus", - "without", - "doesn't", - "doesnt", - "don't", - "dont", - "didn't", - "didnt", - "wasn't", - "wasnt", - "weren't", - "werent", - "isn't", - "isnt", - "aren't", - "arent", - "cannot", - "can't", - "cant", - "couldn't", - "couldnt", - "never", -] -en["preceding_negations"] = preceding - -following = [ - "declined", - "unlikely", - "was not", - "were not", - "wasn't", - "wasnt", - "weren't", - "werent", -] -en["following_negations"] = following - -termination = [ - "although", - "apart from", - "as there are", - "aside from", - "but", - "except", - "however", - "involving", - "nevertheless", - "still", - "though", - "which", - "yet", -] -en["termination"] = termination - -LANGUAGES["en"] = en - -# en_clinical builds upon en -en_clinical = dict() -pseudo_clinical = pseudo + [ - "gram negative", - "not rule out", - "not ruled out", - "not been ruled out", - "not drain", - "no suspicious change", - "no interval change", - "no significant interval change", -] -en_clinical["pseudo_negations"] = pseudo_clinical - -preceding_clinical = preceding + [ - "patient was not", - "without indication of", - "without sign of", - "without signs of", - "without any reactions or signs of", - "no complaints of", - "no evidence of", - "no cause of", - "evaluate for", - "fails to reveal", - "free of", - "never developed", - "never had", - "did not exhibit", - "rules out", - "rule out", - "rule him out", - "rule her out", - "rule patient out", - "rule the patient out", - "ruled out", - "ruled him out", - "ruled her out", - "ruled patient out", - "ruled the patient out", - "r/o", - "ro", -] -en_clinical["preceding_negations"] = preceding_clinical - -following_clinical = following + ["was ruled out", "were ruled out", "free"] -en_clinical["following_negations"] = following_clinical - -termination_clinical = termination + [ - "cause for", - "cause of", - "causes for", - "causes of", - "etiology for", - "etiology of", - "origin for", - "origin of", - "origins for", - "origins of", - "other possibilities of", - "reason for", - "reason of", - "reasons for", - "reasons of", - "secondary to", - "source for", - "source of", - "sources for", - "sources of", - "trigger event for", -] -en_clinical["termination"] = termination_clinical -LANGUAGES["en_clinical"] = en_clinical - -en_clinical_sensitive = dict() - -preceding_clinical_sensitive = preceding_clinical + [ - "concern for", - "supposed", - "which causes", - "leads to", - "h/o", - "history of", - "instead of", - "if you experience", - "if you get", - "teaching the patient", - "taught the patient", - "teach the patient", - "educated the patient", - "educate the patient", - "educating the patient", - "monitored for", - "monitor for", - "test for", - "tested for", -] -en_clinical_sensitive["pseudo_negations"] = pseudo_clinical -en_clinical_sensitive["preceding_negations"] = preceding_clinical_sensitive -en_clinical_sensitive["following_negations"] = following_clinical -en_clinical_sensitive["termination"] = termination_clinical - -LANGUAGES["en_clinical_sensitive"] = en_clinical_sensitive - - -class termset: - def __init__(self, termset_lang): - self.pattern_types = [ - "pseudo_negations", - "preceding_negations", - "following_negations", - "termination", - ] - self.terms = LANGUAGES[termset_lang] - - def get_patterns(self): - return self.terms - - def remove_patterns(self, pattern_dict): - for key, value in pattern_dict.items(): - if key in self.pattern_types: - self.terms[key] = [i for i in self.terms[key] if i not in value] - else: - raise ValueError(f"Unexpected key: {key} not in {self.pattern_types}") - - def add_patterns(self, pattern_dict): - for key, value in pattern_dict.items(): - if key in self.pattern_types: - self.terms[key] = list(set(self.terms[key] + value)) - else: - raise ValueError(f"Unexpected key: {key} not in {self.pattern_types}") diff --git a/src/great_ai/utilities/lemmatize_text.py b/src/great_ai/utilities/lemmatize_text.py deleted file mode 100644 index bcf0d0f..0000000 --- a/src/great_ai/utilities/lemmatize_text.py +++ /dev/null @@ -1,21 +0,0 @@ -from typing import List - -from .lemmatize_token import lemmatize_token -from .nlp import nlp - - -def lemmatize_text( - text: str, - add_negation: bool = False, - add_part_of_speech: bool = False, -) -> List[str]: - doc = nlp(text) - - return [ - lemmatize_token( - t, - add_negation=add_negation, - add_part_of_speech=add_part_of_speech, - ) - for t in doc - ] diff --git a/src/great_ai/utilities/lemmatize_token.py b/src/great_ai/utilities/lemmatize_token.py deleted file mode 100644 index f7a7ab8..0000000 --- a/src/great_ai/utilities/lemmatize_token.py +++ /dev/null @@ -1,20 +0,0 @@ -from spacy.tokens import Token - -from .data import american_spellings - - -def lemmatize_token( - token: Token, - add_negation: bool = False, - add_part_of_speech: bool = False, -) -> str: - lemma = token.lemma_.lower() - - lemma = american_spellings.get(lemma, lemma) - - if add_part_of_speech: - lemma = f"{lemma}_{token.pos_}" - if add_negation and token._.negex: - lemma = f"NOT_{lemma}" - - return lemma diff --git a/src/great_ai/utilities/matcher/__init__.py b/src/great_ai/utilities/matcher/__init__.py deleted file mode 100644 index ce2e5ab..0000000 --- a/src/great_ai/utilities/matcher/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .fast_tokenize import fast_tokenize -from .filter_sentences import filter_sentences -from .normalize import normalize diff --git a/src/great_ai/utilities/matcher/fast_tokenize.py b/src/great_ai/utilities/matcher/fast_tokenize.py deleted file mode 100644 index 4dcd87c..0000000 --- a/src/great_ai/utilities/matcher/fast_tokenize.py +++ /dev/null @@ -1,30 +0,0 @@ -import re -from typing import List, Union - -from segtok.tokenizer import word_tokenizer - -from ..get_sentences import get_sentences -from .normalize import normalize - - -def fast_tokenize( - text: Union[List[str], str], ignore_partial: bool = False -) -> List[List[str]]: - if isinstance(text, str): - text = normalize(text) - text = get_sentences(text, ignore_partial=ignore_partial) - - results: List[List[str]] = [] - - for sentence in text: - sentence = re.sub(r"\bare\b", "is", sentence) - sentence = re.sub(r"\ban\b", "a", sentence) - sentence = re.sub(r"\bthese\b", "this", sentence) - results.append( - [ - token.lower() if token not in {"CITATION", "NUMBER"} else token - for token in word_tokenizer(sentence) - ] - ) - - return results diff --git a/src/great_ai/utilities/matcher/filter_sentences.py b/src/great_ai/utilities/matcher/filter_sentences.py deleted file mode 100644 index 25c1ba7..0000000 --- a/src/great_ai/utilities/matcher/filter_sentences.py +++ /dev/null @@ -1,66 +0,0 @@ -from pathlib import Path -from typing import Dict, List, Union - -import yaml -from spacy.matcher import Matcher - -from ..get_sentences import get_sentences -from ..nlp import nlp -from .fast_tokenize import fast_tokenize -from .normalize import normalize - -rules_cache: Dict[str, Matcher] = {} - - -def filter_sentences( - sentences: str, - rules_file: Path, - inverse: bool = False, - ignore_partial: bool = False, -) -> List[str]: - if str(rules_file) not in rules_cache: - with open(rules_file, encoding="utf-8") as f: - rule_patterns = yaml.safe_load(f).keys() - - matcher = Matcher(nlp.vocab) - rules = [_pattern_to_rule(p) for p in rule_patterns] - matcher.add("", rules) - rules_cache[str(rules_file)] = matcher - - matcher = rules_cache[str(rules_file)] - - original_sentences = get_sentences(sentences, ignore_partial=ignore_partial) - - tokenized = fast_tokenize(original_sentences, ignore_partial=ignore_partial) - - results: List[str] = [] - for original_sentence, sentence in zip(original_sentences, tokenized): - doc = nlp(normalize(" ".join(sentence))) - matches = matcher(doc) - if matches: - # _, start, end = max( - # matches, - # key=lambda v: v[2] - v[1], - # ) - # print(str(doc[start:end])) - - if not inverse: - results.append(original_sentence) - elif inverse: - results.append(original_sentence) - - return results - - -def _pattern_to_rule(pattern: str) -> List[Dict[str, Union[bool, str]]]: - result: List[Dict[str, Union[bool, str]]] = [] - for t in pattern.split(): - if t == "*": - result.extend([{"OP": "?"}, {"OP": "?"}]) - elif t == "CITATION": - result.append({"ORTH": "CITATION"}) - elif t == "NUMBER": - result.append({"ORTH": "NUMBER"}) - else: - result.append({"LOWER": t}) - return result diff --git a/src/great_ai/utilities/matcher/normalize.py b/src/great_ai/utilities/matcher/normalize.py deleted file mode 100644 index 54d640b..0000000 --- a/src/great_ai/utilities/matcher/normalize.py +++ /dev/null @@ -1,22 +0,0 @@ -import re - -from ..clean import clean - - -def normalize(text: str) -> str: - text = re.sub( - r""" - ([A-Z]\w+\W+(et\ al.)) # inline reference: Bank et al. - | (\[[0-9-, ]+\]) # IEEE style: [1], [2-4], [3, 5] - | (\(.*?,?\W+?\d+\)) # APA style: (Bank, 2020) - | ([A-Z]\w+ \(?\d+\)) # APA style: Bank (2020) - """, - " CITATION ", - text, - flags=re.VERBOSE, - ) - text = re.sub(r"\d[\d,. -]*(st|nd|rd|th)?", " NUMBER ", text) - text = clean(text, convert_to_ascii=True) - text = re.sub(r"[^a-zA-Z.?!,:;'\" -]", "", text) - - return text diff --git a/src/great_ai/utilities/nlp.py b/src/great_ai/utilities/nlp.py deleted file mode 100644 index 87a0da9..0000000 --- a/src/great_ai/utilities/nlp.py +++ /dev/null @@ -1,21 +0,0 @@ -try: - import en_core_web_sm -except ImportError: - import subprocess - - print("Spacy model en_core_web_sm not found locally, downloading...") - - subprocess.call( - [ - "pip", - "install", - "https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl", - ] - ) - import en_core_web_sm - -from .external.negspacy import negation # noqa: F401 it's important to import this - -nlp = en_core_web_sm.load() - -nlp.add_pipe("negex") diff --git a/src/great_ai/utilities/publication_tei/__init__.py b/src/great_ai/utilities/publication_tei/__init__.py deleted file mode 100644 index dc85fcf..0000000 --- a/src/great_ai/utilities/publication_tei/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .models import * -from .publication_tei import PublicationTEI diff --git a/src/great_ai/utilities/publication_tei/models/__init__.py b/src/great_ai/utilities/publication_tei/models/__init__.py deleted file mode 100644 index 9bd3593..0000000 --- a/src/great_ai/utilities/publication_tei/models/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from .affiliation import Affiliation -from .author import Author -from .bookmark import Bookmark -from .bookmark_title import BookmarkTitle -from .element import Element, Meta, MetaType, Paragraph, Title -from .publication_metadata import PublicationMetadata -from .text import Text diff --git a/src/great_ai/utilities/publication_tei/models/affiliation.py b/src/great_ai/utilities/publication_tei/models/affiliation.py deleted file mode 100644 index 66f2188..0000000 --- a/src/great_ai/utilities/publication_tei/models/affiliation.py +++ /dev/null @@ -1,11 +0,0 @@ -from typing import Optional, Tuple - -from pydantic import BaseModel - - -class Affiliation(BaseModel): - institutions: Tuple[str, ...] - departments: Tuple[str, ...] - laboratories: Tuple[str, ...] - country: Optional[str] - settlement: Optional[str] diff --git a/src/great_ai/utilities/publication_tei/models/author.py b/src/great_ai/utilities/publication_tei/models/author.py deleted file mode 100644 index d31ebe2..0000000 --- a/src/great_ai/utilities/publication_tei/models/author.py +++ /dev/null @@ -1,14 +0,0 @@ -from typing import List, Optional - -from pydantic import BaseModel - -from .affiliation import Affiliation - - -class Author(BaseModel): - name: Optional[str] - orcid: Optional[str] - email: Optional[str] - corresponding: bool - affiliations: List[Affiliation] - coordinates: Optional[str] diff --git a/src/great_ai/utilities/publication_tei/models/bookmark.py b/src/great_ai/utilities/publication_tei/models/bookmark.py deleted file mode 100644 index e8df38d..0000000 --- a/src/great_ai/utilities/publication_tei/models/bookmark.py +++ /dev/null @@ -1,10 +0,0 @@ -from pydantic import BaseModel - -from .bookmark_title import BookmarkTitle - - -class Bookmark(BaseModel): - title: BookmarkTitle - original_title: str - document_order: int - coordinates: str diff --git a/src/great_ai/utilities/publication_tei/models/bookmark_title.py b/src/great_ai/utilities/publication_tei/models/bookmark_title.py deleted file mode 100644 index 0258c30..0000000 --- a/src/great_ai/utilities/publication_tei/models/bookmark_title.py +++ /dev/null @@ -1,16 +0,0 @@ -from typing import Literal - -BookmarkTitle = Literal[ - "Abstract", - "Author contribution", - "Introduction", - "Background", - "Methods", - "Results", - "Discussion", - "Outlook", - "Conclusion", - "Conflict of interest", - "Acknowledgement", - "Annex", -] diff --git a/src/great_ai/utilities/publication_tei/models/element.py b/src/great_ai/utilities/publication_tei/models/element.py deleted file mode 100644 index f942d5a..0000000 --- a/src/great_ai/utilities/publication_tei/models/element.py +++ /dev/null @@ -1,30 +0,0 @@ -from typing import List, Literal, Union - -from pydantic import BaseModel - -from .text import Text - - -class Title(BaseModel): - text: Text - - -class Paragraph(BaseModel): - sentences: List[Text] - - -MetaType = Literal[ - "abstract_start", - "abstract_end", - "acknowledgements_start", - "acknowledgements_end", - "annex_start", - "annex_end", -] - - -class Meta(BaseModel): - meta_type: MetaType - - -Element = Union[Title, Paragraph, Meta] diff --git a/src/great_ai/utilities/publication_tei/models/publication_metadata.py b/src/great_ai/utilities/publication_tei/models/publication_metadata.py deleted file mode 100644 index 904bbf5..0000000 --- a/src/great_ai/utilities/publication_tei/models/publication_metadata.py +++ /dev/null @@ -1,14 +0,0 @@ -from typing import List, Optional - -from pydantic import BaseModel - - -class PublicationMetadata(BaseModel): - language: Optional[str] - title: Optional[str] - publisher: Optional[str] - doi: Optional[str] - md5: Optional[str] - publication_date: Optional[str] - keywords: List[str] - reference_count: int diff --git a/src/great_ai/utilities/publication_tei/models/text.py b/src/great_ai/utilities/publication_tei/models/text.py deleted file mode 100644 index 3b74b36..0000000 --- a/src/great_ai/utilities/publication_tei/models/text.py +++ /dev/null @@ -1,9 +0,0 @@ -from typing import Optional - -from pydantic import BaseModel - - -class Text(BaseModel): - content: str - document_order: int - coordinates: Optional[str] diff --git a/src/great_ai/utilities/publication_tei/publication_tei.py b/src/great_ai/utilities/publication_tei/publication_tei.py deleted file mode 100644 index 49eeaa5..0000000 --- a/src/great_ai/utilities/publication_tei/publication_tei.py +++ /dev/null @@ -1,423 +0,0 @@ -import os -import re -from functools import cached_property, lru_cache -from pathlib import Path -from typing import Any, List, Optional, Pattern, Tuple, Union - -from bs4 import BeautifulSoup -from bs4.element import NavigableString, Tag - -from ..clean import clean -from ..lemmatize_text import lemmatize_text -from ..matcher import filter_sentences -from ..unique import unique -from .models import ( - Affiliation, - Author, - Bookmark, - BookmarkTitle, - Element, - Meta, - MetaType, - Paragraph, - PublicationMetadata, - Text, - Title, -) -from .titles_of_interest import titles_of_interest - -THIS_FOLDER = Path(os.path.dirname(os.path.abspath(__file__))) - - -class PublicationTEI: - # remove template sentences, such as copyright notices - aggressive_cleaning_enabled = True - - def __init__(self, tei: str): - self._document_order_counter = 0 - - if tei: - self.soup = BeautifulSoup(tei, "xml") - else: - self.soup = BeautifulSoup() - - @cached_property - def publication_metadata(self) -> PublicationMetadata: - publication_date = ( - self.soup.publicationStmt.date.get("when") - if self.soup.publicationStmt and self.soup.publicationStmt.date - else None - ) - - keywords = ( - [self._element_to_text(k) for k in self.soup.keywords.find_all("term")] - if self.soup.keywords - else [] - ) - - return PublicationMetadata( - language=self.soup.teiHeader.get("xml:lang") - if self.soup.teiHeader - else None, - title=self._element_to_text(self.soup.title), - publisher=self._element_to_text(self.soup.publisher), - doi=self._element_to_text(self.soup.find("idno", type="DOI")), - md5=self._element_to_text(self.soup.find("idno", type="MD5")), - publication_date=publication_date, - keywords=keywords, - reference_count=self.get_reference_count(), - ) - - def get_reference_count(self) -> int: - references = self.soup.find("div", {"type": "references"}) - - if not references: - return 0 - - return len(references.findAll("biblStruct")) - - @cached_property - def authors(self) -> List[Author]: - if not self.soup.analytic: - return [] - - return [ - self._parse_author(author) - for author in self.soup.analytic.find_all("author") - ] - - @cached_property - def content(self) -> List[Element]: - self._document_order_counter = 0 - return self._get_elements(self.soup) - - @cached_property - def sentences(self) -> List[Text]: - return [ - sentence - for element in self.content - if isinstance(element, Paragraph) - for sentence in element.sentences - ] - - @cached_property - def bookmarks(self) -> List[Bookmark]: - candidates: List[Bookmark] = [ - *self._find_matching_meta("Abstract", "abstract_start")[0], - *self._find_matching_title("Abstract"), - *self._find_matching_title("Author contribution"), - *self._find_matching_meta("Acknowledgement", "acknowledgements_start")[0], - *self._find_matching_title("Acknowledgement"), - *self._find_matching_title("Conflict of interest"), - ] - - _, start = self._find_matching_meta(None, "abstract_end") - - candidates += [ - *self._find_matching_title("Background", start), - *self._find_matching_title("Methods", start), - *self._find_matching_title("Results", start), - *self._find_matching_title("Discussion", start), - *self._find_matching_title("Introduction", start), - *self._find_matching_title("Conclusion", start), - *self._find_matching_title("Outlook", start), - *self._find_matching_meta("Annex", "annex_start", start)[0], - ] - - candidates = sorted(candidates, key=lambda c: c.document_order) - candidates = unique(candidates, key=lambda c: c.document_order) - candidates = unique(candidates, key=lambda c: c.title) - - return candidates - - @cached_property - def abstract_sentences(self) -> List[Text]: - try: - abstract_start = next( - i - for i, m in enumerate(self.content) - if isinstance(m, Meta) and m.meta_type == "abstract_start" - ) - abstract_end = next( - i - for i, m in enumerate(self.content) - if isinstance(m, Meta) and m.meta_type == "abstract_end" - ) - return [ - s - for p in self.content[abstract_start:abstract_end] - if isinstance(p, Paragraph) - for s in p.sentences - ] - except StopIteration: - pass # let's try another way - - try: - abstract_start = next( - m.document_order for m in self.bookmarks if m.title == "Abstract" - ) - abstract_sentences: List[Text] = [] - for c in self.content: - if isinstance(c, Paragraph): - abstract_sentences.extend( - s for s in c.sentences if s.document_order > abstract_start - ) - elif len(abstract_sentences) >= 5: - break - return abstract_sentences - except StopIteration: - pass # let's try another way - - return self.sentences[:10] - - @cached_property - def introduction_sentences(self) -> List[Text]: - """Includes abstract""" - introduction_end = 4 - - try: - introduction_start = [ - m.document_order for m in self.bookmarks if m.title == "Introduction" - ][-1] - - for m in self.bookmarks: - if m.title != "Introduction" and m.document_order > introduction_start: - introduction_end = m.document_order - break - except IndexError: - pass - - try: - introduction_end = max( - next( - i - for i, m in enumerate(self.content) - if isinstance(m, Meta) and m.meta_type == "abstract_end" - ), - introduction_end, - ) - except StopIteration: - pass - - introduction_sentences: List[Text] = [] - for c in self.content: - if isinstance(c, Paragraph): - introduction_sentences.extend( - s for s in c.sentences if s.document_order < introduction_end - ) - - return introduction_sentences - - @cached_property - def conclusion_sentences(self) -> List[Text]: - try: - conclusion_start = next( - m.document_order for m in self.bookmarks if m.title == "Conclusion" - ) - conclusion_sentences: List[Text] = [] - for c in self.content: - if isinstance(c, Paragraph): - conclusion_sentences.extend( - s for s in c.sentences if s.document_order > conclusion_start - ) - elif len(conclusion_sentences) >= 8: - break - return conclusion_sentences - except StopIteration: - return self.sentences[-10:] - - def _parse_author(self, raw: Tag) -> Author: - return Author( - name=( - clean(" ".join(name.get_text() for name in raw.persName)) - if raw.persName - else None - ), - orcid=self._element_to_text(raw.find(attrs={"type": "ORCID"})), - email=self._element_to_text(raw.email), - corresponding=raw.get("role") == "corresp", - affiliations=[ - self._parse_affiliation(aff) for aff in raw.find_all("affiliation") - ], - coordinates=raw.persName.get("coords") if raw.persName else None, - ) - - def _parse_affiliation(self, raw: Tag) -> Affiliation: - return Affiliation( - institutions=[ - self._element_to_text(v) - for v in raw.find_all("orgName", attrs={"type": "institution"}) - ], - departments=[ - self._element_to_text(v) - for v in raw.find_all("orgName", attrs={"type": "department"}) - ], - laboratories=[ - self._element_to_text(v) - for v in raw.find_all("orgName", attrs={"type": "laboratory"}) - ], - country=self._element_to_text(raw.address.country) - if raw.address and raw.address.country - else None, - settlement=self._element_to_text(raw.address.settlement) - if raw.address and raw.address.settlement - else None, - ) - - def _get_elements(self, raw: Tag) -> List[Element]: - results: List[Element] = [] - - for r in raw.find_all(["abstract", "div", "head", "p"]): - if r.name == "abstract": - results.append(Meta(meta_type="abstract_start")) - results.extend(self._get_primitives(r)) - results.append(Meta(meta_type="abstract_end")) - elif r.name == "div" and r.get("type") == "acknowledgement": - results.append(Meta(meta_type="acknowledgements_start")) - results.extend(self._get_primitives(r)) - results.append(Meta(meta_type="acknowledgements_end")) - elif r.name == "div" and r.get("type") == "annex": - results.append(Meta(meta_type="annex_start")) - results.extend(self._get_primitives(r)) - results.append(Meta(meta_type="annex_end")) - elif not r.find_parents( - ["abstract", "div", "figure"] - ): # figures are omitted as well - results.extend(self._get_primitives(r)) - - return results - - def _get_primitives(self, raw: Tag) -> List[Element]: - results: List[Element] = [] - - for r in raw.find_all(["head", "p"]): - if r.name == "head" and r.get("coords") and r.get_text(): - text = self._parse_text(r) - if text: - results.append(Title(text=text)) - elif r.name == "p" and r.find_all("s"): - results.append( - Paragraph( - sentences=[ - t - for t in [ - self._parse_text(sentence, ignore_partial=True) - for sentence in r.find_all("s") - if sentence.get_text() - ] - if t - ] - ) - ) - - return results - - def _element_to_text( - self, element: Optional[NavigableString], default: Any = None - ) -> Union[str, Any]: - return ( - clean(element.get_text(separator=" ", strip=True)) if element else default - ) - - def _parse_text(self, raw: Tag, ignore_partial: bool = False) -> Optional[Text]: - text = raw.get_text() - if text is None: - return None - - if self.aggressive_cleaning_enabled: - filtered = filter_sentences( - text, - THIS_FOLDER / "templates.yaml", - inverse=True, - ignore_partial=ignore_partial, - ) - text = " ".join(filtered) - - if not text.strip(): - return None - - return Text( - content=text, - document_order=self._generate_document_order_id(), - coordinates=raw.get("coords"), - ) - - def _generate_document_order_id(self) -> int: - value = self._document_order_counter - self._document_order_counter += 1 - return value - - def _find_matching_meta( - self, - bookmark_title: Optional[BookmarkTitle], - meta_type: MetaType, - start: int = 0, - ) -> Tuple[List[Bookmark], int]: - for i, e in enumerate(self.content[start:], start=start): - if not isinstance(e, Meta): - continue - - if e.meta_type == meta_type: - for e_next in self.content[i + 1 :]: - if isinstance(e_next, Title): - return [ - Bookmark( - title=bookmark_title, - original_title=e_next.text.content, - document_order=e_next.text.document_order, - coordinates=e_next.text.coordinates, - ) - ] if bookmark_title else [], i - if isinstance(e_next, Paragraph) and e_next.sentences: - return [ - Bookmark( - title=bookmark_title, - original_title="", - document_order=e_next.sentences[0].document_order, - coordinates=e_next.sentences[0].coordinates, - ) - ] if bookmark_title else [], i - - return [], 0 - - def _find_matching_title( - self, - bookmark_title: BookmarkTitle, - start: int = 0, - ) -> List[Bookmark]: - return [ - Bookmark( - title=bookmark_title, - original_title=e.text.content, - document_order=e.text.document_order, - coordinates=e.text.coordinates, - ) - for e in self.content[start:] - if isinstance(e, Title) - and self._match_title(e.text.content, titles_of_interest[bookmark_title]) - ] - - @staticmethod - def _match_title(title: str, keywords: Tuple[Union[Pattern, str], ...]) -> bool: - title = PublicationTEI._process_section_title(title) - - if any( - PublicationTEI._process_section_title(k) in title - for k in keywords - if isinstance(k, str) - ): - return True - - return any(k.match(title) for k in keywords if isinstance(k, Pattern)) - - @staticmethod - @lru_cache(maxsize=2000) - def _process_section_title(title: str) -> str: - title = re.sub(r"^\s*[ivx]+[.,)]? ", "", title) # Remove leading Roman-numerals - title = clean(title, convert_to_ascii=True) - title = re.sub( - r"[^a-zA-Z ]", "", title - ) # Remove everything but letters and spaces (hypens are also removed) - title_tokens = lemmatize_text(title) - title = " ".join(t for t in title_tokens if t.strip()) - return title diff --git a/src/great_ai/utilities/publication_tei/templates.yaml b/src/great_ai/utilities/publication_tei/templates.yaml deleted file mode 100644 index 230f354..0000000 --- a/src/great_ai/utilities/publication_tei/templates.yaml +++ /dev/null @@ -1,837 +0,0 @@ -included in the article's creative commons: 3827 -is a open access article distributed: 3822 -a open access article distributed under: 3761 -open access article distributed under the: 3761 -access article distributed under the terms: 3761 -the terms and conditions of the: 3688 -this article is a open access: 3678 -article is a open access article: 3678 -article distributed under the terms and: 3678 -distributed under the terms and conditions: 3678 -under the terms and conditions of: 3678 -terms and conditions of the creative: 3678 -and conditions of the creative commons: 3678 -conditions of the creative commons attribution: 3678 -in the article's creative commons licence: 3503 -authors declare that they have no: 3315 -https :// doi.org / NUMBER /: 3189 -the authors declare that they have: 3015 -remains neutral with regard to jurisdictional: 2464 -neutral with regard to jurisdictional claims: 2464 -with regard to jurisdictional claims in: 2464 -regard to jurisdictional claims in published: 2462 -to jurisdictional claims in published maps: 2462 -the original author ( s ): 2461 -jurisdictional claims in published maps and: 2461 -claims in published maps and institutional: 2461 -author ( s ) and the: 2460 -original author ( s ) and: 2456 -in published maps and institutional affiliations: 2415 -nature remains neutral with regard to: 2399 -springer nature remains neutral with regard: 2392 -( s ) and the source: 2353 -of the creative commons attribution CITATION: 2302 -", provide a link to the": 2299 -the source , provide a link: 2293 -source , provide a link to: 2293 -you give appropriate credit to the: 2292 -give appropriate credit to the original: 2292 -appropriate credit to the original author: 2292 -and indicate if changes were made: 2289 -credit to the original author (: 2281 -to the original author ( s: 2281 -provide a link to the creative: 2280 -a link to the creative commons: 2280 -s ) and the source ,: 2279 -) and the source , provide: 2277 -and the source , provide a: 2277 -", and indicate if changes were": 2271 -at https :// doi.org / NUMBER: 2195 -creative commons attribution NUMBER international license: 2158 -", distribution and reproduction in any": 2158 -distribution and reproduction in any medium: 2157 -can be found online at https: 2106 -be found online at https ://: 2106 -article can be found online at: 2069 -license , which permits use ,: 2063 -this article can be found online: 2010 -online version contains supplementary material available: 1980 -supplementary material available at https ://: 1979 -version contains supplementary material available at: 1978 -contains supplementary material available at https: 1973 -the online version contains supplementary material: 1957 -note springer nature remains neutral with: 1956 -other third party material in this: 1945 -to view a copy of this: 1945 -or other third party material in: 1944 -need to obtain permission directly from: 1944 -to obtain permission directly from the: 1944 -images or other third party material: 1943 -regulation or exceeds the permitted use: 1943 -or exceeds the permitted use ,: 1943 -obtain permission directly from the copyright: 1943 -the permitted use , you will: 1942 -by statutory regulation or exceeds the: 1941 -statutory regulation or exceeds the permitted: 1941 -exceeds the permitted use , you: 1941 -use , you will need to: 1941 -", you will need to obtain": 1941 -you will need to obtain permission: 1941 -permitted use , you will need: 1940 -will need to obtain permission directly: 1940 -permitted by statutory regulation or exceeds: 1939 -publisher's note springer nature remains neutral: 1939 -and your intended use is not: 1938 -your intended use is not permitted: 1938 -not permitted by statutory regulation or: 1938 -intended use is not permitted by: 1937 -is not permitted by statutory regulation: 1937 -permission directly from the copyright holder: 1937 -use is not permitted by statutory: 1936 -material is not included in the: 1933 -the images or other third party: 1932 -if material is not included in: 1926 -in a credit line to the: 1924 -otherwise in a credit line to: 1923 -this article is included in the: 1922 -third party material in this article: 1921 -party material in this article is: 1921 -material in this article is included: 1921 -in this article is included in: 1921 -", unless indicated otherwise in a": 1920 -indicated otherwise in a credit line: 1920 -is included in the article's creative: 1919 -unless indicated otherwise in a credit: 1919 -article is included in the article's: 1918 -a credit line to the material: 1914 -not included in the article's creative: 1907 -is not included in the article's: 1906 -is licensed under a creative commons: 1898 -authors declare no conflict of interest: 1883 -and reproduction in any medium or: 1870 -reproduction in any medium or format: 1870 -in any medium or format ,: 1870 -any medium or format , as: 1870 -medium or format , as long: 1870 -or format , as long as: 1870 -format , as long as you: 1869 -", as long as you give": 1869 -as long as you give appropriate: 1869 -long as you give appropriate credit: 1869 -as you give appropriate credit to: 1869 -", adaptation , distribution and reproduction": 1866 -adaptation , distribution and reproduction in: 1866 -use , sharing , adaptation ,: 1861 -", sharing , adaptation , distribution": 1861 -sharing , adaptation , distribution and: 1861 -NUMBER international license , which permits: 1846 -article is licensed under a creative: 1839 -this article is licensed under a: 1838 -under a creative commons attribution NUMBER: 1817 -licensed under a creative commons attribution: 1811 -a creative commons attribution NUMBER international: 1807 -view a copy of this licence: 1799 -a copy of this licence ,: 1799 -copy of this licence , visit: 1799 -of this licence , visit http: 1796 -this licence , visit http ://: 1796 -access this article is licensed under: 1794 -the authors would like to thank: 1793 -which permits use , sharing ,: 1791 -permits use , sharing , adaptation: 1790 -", which permits use , sharing": 1789 -open access this article is licensed: 1782 -international license , which permits use: 1776 -attribution NUMBER international license , which: 1773 -commons attribution NUMBER international license ,: 1772 -creative commons licence and your intended: 1769 -commons licence and your intended use: 1768 -licence and your intended use is: 1768 -the article's creative commons licence and: 1762 -article's creative commons licence and your: 1762 -creative commons licence , unless indicated: 1750 -commons licence , unless indicated otherwise: 1749 -licence , unless indicated otherwise in: 1749 -article's creative commons licence , unless: 1743 -the article's creative commons licence ,: 1742 -the authors declare no conflict of: 1742 -to this article can be found: 1714 -licence , and indicate if changes: 1698 -link to the creative commons licence: 1696 -commons licence , and indicate if: 1695 -to the creative commons licence ,: 1694 -the creative commons licence , and: 1694 -creative commons licence , and indicate: 1694 -the authors declare no competing interests: 1564 -and / or publication of this: 1530 -/ or publication of this article: 1523 -the research , authorship , and: 1410 -data to this article can be: 1395 -supplementary data to this article can: 1392 -competing financial interests or personal relationships: 1388 -research , authorship , and /: 1385 -", authorship , and / or": 1385 -authorship , and / or publication: 1385 -is available from the corresponding author: 1385 -", and / or publication of": 1384 -no known competing financial interests or: 1382 -known competing financial interests or personal: 1382 -declare that they have no known: 1377 -that they have no known competing: 1376 -of the creative commons attribution (: 1372 -have no known competing financial interests: 1371 -financial interests or personal relationships that: 1371 -the creative commons attribution ( cc: 1370 -creative commons attribution ( cc by: 1370 -they have no known competing financial: 1370 -to influence the work reported in: 1369 -influence the work reported in this: 1369 -interests or personal relationships that could: 1368 -appeared to influence the work reported: 1368 -or personal relationships that could have: 1367 -have appeared to influence the work: 1367 -could have appeared to influence the: 1364 -relationships that could have appeared to: 1363 -that could have appeared to influence: 1363 -personal relationships that could have appeared: 1362 -is distributed under the terms of: 1265 -relationships that could be construed as: 1259 -/ licenses / by / NUMBER: 1243 -in the absence of any commercial: 1242 -the absence of any commercial or: 1241 -commercial or financial relationships that could: 1241 -or financial relationships that could be: 1240 -absence of any commercial or financial: 1239 -any commercial or financial relationships that: 1239 -as a potential conflict of interest: 1239 -was conducted in the absence of: 1238 -of any commercial or financial relationships: 1238 -conducted in the absence of any: 1237 -financial relationships that could be construed: 1236 -be construed as a potential conflict: 1236 -the research was conducted in the: 1235 -construed as a potential conflict of: 1235 -that the research was conducted in: 1233 -research was conducted in the absence: 1233 -:// doi.org / NUMBER / s: 1220 -doi.org / NUMBER / s NUMBER: 1220 -declare that the research was conducted: 1212 -authors declare that the research was: 1204 -read and approved the final manuscript: 1196 -study is available from the corresponding: 1163 -creativecommons.org / licenses / by /: 1148 -:// creativecommons.org / licenses / by: 1147 -material available at https :// doi: 1135 -authors read and approved the final: 1074 -http :// creativecommons.org / licenses /: 1063 -that they have no conflict of: 1061 -all authors read and approved the: 1055 -requests for materials should be addressed: 1054 -for materials should be addressed to: 1054 -correspondence and requests for materials should: 1053 -and requests for materials should be: 1052 -findings of this study is available: 1046 -they have no conflict of interest: 1044 -declare that they have no conflict: 1020 -", visit http :// creat iveco": 1016 -licence , visit http :// creat: 1015 -the authors declare that the research: 993 -", on the other hand ,": 989 -available at https :// doi.org /: 925 -informed consent was obtained from all: 904 -this article is protected by copyright: 882 -found online at https :// doi: 871 -additional supporting information may be found: 866 -and moral rights for the publications: 865 -moral rights for the publications made: 865 -rights for the publications made accessible: 865 -for the publications made accessible in: 865 -is retained by the authors and: 865 -retained by the authors and /: 865 -by the authors and / or: 865 -the authors and / or other: 865 -authors and / or other copyright: 865 -and / or other copyright owners: 865 -/ or other copyright owners and: 865 -or other copyright owners and it: 865 -other copyright owners and it is: 865 -copyright owners and it is a: 865 -owners and it is a condition: 865 -and it is a condition of: 865 -it is a condition of accessing: 865 -is a condition of accessing publications: 865 -a condition of accessing publications that: 865 -condition of accessing publications that users: 865 -of accessing publications that users recognise: 865 -accessing publications that users recognise and: 865 -publications that users recognise and abide: 865 -that users recognise and abide by: 865 -users recognise and abide by the: 865 -recognise and abide by the legal: 865 -and abide by the legal requirements: 865 -abide by the legal requirements associated: 865 -by the legal requirements associated with: 865 -the legal requirements associated with this: 865 -copyright and moral rights for the: 864 -the publications made accessible in the: 864 -publications made accessible in the public: 864 -made accessible in the public portal: 864 -accessible in the public portal is: 864 -in the public portal is retained: 864 -the public portal is retained by: 864 -public portal is retained by the: 864 -portal is retained by the authors: 864 -legal requirements associated with this rights: 863 -the terms of the creative commons: 839 -and reproduction in any medium ,: 834 -reproduction in any medium , provided: 833 -declare that they have no competing: 823 -conflicts of interest with respect to: 811 -this work was supported by the: 804 -that they have no competing interests: 783 -respect to the research , authorship: 781 -with respect to the research ,: 780 -potential conflicts of interest with respect: 779 -interest with respect to the research: 776 -of article NUMBER fa of the: 775 -article NUMBER fa of the dutch: 775 -NUMBER fa of the dutch copyright: 775 -fa of the dutch copyright act: 775 -potential conflict of interest was reported: 775 -conflict of interest was reported by: 775 -no potential conflict of interest was: 773 -of interest was reported by the: 770 -in the present study , we: 767 -under the terms of article NUMBER: 766 -the terms of article NUMBER fa: 766 -terms of article NUMBER fa of: 766 -this study is available from the: 765 -support for the research , authorship: 762 -financial support for the research ,: 760 -of the dutch copyright act ,: 758 -distributed under the terms of the: 743 -publication is distributed under the terms: 742 -distributed under the terms of article: 742 -the original work is properly cited: 738 -if the publication is distributed under: 734 -the publication is distributed under the: 734 -the dutch copyright act , indicated: 734 -dutch copyright act , indicated by: 734 -copyright act , indicated by the: 734 -act , indicated by the taverne: 734 -", indicated by the taverne license": 734 -indicated by the taverne license above: 734 -by the taverne license above ,: 734 -the taverne license above , please: 734 -taverne license above , please follow: 734 -license above , please follow below: 734 -above , please follow below link: 734 -", please follow below link for": 734 -please follow below link for the: 734 -follow below link for the end: 734 -online at https :// doi.org /: 733 -of this study is available from: 731 -to the research , authorship ,: 728 -from NUMBER % to NUMBER %: 728 -CITATION , NUMBER CITATION , NUMBER: 723 -reprints and permission information is available: 722 -information is available at http ://: 722 -available at http :// www.nature.com /: 722 -is available at http :// www.nature.com: 721 -at http :// www.nature.com / reprints: 721 -terms of the creative commons attribution: 717 -and permission information is available at: 710 -permission information is available at http: 710 -provided the original work is properly: 707 -", provided the original work is": 706 -http :// www.nature.com / reprints publisher's: 706 -:// www.nature.com / reprints publisher's note: 705 -www.nature.com / reprints publisher's note springer: 705 -/ reprints publisher's note springer nature: 705 -reprints publisher's note springer nature remains: 705 -associated with this article can be: 700 -with this article can be found: 695 -available from the corresponding author upon: 691 -no potential conflicts of interest with: 685 -for the research , authorship ,: 685 -", visit http :// creativecommons.org /": 683 -visit http :// creativecommons.org / licenses: 683 -conflict of interest the authors declare: 678 -declared no potential conflicts of interest: 666 -visit http :// creat iveco mmons: 665 -for this article can be found: 658 -had no role in the design: 651 -material for this article can be: 645 -this is a open access article: 644 -the corresponding author upon reasonable request: 637 -supplementary material for this article can: 633 -from the corresponding author upon reasonable: 633 -the final version of the manuscript: 624 -the author ( s ) declared: 618 -the supplementary material for this article: 617 -in the public , commercial ,: 613 -the public , commercial , or: 612 -by the author ( s ): 599 -of interest the authors declare that: 594 -available from the corresponding author on: 589 -this article can be found ,: 587 -article can be found , in: 587 -link to the creative commons license: 585 -be found in the online version: 584 -", the aim of this study": 583 -be found online in the supporting: 583 -found online in the supporting information: 583 -", it has been shown that": 582 -we would like to thank the: 580 -supporting information may be found online: 579 -information may be found online in: 577 -may be found online in the: 577 -be found , in the online: 577 -found , in the online version: 577 -license , and indicate if changes: 576 -", in the online version ,": 574 -in the online version , at: 574 -for their contribution to the peer: 574 -their contribution to the peer review: 574 -this research did not receive any: 572 -reprints and permissions information is available: 572 -to the creative commons license ,: 571 -the creative commons license , and: 571 -creative commons license , and indicate: 571 -commons license , and indicate if: 571 -the funders had no role in: 570 -and permissions information is available at: 570 -did not receive any specific grant: 566 -agencies in the public , commercial: 566 -from the corresponding author on reasonable: 564 -in any medium , provided the: 563 -any medium , provided the original: 562 -to the peer review of this: 562 -the peer review of this work: 562 -not receive any specific grant from: 561 -from funding agencies in the public: 561 -the corresponding author on reasonable request: 561 -is available at www.nature.com / reprints: 559 -funding agencies in the public ,: 558 -permissions information is available at www.nature.com: 558 -information is available at www.nature.com /: 558 -research did not receive any specific: 552 -/ NUMBER / s NUMBER correspondence: 552 -NUMBER correspondence and requests for materials: 552 -below link for the end user: 551 -NUMBER / s NUMBER correspondence and: 551 -/ s NUMBER correspondence and requests: 551 -s NUMBER correspondence and requests for: 551 -online in the supporting information section: 550 -public , commercial , or not-for-profit: 549 -found online at https :// www.frontiersin.org: 549 -online at https :// www.frontiersin.org /: 549 -at https :// www.frontiersin.org / articles: 548 -https :// www.frontiersin.org / articles /: 548 -", distribution , and reproduction in": 546 -distribution , and reproduction in any: 546 -", and reproduction in any medium": 546 -:// www.frontiersin.org / articles / NUMBER: 545 -www.frontiersin.org / articles / NUMBER /: 544 -contribution to the peer review of: 542 -grant from funding agencies in the: 541 -use , distribution , and reproduction: 537 -", commercial , or not-for-profit sectors": 535 -received no financial support for the: 535 -no financial support for the research: 534 -receive any specific grant from funding: 529 -specific grant from funding agencies in: 529 -approved the final version of the: 529 -any specific grant from funding agencies: 528 -author ( s ) declared no: 526 -is available online at https ://: 525 -( s ) declared no potential: 525 -s ) declared no potential conflicts: 525 -) declared no potential conflicts of: 525 -article is distributed under the terms: 520 -this article is distributed under the: 518 -"% and NUMBER % , respectively": 507 -access this article is distributed under: 501 -open access this article is distributed: 498 -authors declare that there is no: 487 -the following is available online at: 485 -NUMBER % , NUMBER % ,: 484 -authors would like to thank the: 479 -is a open access article under: 477 -a open access article under the: 477 -medium , provided the original work: 473 -/ NUMBER /) , which permits: 471 -the online version , at doi: 466 -interest was reported by the author: 464 -supplementary material associated with this article: 452 -material associated with this article can: 452 -we would like to thank all: 449 -http :// creat iveco mmons.org /: 448 -:// creat iveco mmons.org / licen: 446 -creat iveco mmons.org / licen ses: 446 -iveco mmons.org / licen ses /: 446 -following is available online at https: 446 -available online at https :// www.mdpi.com: 445 -online at https :// www.mdpi.com /: 445 -union's horizon NUMBER research and innovation: 443 -during the current study is available: 443 -european union's horizon NUMBER research and: 441 -the netherlands organization for scientific research: 441 -the authors declare that there is: 441 -NUMBER international license ( http ://: 440 -of the authors and do not: 439 -www.mdpi.com / article / NUMBER /: 438 -NUMBER CITATION , NUMBER CITATION ,: 436 -access article under the terms of: 435 -article under the terms of the: 435 -/ by / NUMBER /) ,: 435 -by / NUMBER /) , which: 435 -/ licen ses / by /: 434 -licen ses / by / NUMBER: 434 -open access article under the terms: 434 -current study is available from the: 434 -the current study is available from: 433 -", which permits unrestricted use ,": 431 -mmons.org / licen ses / by: 429 -have no conflicts of interest to: 427 -which permits unrestricted use , distribution: 427 -", there is a need for": 425 -permits unrestricted use , distribution ,: 425 -unrestricted use , distribution , and: 425 -have read and agreed to the: 423 -authors have read and agreed to: 422 -any medium , provided you give: 422 -medium , provided you give appropriate: 422 -", provided you give appropriate credit": 422 -provided you give appropriate credit to: 422 -was reported by the author (: 422 -reported by the author ( s: 422 -NUMBER /) , which permits unrestricted: 421 -in any medium , provided you: 421 -peer review information nature communications thanks: 421 -the european union's horizon NUMBER research: 420 -and agreed to the published version: 419 -agreed to the published version of: 419 -the published version of the manuscript: 419 -to the published version of the: 418 -the authors and do not necessarily: 417 -/) , which permits unrestricted use: 417 -read and agreed to the published: 416 -those of the authors and do: 414 -all authors have read and agreed: 413 -that there is no conflict of: 411 -attribution NUMBER international license ( http: 410 -author ( s ) received no: 405 -consent was obtained from all subjects: 404 -authors have no conflicts of interest: 403 -( s ) received no financial: 403 -s ) received no financial support: 403 -) received no financial support for: 403 -NUMBER % and NUMBER % of: 400 -contributed to the article and approved: 400 -to the article and approved the: 400 -authors contributed to the article and: 397 -the author ( s ) received: 396 -the article and approved the submitted: 396 -article and approved the submitted version: 395 -informed consent statement informed consent was: 393 -accountable for all aspects of the: 389 -https :// www.mdpi.com / article /: 389 -:// www.mdpi.com / article / NUMBER: 389 -consent statement informed consent was obtained: 389 -at https :// www.mdpi.com / article: 388 -statement informed consent was obtained from: 388 -no role in the design of: 385 -) for their contribution to the: 383 -", we would like to thank": 381 -is available on request from the: 378 -had no role in study design: 377 -responsibility for the integrity of the: 375 -of the creative commons attribution NUMBER: 375 -in the supporting information section at: 374 -the supporting information section at the: 374 -supporting information section at the end: 374 -reviewer ( s ) for their: 374 -this study is available on request: 373 -all authors contributed to the article: 373 -( s ) for their contribution: 372 -s ) for their contribution to: 372 -", analysis and interpretation of data": 371 -available on request from the corresponding: 370 -in accordance with the ethical standards: 370 -the manuscript for important intellectual content: 368 -for all aspects of the work: 367 -role in the design of the: 367 -authors declare no conflicts of interest: 366 -", analysis , and interpretation of": 365 -on request from the corresponding author: 364 -be accountable for all aspects of: 362 -obtained from all subjects involved in: 361 -funders had no role in the: 360 -for scientific research ( nwo ): 359 -the creative commons attribution NUMBER international: 359 -was obtained from all subjects involved: 359 -authors and do not necessarily represent: 358 -to be accountable for all aspects: 356 -original work is properly cited ,: 355 -data in the writing of the: 354 -no role in study design ,: 353 -from all subjects involved in the: 353 -all subjects involved in the study: 351 -provenance and peer review not commissioned: 346 -study is available on request from: 346 -there is no conflict of interest: 346 -wrote the first draft of the: 345 -NUMBER / NUMBER / NUMBER /: 345 -written informed consent to participate in: 341 -commons attribution NUMBER international license (: 341 -interpretation of data in the writing: 339 -of data in the writing of: 339 -ses / by / NUMBER /: 337 -we would also like to thank: 336 -the integrity of the data and: 335 -and peer review not commissioned externally: 334 -of this study was to assess: 334 -peer review not commissioned externally peer: 332 -review not commissioned externally peer reviewed: 332 -declare that there is no conflict: 331 -role in study design , data: 328 -written informed consent was obtained from: 327 -no conflicts of interest to declare: 326 -consent to participate in this study: 326 -in the article's creative commons license: 326 -do not necessarily represent those of: 325 -( http :// creativecommons.org / licenses: 324 -the authors have no conflicts of: 322 -informed consent to participate in this: 322 -with the ethical standards of the: 322 -license ( http :// creativecommons.org /: 322 -solely those of the authors and: 322 -and do not necessarily represent those: 322 -in this article is solely those: 320 -this article is solely those of: 320 -article is solely those of the: 320 -integrity of the data and the: 319 -of the data and the accuracy: 319 -the data and the accuracy of: 319 -accordance with the ethical standards of: 319 -data and the accuracy of the: 318 -during the conduct of the study: 317 -the creative commons attribution CITATION ,: 317 -expressed in this article is solely: 316 -and approved the final version of: 315 -publisher's note all claims expressed in: 315 -note all claims expressed in this: 315 -all claims expressed in this article: 315 -claims expressed in this article is: 315 -not necessarily represent those of their: 315 -necessarily represent those of their affiliated: 315 -represent those of their affiliated organizations: 315 -or those of the publisher ,: 315 -those of the publisher , the: 314 -of the publisher , the editors: 314 -in the writing of the manuscript: 313 -affiliated organizations , or those of: 312 -organizations , or those of the: 312 -or claim that may be made: 312 -claim that may be made by: 312 -that may be made by its: 312 -is not guaranteed or endorsed by: 312 -not guaranteed or endorsed by the: 312 -guaranteed or endorsed by the publisher: 312 -authors would like to thank all: 311 -related to this article can be: 311 -those of their affiliated organizations ,: 311 -of their affiliated organizations , or: 311 -their affiliated organizations , or those: 311 -", or those of the publisher": 311 -any product that may be evaluated: 311 -this article , or claim that: 311 -article , or claim that may: 311 -", or claim that may be": 311 -product that may be evaluated in: 310 -may be made by its manufacturer: 310 -the accuracy of the data analysis: 309 -", there is a need to": 309 -of the creative commons attribution license: 309 -that may be evaluated in this: 309 -may be evaluated in this article: 309 -be made by its manufacturer ,: 309 -made by its manufacturer , is: 309 -by its manufacturer , is not: 309 -its manufacturer , is not guaranteed: 309 -manufacturer , is not guaranteed or: 309 -", is not guaranteed or endorsed": 309 -", there is a lack of": 308 -the authors declare no conflicts of: 308 -be evaluated in this article ,: 308 -evaluated in this article , or: 308 -in this article , or claim: 308 -the data in the study and: 306 -in this paper , we present: 306 -information may be found in the: 305 -this study was to investigate the: 305 -", on the one hand ,": 305 -may be found in the online: 304 -is not publicly available due to: 304 -authors report no conflicts of interest: 304 -online version , at doi NUMBER: 303 -version , at doi NUMBER /: 303 -supporting information may be found in: 302 -in the public , commercial or: 301 -from any funding agency in the: 300 -any funding agency in the public: 300 -were in accordance with the ethical: 300 -the publisher , the editors and: 300 -publisher , the editors and the: 300 -", the editors and the reviewers": 300 -funding agency in the public ,: 299 -agency in the public , commercial: 299 -found in the online version of: 297 -there is no conflicts of interest: 295 -can be found in the online: 295 -article can be found in the: 291 -use , distribution and reproduction in: 289 -the online version of this article: 287 -before it is published in its: 287 -it is published in its final: 287 -to submit the manuscript for publication: 286 -international license ( http :// creativecommons.org: 285 -is published in its final form: 285 -decision to submit the manuscript for: 284 -", which permits use , distribution": 284 -which permits use , distribution and: 284 -permits use , distribution and reproduction: 284 -", as shown in fig. NUMBER": 284 -errors may be discovered which could: 284 -may be discovered which could affect: 284 -be discovered which could affect the: 284 -discovered which could affect the content: 284 -which could affect the content ,: 284 -could affect the content , and: 284 -affect the content , and all: 284 -the content , and all legal: 284 -content , and all legal disclaimers: 284 -all legal disclaimers that apply to: 283 -legal disclaimers that apply to the: 283 -disclaimers that apply to the journal: 283 -design , data collection and analysis: 282 -", and all legal disclaimers that": 282 -and all legal disclaimers that apply: 282 -from the european union's horizon NUMBER: 280 -and the other , anonymous ,: 280 -is a pdf file of a: 278 -this article can be found in: 277 -helsinki declaration and its later amendments: 276 -NUMBER helsinki declaration and its later: 275 -the NUMBER helsinki declaration and its: 274 -during the production process , errors: 274 -the production process , errors may: 274 -production process , errors may be: 274 -process , errors may be discovered: 274 -this is a pdf file of: 273 -review before it is published in: 273 -", errors may be discovered which": 273 -will undergo additional copyediting , typesetting: 271 -and review before it is published: 271 -note that , during the production: 271 -that , during the production process: 271 -", during the production process ,": 271 -with the NUMBER helsinki declaration and: 270 -this version will undergo additional copyediting: 270 -version will undergo additional copyediting ,: 270 -undergo additional copyediting , typesetting and: 270 -additional copyediting , typesetting and review: 270 -copyediting , typesetting and review before: 270 -", typesetting and review before it": 270 -typesetting and review before it is: 270 -published in its final form ,: 270 -in its final form , but: 270 -its final form , but we: 270 -final form , but we is: 270 -form , but we is providing: 270 -", but we is providing this": 270 -but we is providing this version: 270 -we is providing this version to: 270 -is providing this version to give: 270 -providing this version to give early: 270 -this version to give early visibility: 270 -version to give early visibility of: 270 -to give early visibility of the: 270 -give early visibility of the article: 270 -please note that , during the: 270 -netherlands organization for scientific research (: 269 -acute respiratory syndrome coronavirus NUMBER CITATION: 268 -a pdf file of a article: 268 -pdf file of a article that: 268 -file of a article that has: 268 -of a article that has undergone: 268 -a article that has undergone enhancements: 268 -article that has undergone enhancements after: 268 -that has undergone enhancements after acceptance: 268 -has undergone enhancements after acceptance ,: 268 -undergone enhancements after acceptance , such: 268 -enhancements after acceptance , such as: 268 -after acceptance , such as the: 268 -acceptance , such as the addition: 268 -as the addition of a cover: 268 -the addition of a cover page: 268 -addition of a cover page and: 268 -of a cover page and metadata: 268 -a cover page and metadata ,: 268 -cover page and metadata , and: 268 -page and metadata , and formatting: 268 -and metadata , and formatting for: 268 -metadata , and formatting for readability: 268 -", and formatting for readability ,": 268 -and formatting for readability , but: 268 -formatting for readability , but it: 268 -for readability , but it is: 268 -readability , but it is not: 268 -but it is not yet the: 268 -it is not yet the definitive: 268 -is not yet the definitive version: 268 -not yet the definitive version of: 268 -yet the definitive version of record: 268 -and its later amendments or comparable: 267 -its later amendments or comparable ethical: 267 -the netherlands organisation for scientific research: 265 -analysis and interpretation of data ,: 265 -NUMBER https :// doi.org / NUMBER: 265 -later amendments or comparable ethical standards: 265 -the public , commercial or not-for-profit: 265 -the aim of this paper is: 263 -"% , NUMBER % and NUMBER": 262 -the other , anonymous , reviewer: 261 -and with the NUMBER helsinki declaration: 260 -human participants were reviewed and approved: 259 -participants were reviewed and approved by: 259 -involving human participants were reviewed and: 258 -studies involving human participants were reviewed: 257 -and analysis , decision to publish: 257 -it should be noted that the: 257 -in the decision to publish the: 256 -study design , data collection ,: 256 -found in the online version at: 256 -the authors report no conflicts of: 255 -under the terms of the creat: 255 -the terms of the creat ive: 255 -terms of the creat ive commo: 255 -of the creat ive commo ns: 255 -the creat ive commo ns attri: 255 -they have no conflicts of interest: 255 -data presented in this study is: 255 -participants provided their written informed consent: 254 -their written informed consent to participate: 254 -the decision to publish the results: 254 -the data presented in this study: 254 -provided their written informed consent to: 253 -CITATION license , which permits others: 252 -license , which permits others to: 252 -all authors approved the final version: 251 -that apply to the journal pertain: 251 -anonymous , reviewer ( s ): 251 -study is included in the article: 250 -other , anonymous , reviewer (: 250 -", anonymous , reviewer ( s": 250 -", reviewer ( s ) for": 250 diff --git a/src/great_ai/utilities/publication_tei/titles_of_interest.py b/src/great_ai/utilities/publication_tei/titles_of_interest.py deleted file mode 100644 index db534d0..0000000 --- a/src/great_ai/utilities/publication_tei/titles_of_interest.py +++ /dev/null @@ -1,145 +0,0 @@ -import re -from typing import Dict, Pattern, Tuple, Union - -from .models import BookmarkTitle - -# partly inspired by https://github.com/KMCS-NII/AASC/blob/master/section_classify/section_classify.pl -titles_of_interest: Dict[BookmarkTitle, Tuple[Union[str, Pattern], ...]] = { - "Abstract": ("abstract",), - "Author contribution": ( - "author contribution", - "credit authorship contribution statement", - "credit author statement", - "corresponding author", - ), - "Introduction": ( - "introduction", - "preliminaries", - "research question", - "objective", - "purpose", - re.compile(r"^aim$"), - "proposition", - "what this study adds", - "what does this study add", - ), - "Background": ( - "background", - "previous", - "prior", - "relevant work", - "state of the art", - "related work", - "research in context", - "earlier work", - "related work", - "problem", - "challenge", - "goal", - "literature review", - re.compile(r"^review$"), - re.compile(r"^existing"), - "comparison with existing literature", - "literature search", - "comparison with other studies", - "study area", - ), - "Methods": ( - "method", - "approach", - re.compile(r"^our"), - "proposed", - "study design", - "procedure", - "formulation", - "methodology", - "research design", - "study protocol", - "design", - re.compile(r"experimental (?!result)"), - "overview", - "approach", - "system", - "definition", - "algorithm", - "model", - "setup", - "implementation", - "evaluation metric", - "scheme", - ), - "Results": ( - "result", - "data", - "outcome", - "statistic", - "statistics", - "statistical analysis", - "analysis", - "measure", - "experiment", - "finding", - "hypothesis testing", - "proof of proposition", - "proof of concept", - "report", - "implication of all the available evidence", - "contribution", - "evaluation", - "performance", - "demonstration", - "demonstrating", - "example", - "study", - "studies", - "qualitative comparison", - "quantitative comparison", - "validation", - ), - "Discussion": ( - "discussion", - "recommendation", - "implication", - "findings", - "impact", - "limitation", - "assessment", - "quality assessment", - "interpretation", - "contribution", - "explanation", - ), - "Outlook": ( - "outlook", - "future research", - "future", - "follow up", - "remaining", - "prospect", - "perspective", - ), - "Conclusion": ( - "conclusion", - "conclude", - "concluding", - "concluding", - "final remark", - "remark", - "conclusie", - ), - "Conflict of interest": ( - "compete interest", - "funding support and author disclosure", - "conflict of interest", - "disclosure statement", - "role of the funding source", - "disclaimer", - "conflicting interest", - "declaration of interest", - "risk of bias", - "funding", - "disclosure", - "declaration", - ), - "Acknowledgement": ("acknowledgement", "acknowledgment"), -} diff --git a/tests/utilities/test_lemmatize_text.py b/tests/utilities/test_lemmatize_text.py deleted file mode 100644 index 315cf85..0000000 --- a/tests/utilities/test_lemmatize_text.py +++ /dev/null @@ -1,139 +0,0 @@ -import unittest - -from src.great_ai.utilities import lemmatize_text - - -class TestLemmatizeText(unittest.TestCase): - def test_simple(self) -> None: - text = "The state-of-the-art could not be improved, however we managed to create a less resource-intensive implementation of it." - - lemmatized = [ - "the", - "state", - "-", - "of", - "-", - "the", - "-", - "art", - "could", - "not", - "be", - "improve", - ",", - "however", - "we", - "manage", - "to", - "create", - "a", - "less", - "resource", - "-", - "intensive", - "implementation", - "of", - "it", - ".", - ] - - lemmatized_pos = [ - "the_DET", - "state_NOUN", - "-_PUNCT", - "of_ADP", - "-_PUNCT", - "the_DET", - "-_PUNCT", - "art_NOUN", - "could_AUX", - "not_PART", - "be_AUX", - "improve_VERB", - ",_PUNCT", - "however_ADV", - "we_PRON", - "manage_VERB", - "to_PART", - "create_VERB", - "a_DET", - "less_ADV", - "resource_NOUN", - "-_PUNCT", - "intensive_ADJ", - "implementation_NOUN", - "of_ADP", - "it_PRON", - "._PUNCT", - ] - - lemmatized_neg = [ - "the", - "state", - "-", - "of", - "-", - "the", - "-", - "art", - "could", - "not", - "NOT_be", - "NOT_improve", - "NOT_,", - "however", - "we", - "manage", - "to", - "create", - "a", - "less", - "resource", - "-", - "intensive", - "implementation", - "of", - "it", - ".", - ] - - lemmatized_pos_neg = [ - "the_DET", - "state_NOUN", - "-_PUNCT", - "of_ADP", - "-_PUNCT", - "the_DET", - "-_PUNCT", - "art_NOUN", - "could_AUX", - "not_PART", - "NOT_be_AUX", - "NOT_improve_VERB", - "NOT_,_PUNCT", - "however_ADV", - "we_PRON", - "manage_VERB", - "to_PART", - "create_VERB", - "a_DET", - "less_ADV", - "resource_NOUN", - "-_PUNCT", - "intensive_ADJ", - "implementation_NOUN", - "of_ADP", - "it_PRON", - "._PUNCT", - ] - - assert lemmatize_text(text) == lemmatized - assert lemmatize_text(text, add_part_of_speech=True) == lemmatized_pos - assert lemmatize_text(text, add_negation=True) == lemmatized_neg - self.assertEqual( - lemmatize_text(text, add_negation=True, add_part_of_speech=True), - lemmatized_pos_neg, - ) - - def test_empty(self) -> None: - assert lemmatize_text("") == [] diff --git a/tests/utilities/test_lemmatize_token.py b/tests/utilities/test_lemmatize_token.py deleted file mode 100644 index 308e612..0000000 --- a/tests/utilities/test_lemmatize_token.py +++ /dev/null @@ -1,19 +0,0 @@ -import unittest - -from src.great_ai.utilities import lemmatize_token, nlp - - -class TestLemmatizeToken(unittest.TestCase): - def test_simple(self) -> None: - token = nlp("Center")[0] - - assert lemmatize_token(token) == "centre" - assert lemmatize_token(token, add_negation=True) == "centre" - assert lemmatize_token(token, add_part_of_speech=True) == "centre_NOUN" - - def test_punctuation(self) -> None: - token = nlp("This.")[1] - - assert lemmatize_token(token) == "." - assert lemmatize_token(token, add_negation=True) == "." - assert lemmatize_token(token, add_part_of_speech=True) == "._PUNCT" diff --git a/tests/utilities/test_publication_tei.py b/tests/utilities/test_publication_tei.py deleted file mode 100644 index 35c84c6..0000000 --- a/tests/utilities/test_publication_tei.py +++ /dev/null @@ -1,67 +0,0 @@ -import unittest -from pathlib import Path - -from src.great_ai.utilities import PublicationTEI - -from .data.parsed import ( - abstract, - authors, - bookmarks, - conclusion, - content, - introduction, - metadata, - sentences, -) - -DATA_PATH = Path(__file__).parent.resolve() / "data" - - -class TestPublicationTEI(unittest.TestCase): - test_xml: str - - @classmethod - def setUpClass(cls) -> None: - with open( - DATA_PATH / "10.1136_bmjspcare-2021-003026.pdf.tei.xml", encoding="utf-8" - ) as f: - cls.test_xml = f.read() - - def test_metadata_extraction(self) -> None: - assert PublicationTEI(self.test_xml).publication_metadata == metadata - - def test_authors(self) -> None: - assert PublicationTEI(self.test_xml).authors == authors - - def test_content(self) -> None: - assert PublicationTEI(self.test_xml).content == content - - def test_sentences(self) -> None: - assert PublicationTEI(self.test_xml).sentences == sentences - - def test_bookmarks(self) -> None: - assert PublicationTEI(self.test_xml).bookmarks == bookmarks - - def test_abstract(self) -> None: - assert PublicationTEI(self.test_xml).abstract_sentences == abstract - - def test_introduction(self) -> None: - self.assertEqual( - PublicationTEI(self.test_xml).introduction_sentences, introduction - ) - - def test_conclusion(self) -> None: - assert PublicationTEI(self.test_xml).conclusion_sentences == conclusion - - def test_empty1(self) -> None: - tei = PublicationTEI("") - tei.publication_metadata, tei.publication_metadata, tei.authors, tei.content, tei.sentences - - def test_empty2(self) -> None: - tei = PublicationTEI("") - tei.publication_metadata, tei.publication_metadata, tei.authors, tei.content, tei.sentences - - def test_missing_fields(self) -> None: - with open(DATA_PATH / "bad.tei.xml", encoding="utf-8") as f: - tei = PublicationTEI(f.read()) - tei.publication_metadata, tei.publication_metadata, tei.authors, tei.content, tei.sentences