Remove Spacy and Publication TEI
This commit is contained in:
parent
e16db17db0
commit
46c99a06f9
30 changed files with 5 additions and 2412 deletions
|
|
@ -19,9 +19,6 @@ RUN python3 -m pip --no-cache-dir install --upgrade pip &&\
|
||||||
pip install --no-cache-dir ./great_ai &&\
|
pip install --no-cache-dir ./great_ai &&\
|
||||||
rm -rf great_ai
|
rm -rf great_ai
|
||||||
|
|
||||||
# great_ai.utilities.nlp depends on this
|
|
||||||
RUN pip3 install --no-cache-dir en-core-web-sm@https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl
|
|
||||||
|
|
||||||
HEALTHCHECK \
|
HEALTHCHECK \
|
||||||
--interval=60s \
|
--interval=60s \
|
||||||
--timeout=60s \
|
--timeout=60s \
|
||||||
|
|
|
||||||
|
|
@ -1,32 +1,14 @@
|
||||||
# **S**coutinScience **U**tilitie**S** for text processing [](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml)
|
# **S**coutinScience **U**tilitie**S** for text processing [](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml)
|
||||||
|
|
||||||
> amogus
|
|
||||||
|
|
||||||
## Exports
|
## Exports
|
||||||
|
|
||||||
- [clean](src/sus/clean.py)
|
- [clean](src/sus/clean.py)
|
||||||
- [unique](src/sus/unique.py)
|
- [unique](src/sus/unique.py)
|
||||||
- [parallel_map](src/sus/parallel_map.py)
|
- [parallel_map](src/sus/parallel_map.py)
|
||||||
- [match_names](src/sus/match_names/match_names.py)
|
- [lemmatize](src/sus/lemmatize.py)
|
||||||
- [evaluate_ranking](src/sus/evaluate_ranking/evaluate_ranking.py)
|
- [evaluate_ranking](src/sus/evaluate_ranking/evaluate_ranking.py)
|
||||||
- [get_sentences](src/sus/get_sentences.py)
|
- [get_sentences](src/sus/get_sentences.py)
|
||||||
|
|
||||||
### Requires loading spacy model
|
|
||||||
|
|
||||||
> This is automatic but will require some time.
|
|
||||||
|
|
||||||
> Add this to the Dockerfile for caching the spaCy model:
|
|
||||||
>
|
|
||||||
> ```docker
|
|
||||||
> RUN pip install --no-cache-dir en-core-web-sm@https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl
|
|
||||||
> ```
|
|
||||||
|
|
||||||
- [publication TEI](src/sus/publication_tei/publication_tei.py)
|
|
||||||
- [lemmatize_text](src/sus/lemmatize_text.py)
|
|
||||||
- [lemmatize_token](src/sus/lemmatize_token.py)
|
|
||||||
- [spacy model (nlp)](src/sus/nlp.py)
|
|
||||||
- [filter_sentences](src/sus/matcher/filter_sentences.py)
|
|
||||||
|
|
||||||
## Development
|
## Development
|
||||||
|
|
||||||
- Optional booleans must have a default value of `False`.
|
- Optional booleans must have a default value of `False`.
|
||||||
|
|
|
||||||
|
|
@ -23,9 +23,6 @@ install_requires =
|
||||||
unidecode >= 1.3.0
|
unidecode >= 1.3.0
|
||||||
multiprocess >= 0.70.0.0
|
multiprocess >= 0.70.0.0
|
||||||
tqdm >= 4.0.0
|
tqdm >= 4.0.0
|
||||||
beautifulsoup4 >= 4.10.0
|
|
||||||
lxml >= 4.6.0
|
|
||||||
spacy >= 3.3.0
|
|
||||||
scikit-learn == 1.1.1
|
scikit-learn == 1.1.1
|
||||||
matplotlib >= 3.5.0
|
matplotlib >= 3.5.0
|
||||||
numpy >= 1.22.0
|
numpy >= 1.22.0
|
||||||
|
|
@ -34,7 +31,6 @@ install_requires =
|
||||||
langdetect >= 1.0.9
|
langdetect >= 1.0.9
|
||||||
tinydb >= 4.7.0
|
tinydb >= 4.7.0
|
||||||
pandas >= 1.4.0
|
pandas >= 1.4.0
|
||||||
pyaml >= 21.0.0
|
|
||||||
boto3 >= 1.23.0
|
boto3 >= 1.23.0
|
||||||
fastapi >= 0.70.0
|
fastapi >= 0.70.0
|
||||||
plotly >= 5.8.0
|
plotly >= 5.8.0
|
||||||
|
|
|
||||||
|
|
@ -1,14 +1,10 @@
|
||||||
from .clean import clean
|
from .clean import clean
|
||||||
|
from .parallel_map import parallel_map
|
||||||
|
from .unique import unique
|
||||||
from .config_file import ConfigFile, ParseError
|
from .config_file import ConfigFile, ParseError
|
||||||
from .evaluate_ranking import evaluate_ranking
|
|
||||||
from .get_sentences import get_sentences
|
from .get_sentences import get_sentences
|
||||||
from .language import english_name_of_language, is_english, predict_language
|
from .language import english_name_of_language, is_english, predict_language
|
||||||
from .lemmatize_text import lemmatize_text
|
from .lemmatize import lemmatize
|
||||||
from .lemmatize_token import lemmatize_token
|
|
||||||
from .logger import get_logger
|
from .logger import get_logger
|
||||||
|
from .evaluate_ranking import evaluate_ranking
|
||||||
from .match_names import match_names
|
from .match_names import match_names
|
||||||
from .matcher import fast_tokenize, filter_sentences, normalize
|
|
||||||
from .nlp import nlp
|
|
||||||
from .parallel_map import parallel_map
|
|
||||||
from .publication_tei import PublicationTEI
|
|
||||||
from .unique import unique
|
|
||||||
|
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
https://github.com/jenojp/negspacy
|
|
||||||
222
src/great_ai/utilities/external/negspacy/negation.py
vendored
222
src/great_ai/utilities/external/negspacy/negation.py
vendored
|
|
@ -1,222 +0,0 @@
|
||||||
import logging
|
|
||||||
|
|
||||||
from spacy.language import Language
|
|
||||||
from spacy.matcher import PhraseMatcher
|
|
||||||
from spacy.tokens import Token
|
|
||||||
|
|
||||||
from .termsets import termset
|
|
||||||
|
|
||||||
default_ts = termset("en").get_patterns()
|
|
||||||
|
|
||||||
|
|
||||||
@Language.factory(
|
|
||||||
"negex",
|
|
||||||
default_config={
|
|
||||||
"neg_termset": default_ts,
|
|
||||||
"extension_name": "negex",
|
|
||||||
"chunk_prefix": list(),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
class Negex:
|
|
||||||
"""
|
|
||||||
A spaCy pipeline component which identifies negated tokens in text.
|
|
||||||
|
|
||||||
Based on: NegEx - A Simple Algorithm for Identifying Negated Findings and Diseasesin Discharge Summaries
|
|
||||||
Chapman, Bridewell, Hanbury, Cooper, Buchanan
|
|
||||||
|
|
||||||
Parameters
|
|
||||||
----------
|
|
||||||
nlp: object
|
|
||||||
spaCy language object
|
|
||||||
termset_lang: str
|
|
||||||
language code, if using default termsets (e.g. "en" for english)
|
|
||||||
extension_name: str
|
|
||||||
defaults to "negex"; whether entity is negated is then available as ent._.negex
|
|
||||||
pseudo_negations: list
|
|
||||||
list of phrases that cancel out a negation, if empty, defaults are used
|
|
||||||
preceding_negations: list
|
|
||||||
negations that appear before an entity, if empty, defaults are used
|
|
||||||
following_negations: list
|
|
||||||
negations that appear after an entity, if empty, defaults are used
|
|
||||||
termination: list
|
|
||||||
phrases that "terminate" a sentence for processing purposes such as "but". If empty, defaults are used
|
|
||||||
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
nlp: Language,
|
|
||||||
name: str,
|
|
||||||
neg_termset: dict,
|
|
||||||
extension_name: str,
|
|
||||||
chunk_prefix: list,
|
|
||||||
):
|
|
||||||
if not Token.has_extension(extension_name):
|
|
||||||
Token.set_extension(extension_name, default=False, force=True)
|
|
||||||
|
|
||||||
ts = neg_termset
|
|
||||||
expected_keys = [
|
|
||||||
"pseudo_negations",
|
|
||||||
"preceding_negations",
|
|
||||||
"following_negations",
|
|
||||||
"termination",
|
|
||||||
]
|
|
||||||
if not set(ts.keys()) == set(expected_keys):
|
|
||||||
raise KeyError(
|
|
||||||
f"Unexpected or missing keys in 'neg_termset', expected: {expected_keys}, instead got: {list(ts.keys())}"
|
|
||||||
)
|
|
||||||
|
|
||||||
self.pseudo_negations = ts["pseudo_negations"]
|
|
||||||
self.preceding_negations = ts["preceding_negations"]
|
|
||||||
self.following_negations = ts["following_negations"]
|
|
||||||
self.termination = ts["termination"]
|
|
||||||
|
|
||||||
self.nlp = nlp
|
|
||||||
self.extension_name = extension_name
|
|
||||||
self.build_patterns()
|
|
||||||
self.chunk_prefix = list(nlp.tokenizer.pipe(chunk_prefix))
|
|
||||||
|
|
||||||
def build_patterns(self):
|
|
||||||
# efficiently build spaCy matcher patterns
|
|
||||||
self.matcher = PhraseMatcher(self.nlp.vocab, attr="LOWER")
|
|
||||||
|
|
||||||
self.pseudo_patterns = list(self.nlp.tokenizer.pipe(self.pseudo_negations))
|
|
||||||
self.matcher.add("pseudo", None, *self.pseudo_patterns)
|
|
||||||
|
|
||||||
self.preceding_patterns = list(
|
|
||||||
self.nlp.tokenizer.pipe(self.preceding_negations)
|
|
||||||
)
|
|
||||||
self.matcher.add("Preceding", None, *self.preceding_patterns)
|
|
||||||
|
|
||||||
self.following_patterns = list(
|
|
||||||
self.nlp.tokenizer.pipe(self.following_negations)
|
|
||||||
)
|
|
||||||
self.matcher.add("Following", None, *self.following_patterns)
|
|
||||||
|
|
||||||
self.termination_patterns = list(self.nlp.tokenizer.pipe(self.termination))
|
|
||||||
self.matcher.add("Termination", None, *self.termination_patterns)
|
|
||||||
|
|
||||||
def process_negations(self, doc):
|
|
||||||
"""
|
|
||||||
Find negations in doc and clean candidate negations to remove pseudo negations
|
|
||||||
|
|
||||||
Parameters
|
|
||||||
----------
|
|
||||||
doc: object
|
|
||||||
spaCy Doc object
|
|
||||||
|
|
||||||
Returns
|
|
||||||
-------
|
|
||||||
preceding: list
|
|
||||||
list of tuples for preceding negations
|
|
||||||
following: list
|
|
||||||
list of tuples for following negations
|
|
||||||
terminating: list
|
|
||||||
list of tuples of terminating phrases
|
|
||||||
|
|
||||||
"""
|
|
||||||
###
|
|
||||||
# does not work properly in spacy 2.1.8. Will incorporate after 2.2.
|
|
||||||
# Relying on user to use NER in meantime
|
|
||||||
# see https://github.com/jenojp/negspacy/issues/7
|
|
||||||
###
|
|
||||||
# if not doc.is_nered:
|
|
||||||
# raise ValueError(
|
|
||||||
# "Negations are evaluated for Named Entities found in text. "
|
|
||||||
# "Your SpaCy pipeline does not included Named Entity resolution. "
|
|
||||||
# "Please ensure it is enabled or choose a different language model that includes it."
|
|
||||||
# )
|
|
||||||
preceding = list()
|
|
||||||
following = list()
|
|
||||||
terminating = list()
|
|
||||||
|
|
||||||
matches = self.matcher(doc)
|
|
||||||
pseudo = [
|
|
||||||
(match_id, start, end)
|
|
||||||
for match_id, start, end in matches
|
|
||||||
if self.nlp.vocab.strings[match_id] == "pseudo"
|
|
||||||
]
|
|
||||||
|
|
||||||
for match_id, start, end in matches:
|
|
||||||
if self.nlp.vocab.strings[match_id] == "pseudo":
|
|
||||||
continue
|
|
||||||
pseudo_flag = False
|
|
||||||
for p in pseudo:
|
|
||||||
if start >= p[1] and start <= p[2]:
|
|
||||||
pseudo_flag = True
|
|
||||||
continue
|
|
||||||
if not pseudo_flag:
|
|
||||||
if self.nlp.vocab.strings[match_id] == "Preceding":
|
|
||||||
preceding.append((match_id, start, end))
|
|
||||||
elif self.nlp.vocab.strings[match_id] == "Following":
|
|
||||||
following.append((match_id, start, end))
|
|
||||||
elif self.nlp.vocab.strings[match_id] == "Termination":
|
|
||||||
terminating.append((match_id, start, end))
|
|
||||||
else:
|
|
||||||
logging.warnings(
|
|
||||||
f"phrase {doc[start:end].text} not in one of the expected matcher types."
|
|
||||||
)
|
|
||||||
return preceding, following, terminating
|
|
||||||
|
|
||||||
def termination_boundaries(self, doc, terminating):
|
|
||||||
"""
|
|
||||||
Create sub sentences based on terminations found in text.
|
|
||||||
|
|
||||||
Parameters
|
|
||||||
----------
|
|
||||||
doc: object
|
|
||||||
spaCy Doc object
|
|
||||||
terminating: list
|
|
||||||
list of tuples with (match_id, start, end)
|
|
||||||
|
|
||||||
returns
|
|
||||||
-------
|
|
||||||
boundaries: list
|
|
||||||
list of tuples with (start, end) of spans
|
|
||||||
|
|
||||||
"""
|
|
||||||
sent_starts = [sent.start for sent in doc.sents]
|
|
||||||
terminating_starts = [t[1] for t in terminating]
|
|
||||||
starts = sent_starts + terminating_starts + [len(doc)]
|
|
||||||
starts.sort()
|
|
||||||
boundaries = list()
|
|
||||||
index = 0
|
|
||||||
for i, start in enumerate(starts):
|
|
||||||
if not i == 0:
|
|
||||||
boundaries.append((index, start))
|
|
||||||
index = start
|
|
||||||
return boundaries
|
|
||||||
|
|
||||||
def negex(self, doc):
|
|
||||||
"""
|
|
||||||
Negates entities of interest
|
|
||||||
|
|
||||||
Parameters
|
|
||||||
----------
|
|
||||||
doc: object
|
|
||||||
spaCy Doc object
|
|
||||||
|
|
||||||
"""
|
|
||||||
preceding, following, terminating = self.process_negations(doc)
|
|
||||||
boundaries = self.termination_boundaries(doc, terminating)
|
|
||||||
for b in boundaries:
|
|
||||||
sub_preceding = [i for i in preceding if b[0] <= i[1] < b[1]]
|
|
||||||
sub_following = [i for i in following if b[0] <= i[1] < b[1]]
|
|
||||||
|
|
||||||
for e in doc[b[0] : b[1]]:
|
|
||||||
if any(pre < e.i for pre in [i[1] for i in sub_preceding]):
|
|
||||||
e._.set(self.extension_name, True)
|
|
||||||
continue
|
|
||||||
if any(fol > e.i for fol in [i[2] for i in sub_following]):
|
|
||||||
e._.set(self.extension_name, True)
|
|
||||||
continue
|
|
||||||
if self.chunk_prefix:
|
|
||||||
if any(
|
|
||||||
e.text.lower().startswith(c.text.lower())
|
|
||||||
for c in self.chunk_prefix
|
|
||||||
):
|
|
||||||
e._.set(self.extension_name, True)
|
|
||||||
return doc
|
|
||||||
|
|
||||||
def __call__(self, doc):
|
|
||||||
return self.negex(doc)
|
|
||||||
229
src/great_ai/utilities/external/negspacy/termsets.py
vendored
229
src/great_ai/utilities/external/negspacy/termsets.py
vendored
|
|
@ -1,229 +0,0 @@
|
||||||
"""
|
|
||||||
Default termsets for various languages
|
|
||||||
"""
|
|
||||||
|
|
||||||
LANGUAGES = dict()
|
|
||||||
|
|
||||||
# english termset dictionary
|
|
||||||
en = dict()
|
|
||||||
pseudo = [
|
|
||||||
"no further",
|
|
||||||
"not able to be",
|
|
||||||
"not certain if",
|
|
||||||
"not certain whether",
|
|
||||||
"not necessarily",
|
|
||||||
"without any further",
|
|
||||||
"without difficulty",
|
|
||||||
"without further",
|
|
||||||
"might not",
|
|
||||||
"not only",
|
|
||||||
"no increase",
|
|
||||||
"no significant change",
|
|
||||||
"no change",
|
|
||||||
"no definite change",
|
|
||||||
"not extend",
|
|
||||||
"not cause",
|
|
||||||
]
|
|
||||||
en["pseudo_negations"] = pseudo
|
|
||||||
|
|
||||||
preceding = [
|
|
||||||
"absence of",
|
|
||||||
"declined",
|
|
||||||
"denied",
|
|
||||||
"denies",
|
|
||||||
"denying",
|
|
||||||
"no sign of",
|
|
||||||
"no signs of",
|
|
||||||
"not",
|
|
||||||
"not demonstrate",
|
|
||||||
"symptoms atypical",
|
|
||||||
"doubt",
|
|
||||||
"negative for",
|
|
||||||
"no",
|
|
||||||
"versus",
|
|
||||||
"without",
|
|
||||||
"doesn't",
|
|
||||||
"doesnt",
|
|
||||||
"don't",
|
|
||||||
"dont",
|
|
||||||
"didn't",
|
|
||||||
"didnt",
|
|
||||||
"wasn't",
|
|
||||||
"wasnt",
|
|
||||||
"weren't",
|
|
||||||
"werent",
|
|
||||||
"isn't",
|
|
||||||
"isnt",
|
|
||||||
"aren't",
|
|
||||||
"arent",
|
|
||||||
"cannot",
|
|
||||||
"can't",
|
|
||||||
"cant",
|
|
||||||
"couldn't",
|
|
||||||
"couldnt",
|
|
||||||
"never",
|
|
||||||
]
|
|
||||||
en["preceding_negations"] = preceding
|
|
||||||
|
|
||||||
following = [
|
|
||||||
"declined",
|
|
||||||
"unlikely",
|
|
||||||
"was not",
|
|
||||||
"were not",
|
|
||||||
"wasn't",
|
|
||||||
"wasnt",
|
|
||||||
"weren't",
|
|
||||||
"werent",
|
|
||||||
]
|
|
||||||
en["following_negations"] = following
|
|
||||||
|
|
||||||
termination = [
|
|
||||||
"although",
|
|
||||||
"apart from",
|
|
||||||
"as there are",
|
|
||||||
"aside from",
|
|
||||||
"but",
|
|
||||||
"except",
|
|
||||||
"however",
|
|
||||||
"involving",
|
|
||||||
"nevertheless",
|
|
||||||
"still",
|
|
||||||
"though",
|
|
||||||
"which",
|
|
||||||
"yet",
|
|
||||||
]
|
|
||||||
en["termination"] = termination
|
|
||||||
|
|
||||||
LANGUAGES["en"] = en
|
|
||||||
|
|
||||||
# en_clinical builds upon en
|
|
||||||
en_clinical = dict()
|
|
||||||
pseudo_clinical = pseudo + [
|
|
||||||
"gram negative",
|
|
||||||
"not rule out",
|
|
||||||
"not ruled out",
|
|
||||||
"not been ruled out",
|
|
||||||
"not drain",
|
|
||||||
"no suspicious change",
|
|
||||||
"no interval change",
|
|
||||||
"no significant interval change",
|
|
||||||
]
|
|
||||||
en_clinical["pseudo_negations"] = pseudo_clinical
|
|
||||||
|
|
||||||
preceding_clinical = preceding + [
|
|
||||||
"patient was not",
|
|
||||||
"without indication of",
|
|
||||||
"without sign of",
|
|
||||||
"without signs of",
|
|
||||||
"without any reactions or signs of",
|
|
||||||
"no complaints of",
|
|
||||||
"no evidence of",
|
|
||||||
"no cause of",
|
|
||||||
"evaluate for",
|
|
||||||
"fails to reveal",
|
|
||||||
"free of",
|
|
||||||
"never developed",
|
|
||||||
"never had",
|
|
||||||
"did not exhibit",
|
|
||||||
"rules out",
|
|
||||||
"rule out",
|
|
||||||
"rule him out",
|
|
||||||
"rule her out",
|
|
||||||
"rule patient out",
|
|
||||||
"rule the patient out",
|
|
||||||
"ruled out",
|
|
||||||
"ruled him out",
|
|
||||||
"ruled her out",
|
|
||||||
"ruled patient out",
|
|
||||||
"ruled the patient out",
|
|
||||||
"r/o",
|
|
||||||
"ro",
|
|
||||||
]
|
|
||||||
en_clinical["preceding_negations"] = preceding_clinical
|
|
||||||
|
|
||||||
following_clinical = following + ["was ruled out", "were ruled out", "free"]
|
|
||||||
en_clinical["following_negations"] = following_clinical
|
|
||||||
|
|
||||||
termination_clinical = termination + [
|
|
||||||
"cause for",
|
|
||||||
"cause of",
|
|
||||||
"causes for",
|
|
||||||
"causes of",
|
|
||||||
"etiology for",
|
|
||||||
"etiology of",
|
|
||||||
"origin for",
|
|
||||||
"origin of",
|
|
||||||
"origins for",
|
|
||||||
"origins of",
|
|
||||||
"other possibilities of",
|
|
||||||
"reason for",
|
|
||||||
"reason of",
|
|
||||||
"reasons for",
|
|
||||||
"reasons of",
|
|
||||||
"secondary to",
|
|
||||||
"source for",
|
|
||||||
"source of",
|
|
||||||
"sources for",
|
|
||||||
"sources of",
|
|
||||||
"trigger event for",
|
|
||||||
]
|
|
||||||
en_clinical["termination"] = termination_clinical
|
|
||||||
LANGUAGES["en_clinical"] = en_clinical
|
|
||||||
|
|
||||||
en_clinical_sensitive = dict()
|
|
||||||
|
|
||||||
preceding_clinical_sensitive = preceding_clinical + [
|
|
||||||
"concern for",
|
|
||||||
"supposed",
|
|
||||||
"which causes",
|
|
||||||
"leads to",
|
|
||||||
"h/o",
|
|
||||||
"history of",
|
|
||||||
"instead of",
|
|
||||||
"if you experience",
|
|
||||||
"if you get",
|
|
||||||
"teaching the patient",
|
|
||||||
"taught the patient",
|
|
||||||
"teach the patient",
|
|
||||||
"educated the patient",
|
|
||||||
"educate the patient",
|
|
||||||
"educating the patient",
|
|
||||||
"monitored for",
|
|
||||||
"monitor for",
|
|
||||||
"test for",
|
|
||||||
"tested for",
|
|
||||||
]
|
|
||||||
en_clinical_sensitive["pseudo_negations"] = pseudo_clinical
|
|
||||||
en_clinical_sensitive["preceding_negations"] = preceding_clinical_sensitive
|
|
||||||
en_clinical_sensitive["following_negations"] = following_clinical
|
|
||||||
en_clinical_sensitive["termination"] = termination_clinical
|
|
||||||
|
|
||||||
LANGUAGES["en_clinical_sensitive"] = en_clinical_sensitive
|
|
||||||
|
|
||||||
|
|
||||||
class termset:
|
|
||||||
def __init__(self, termset_lang):
|
|
||||||
self.pattern_types = [
|
|
||||||
"pseudo_negations",
|
|
||||||
"preceding_negations",
|
|
||||||
"following_negations",
|
|
||||||
"termination",
|
|
||||||
]
|
|
||||||
self.terms = LANGUAGES[termset_lang]
|
|
||||||
|
|
||||||
def get_patterns(self):
|
|
||||||
return self.terms
|
|
||||||
|
|
||||||
def remove_patterns(self, pattern_dict):
|
|
||||||
for key, value in pattern_dict.items():
|
|
||||||
if key in self.pattern_types:
|
|
||||||
self.terms[key] = [i for i in self.terms[key] if i not in value]
|
|
||||||
else:
|
|
||||||
raise ValueError(f"Unexpected key: {key} not in {self.pattern_types}")
|
|
||||||
|
|
||||||
def add_patterns(self, pattern_dict):
|
|
||||||
for key, value in pattern_dict.items():
|
|
||||||
if key in self.pattern_types:
|
|
||||||
self.terms[key] = list(set(self.terms[key] + value))
|
|
||||||
else:
|
|
||||||
raise ValueError(f"Unexpected key: {key} not in {self.pattern_types}")
|
|
||||||
|
|
@ -1,21 +0,0 @@
|
||||||
from typing import List
|
|
||||||
|
|
||||||
from .lemmatize_token import lemmatize_token
|
|
||||||
from .nlp import nlp
|
|
||||||
|
|
||||||
|
|
||||||
def lemmatize_text(
|
|
||||||
text: str,
|
|
||||||
add_negation: bool = False,
|
|
||||||
add_part_of_speech: bool = False,
|
|
||||||
) -> List[str]:
|
|
||||||
doc = nlp(text)
|
|
||||||
|
|
||||||
return [
|
|
||||||
lemmatize_token(
|
|
||||||
t,
|
|
||||||
add_negation=add_negation,
|
|
||||||
add_part_of_speech=add_part_of_speech,
|
|
||||||
)
|
|
||||||
for t in doc
|
|
||||||
]
|
|
||||||
|
|
@ -1,20 +0,0 @@
|
||||||
from spacy.tokens import Token
|
|
||||||
|
|
||||||
from .data import american_spellings
|
|
||||||
|
|
||||||
|
|
||||||
def lemmatize_token(
|
|
||||||
token: Token,
|
|
||||||
add_negation: bool = False,
|
|
||||||
add_part_of_speech: bool = False,
|
|
||||||
) -> str:
|
|
||||||
lemma = token.lemma_.lower()
|
|
||||||
|
|
||||||
lemma = american_spellings.get(lemma, lemma)
|
|
||||||
|
|
||||||
if add_part_of_speech:
|
|
||||||
lemma = f"{lemma}_{token.pos_}"
|
|
||||||
if add_negation and token._.negex:
|
|
||||||
lemma = f"NOT_{lemma}"
|
|
||||||
|
|
||||||
return lemma
|
|
||||||
|
|
@ -1,3 +0,0 @@
|
||||||
from .fast_tokenize import fast_tokenize
|
|
||||||
from .filter_sentences import filter_sentences
|
|
||||||
from .normalize import normalize
|
|
||||||
|
|
@ -1,30 +0,0 @@
|
||||||
import re
|
|
||||||
from typing import List, Union
|
|
||||||
|
|
||||||
from segtok.tokenizer import word_tokenizer
|
|
||||||
|
|
||||||
from ..get_sentences import get_sentences
|
|
||||||
from .normalize import normalize
|
|
||||||
|
|
||||||
|
|
||||||
def fast_tokenize(
|
|
||||||
text: Union[List[str], str], ignore_partial: bool = False
|
|
||||||
) -> List[List[str]]:
|
|
||||||
if isinstance(text, str):
|
|
||||||
text = normalize(text)
|
|
||||||
text = get_sentences(text, ignore_partial=ignore_partial)
|
|
||||||
|
|
||||||
results: List[List[str]] = []
|
|
||||||
|
|
||||||
for sentence in text:
|
|
||||||
sentence = re.sub(r"\bare\b", "is", sentence)
|
|
||||||
sentence = re.sub(r"\ban\b", "a", sentence)
|
|
||||||
sentence = re.sub(r"\bthese\b", "this", sentence)
|
|
||||||
results.append(
|
|
||||||
[
|
|
||||||
token.lower() if token not in {"CITATION", "NUMBER"} else token
|
|
||||||
for token in word_tokenizer(sentence)
|
|
||||||
]
|
|
||||||
)
|
|
||||||
|
|
||||||
return results
|
|
||||||
|
|
@ -1,66 +0,0 @@
|
||||||
from pathlib import Path
|
|
||||||
from typing import Dict, List, Union
|
|
||||||
|
|
||||||
import yaml
|
|
||||||
from spacy.matcher import Matcher
|
|
||||||
|
|
||||||
from ..get_sentences import get_sentences
|
|
||||||
from ..nlp import nlp
|
|
||||||
from .fast_tokenize import fast_tokenize
|
|
||||||
from .normalize import normalize
|
|
||||||
|
|
||||||
rules_cache: Dict[str, Matcher] = {}
|
|
||||||
|
|
||||||
|
|
||||||
def filter_sentences(
|
|
||||||
sentences: str,
|
|
||||||
rules_file: Path,
|
|
||||||
inverse: bool = False,
|
|
||||||
ignore_partial: bool = False,
|
|
||||||
) -> List[str]:
|
|
||||||
if str(rules_file) not in rules_cache:
|
|
||||||
with open(rules_file, encoding="utf-8") as f:
|
|
||||||
rule_patterns = yaml.safe_load(f).keys()
|
|
||||||
|
|
||||||
matcher = Matcher(nlp.vocab)
|
|
||||||
rules = [_pattern_to_rule(p) for p in rule_patterns]
|
|
||||||
matcher.add("", rules)
|
|
||||||
rules_cache[str(rules_file)] = matcher
|
|
||||||
|
|
||||||
matcher = rules_cache[str(rules_file)]
|
|
||||||
|
|
||||||
original_sentences = get_sentences(sentences, ignore_partial=ignore_partial)
|
|
||||||
|
|
||||||
tokenized = fast_tokenize(original_sentences, ignore_partial=ignore_partial)
|
|
||||||
|
|
||||||
results: List[str] = []
|
|
||||||
for original_sentence, sentence in zip(original_sentences, tokenized):
|
|
||||||
doc = nlp(normalize(" ".join(sentence)))
|
|
||||||
matches = matcher(doc)
|
|
||||||
if matches:
|
|
||||||
# _, start, end = max(
|
|
||||||
# matches,
|
|
||||||
# key=lambda v: v[2] - v[1],
|
|
||||||
# )
|
|
||||||
# print(str(doc[start:end]))
|
|
||||||
|
|
||||||
if not inverse:
|
|
||||||
results.append(original_sentence)
|
|
||||||
elif inverse:
|
|
||||||
results.append(original_sentence)
|
|
||||||
|
|
||||||
return results
|
|
||||||
|
|
||||||
|
|
||||||
def _pattern_to_rule(pattern: str) -> List[Dict[str, Union[bool, str]]]:
|
|
||||||
result: List[Dict[str, Union[bool, str]]] = []
|
|
||||||
for t in pattern.split():
|
|
||||||
if t == "*":
|
|
||||||
result.extend([{"OP": "?"}, {"OP": "?"}])
|
|
||||||
elif t == "CITATION":
|
|
||||||
result.append({"ORTH": "CITATION"})
|
|
||||||
elif t == "NUMBER":
|
|
||||||
result.append({"ORTH": "NUMBER"})
|
|
||||||
else:
|
|
||||||
result.append({"LOWER": t})
|
|
||||||
return result
|
|
||||||
|
|
@ -1,22 +0,0 @@
|
||||||
import re
|
|
||||||
|
|
||||||
from ..clean import clean
|
|
||||||
|
|
||||||
|
|
||||||
def normalize(text: str) -> str:
|
|
||||||
text = re.sub(
|
|
||||||
r"""
|
|
||||||
([A-Z]\w+\W+(et\ al.)) # inline reference: Bank et al.
|
|
||||||
| (\[[0-9-, ]+\]) # IEEE style: [1], [2-4], [3, 5]
|
|
||||||
| (\(.*?,?\W+?\d+\)) # APA style: (Bank, 2020)
|
|
||||||
| ([A-Z]\w+ \(?\d+\)) # APA style: Bank (2020)
|
|
||||||
""",
|
|
||||||
" CITATION ",
|
|
||||||
text,
|
|
||||||
flags=re.VERBOSE,
|
|
||||||
)
|
|
||||||
text = re.sub(r"\d[\d,. -]*(st|nd|rd|th)?", " NUMBER ", text)
|
|
||||||
text = clean(text, convert_to_ascii=True)
|
|
||||||
text = re.sub(r"[^a-zA-Z.?!,:;'\" -]", "", text)
|
|
||||||
|
|
||||||
return text
|
|
||||||
|
|
@ -1,21 +0,0 @@
|
||||||
try:
|
|
||||||
import en_core_web_sm
|
|
||||||
except ImportError:
|
|
||||||
import subprocess
|
|
||||||
|
|
||||||
print("Spacy model en_core_web_sm not found locally, downloading...")
|
|
||||||
|
|
||||||
subprocess.call(
|
|
||||||
[
|
|
||||||
"pip",
|
|
||||||
"install",
|
|
||||||
"https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl",
|
|
||||||
]
|
|
||||||
)
|
|
||||||
import en_core_web_sm
|
|
||||||
|
|
||||||
from .external.negspacy import negation # noqa: F401 it's important to import this
|
|
||||||
|
|
||||||
nlp = en_core_web_sm.load()
|
|
||||||
|
|
||||||
nlp.add_pipe("negex")
|
|
||||||
|
|
@ -1,2 +0,0 @@
|
||||||
from .models import *
|
|
||||||
from .publication_tei import PublicationTEI
|
|
||||||
|
|
@ -1,7 +0,0 @@
|
||||||
from .affiliation import Affiliation
|
|
||||||
from .author import Author
|
|
||||||
from .bookmark import Bookmark
|
|
||||||
from .bookmark_title import BookmarkTitle
|
|
||||||
from .element import Element, Meta, MetaType, Paragraph, Title
|
|
||||||
from .publication_metadata import PublicationMetadata
|
|
||||||
from .text import Text
|
|
||||||
|
|
@ -1,11 +0,0 @@
|
||||||
from typing import Optional, Tuple
|
|
||||||
|
|
||||||
from pydantic import BaseModel
|
|
||||||
|
|
||||||
|
|
||||||
class Affiliation(BaseModel):
|
|
||||||
institutions: Tuple[str, ...]
|
|
||||||
departments: Tuple[str, ...]
|
|
||||||
laboratories: Tuple[str, ...]
|
|
||||||
country: Optional[str]
|
|
||||||
settlement: Optional[str]
|
|
||||||
|
|
@ -1,14 +0,0 @@
|
||||||
from typing import List, Optional
|
|
||||||
|
|
||||||
from pydantic import BaseModel
|
|
||||||
|
|
||||||
from .affiliation import Affiliation
|
|
||||||
|
|
||||||
|
|
||||||
class Author(BaseModel):
|
|
||||||
name: Optional[str]
|
|
||||||
orcid: Optional[str]
|
|
||||||
email: Optional[str]
|
|
||||||
corresponding: bool
|
|
||||||
affiliations: List[Affiliation]
|
|
||||||
coordinates: Optional[str]
|
|
||||||
|
|
@ -1,10 +0,0 @@
|
||||||
from pydantic import BaseModel
|
|
||||||
|
|
||||||
from .bookmark_title import BookmarkTitle
|
|
||||||
|
|
||||||
|
|
||||||
class Bookmark(BaseModel):
|
|
||||||
title: BookmarkTitle
|
|
||||||
original_title: str
|
|
||||||
document_order: int
|
|
||||||
coordinates: str
|
|
||||||
|
|
@ -1,16 +0,0 @@
|
||||||
from typing import Literal
|
|
||||||
|
|
||||||
BookmarkTitle = Literal[
|
|
||||||
"Abstract",
|
|
||||||
"Author contribution",
|
|
||||||
"Introduction",
|
|
||||||
"Background",
|
|
||||||
"Methods",
|
|
||||||
"Results",
|
|
||||||
"Discussion",
|
|
||||||
"Outlook",
|
|
||||||
"Conclusion",
|
|
||||||
"Conflict of interest",
|
|
||||||
"Acknowledgement",
|
|
||||||
"Annex",
|
|
||||||
]
|
|
||||||
|
|
@ -1,30 +0,0 @@
|
||||||
from typing import List, Literal, Union
|
|
||||||
|
|
||||||
from pydantic import BaseModel
|
|
||||||
|
|
||||||
from .text import Text
|
|
||||||
|
|
||||||
|
|
||||||
class Title(BaseModel):
|
|
||||||
text: Text
|
|
||||||
|
|
||||||
|
|
||||||
class Paragraph(BaseModel):
|
|
||||||
sentences: List[Text]
|
|
||||||
|
|
||||||
|
|
||||||
MetaType = Literal[
|
|
||||||
"abstract_start",
|
|
||||||
"abstract_end",
|
|
||||||
"acknowledgements_start",
|
|
||||||
"acknowledgements_end",
|
|
||||||
"annex_start",
|
|
||||||
"annex_end",
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
class Meta(BaseModel):
|
|
||||||
meta_type: MetaType
|
|
||||||
|
|
||||||
|
|
||||||
Element = Union[Title, Paragraph, Meta]
|
|
||||||
|
|
@ -1,14 +0,0 @@
|
||||||
from typing import List, Optional
|
|
||||||
|
|
||||||
from pydantic import BaseModel
|
|
||||||
|
|
||||||
|
|
||||||
class PublicationMetadata(BaseModel):
|
|
||||||
language: Optional[str]
|
|
||||||
title: Optional[str]
|
|
||||||
publisher: Optional[str]
|
|
||||||
doi: Optional[str]
|
|
||||||
md5: Optional[str]
|
|
||||||
publication_date: Optional[str]
|
|
||||||
keywords: List[str]
|
|
||||||
reference_count: int
|
|
||||||
|
|
@ -1,9 +0,0 @@
|
||||||
from typing import Optional
|
|
||||||
|
|
||||||
from pydantic import BaseModel
|
|
||||||
|
|
||||||
|
|
||||||
class Text(BaseModel):
|
|
||||||
content: str
|
|
||||||
document_order: int
|
|
||||||
coordinates: Optional[str]
|
|
||||||
|
|
@ -1,423 +0,0 @@
|
||||||
import os
|
|
||||||
import re
|
|
||||||
from functools import cached_property, lru_cache
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any, List, Optional, Pattern, Tuple, Union
|
|
||||||
|
|
||||||
from bs4 import BeautifulSoup
|
|
||||||
from bs4.element import NavigableString, Tag
|
|
||||||
|
|
||||||
from ..clean import clean
|
|
||||||
from ..lemmatize_text import lemmatize_text
|
|
||||||
from ..matcher import filter_sentences
|
|
||||||
from ..unique import unique
|
|
||||||
from .models import (
|
|
||||||
Affiliation,
|
|
||||||
Author,
|
|
||||||
Bookmark,
|
|
||||||
BookmarkTitle,
|
|
||||||
Element,
|
|
||||||
Meta,
|
|
||||||
MetaType,
|
|
||||||
Paragraph,
|
|
||||||
PublicationMetadata,
|
|
||||||
Text,
|
|
||||||
Title,
|
|
||||||
)
|
|
||||||
from .titles_of_interest import titles_of_interest
|
|
||||||
|
|
||||||
THIS_FOLDER = Path(os.path.dirname(os.path.abspath(__file__)))
|
|
||||||
|
|
||||||
|
|
||||||
class PublicationTEI:
|
|
||||||
# remove template sentences, such as copyright notices
|
|
||||||
aggressive_cleaning_enabled = True
|
|
||||||
|
|
||||||
def __init__(self, tei: str):
|
|
||||||
self._document_order_counter = 0
|
|
||||||
|
|
||||||
if tei:
|
|
||||||
self.soup = BeautifulSoup(tei, "xml")
|
|
||||||
else:
|
|
||||||
self.soup = BeautifulSoup()
|
|
||||||
|
|
||||||
@cached_property
|
|
||||||
def publication_metadata(self) -> PublicationMetadata:
|
|
||||||
publication_date = (
|
|
||||||
self.soup.publicationStmt.date.get("when")
|
|
||||||
if self.soup.publicationStmt and self.soup.publicationStmt.date
|
|
||||||
else None
|
|
||||||
)
|
|
||||||
|
|
||||||
keywords = (
|
|
||||||
[self._element_to_text(k) for k in self.soup.keywords.find_all("term")]
|
|
||||||
if self.soup.keywords
|
|
||||||
else []
|
|
||||||
)
|
|
||||||
|
|
||||||
return PublicationMetadata(
|
|
||||||
language=self.soup.teiHeader.get("xml:lang")
|
|
||||||
if self.soup.teiHeader
|
|
||||||
else None,
|
|
||||||
title=self._element_to_text(self.soup.title),
|
|
||||||
publisher=self._element_to_text(self.soup.publisher),
|
|
||||||
doi=self._element_to_text(self.soup.find("idno", type="DOI")),
|
|
||||||
md5=self._element_to_text(self.soup.find("idno", type="MD5")),
|
|
||||||
publication_date=publication_date,
|
|
||||||
keywords=keywords,
|
|
||||||
reference_count=self.get_reference_count(),
|
|
||||||
)
|
|
||||||
|
|
||||||
def get_reference_count(self) -> int:
|
|
||||||
references = self.soup.find("div", {"type": "references"})
|
|
||||||
|
|
||||||
if not references:
|
|
||||||
return 0
|
|
||||||
|
|
||||||
return len(references.findAll("biblStruct"))
|
|
||||||
|
|
||||||
@cached_property
|
|
||||||
def authors(self) -> List[Author]:
|
|
||||||
if not self.soup.analytic:
|
|
||||||
return []
|
|
||||||
|
|
||||||
return [
|
|
||||||
self._parse_author(author)
|
|
||||||
for author in self.soup.analytic.find_all("author")
|
|
||||||
]
|
|
||||||
|
|
||||||
@cached_property
|
|
||||||
def content(self) -> List[Element]:
|
|
||||||
self._document_order_counter = 0
|
|
||||||
return self._get_elements(self.soup)
|
|
||||||
|
|
||||||
@cached_property
|
|
||||||
def sentences(self) -> List[Text]:
|
|
||||||
return [
|
|
||||||
sentence
|
|
||||||
for element in self.content
|
|
||||||
if isinstance(element, Paragraph)
|
|
||||||
for sentence in element.sentences
|
|
||||||
]
|
|
||||||
|
|
||||||
@cached_property
|
|
||||||
def bookmarks(self) -> List[Bookmark]:
|
|
||||||
candidates: List[Bookmark] = [
|
|
||||||
*self._find_matching_meta("Abstract", "abstract_start")[0],
|
|
||||||
*self._find_matching_title("Abstract"),
|
|
||||||
*self._find_matching_title("Author contribution"),
|
|
||||||
*self._find_matching_meta("Acknowledgement", "acknowledgements_start")[0],
|
|
||||||
*self._find_matching_title("Acknowledgement"),
|
|
||||||
*self._find_matching_title("Conflict of interest"),
|
|
||||||
]
|
|
||||||
|
|
||||||
_, start = self._find_matching_meta(None, "abstract_end")
|
|
||||||
|
|
||||||
candidates += [
|
|
||||||
*self._find_matching_title("Background", start),
|
|
||||||
*self._find_matching_title("Methods", start),
|
|
||||||
*self._find_matching_title("Results", start),
|
|
||||||
*self._find_matching_title("Discussion", start),
|
|
||||||
*self._find_matching_title("Introduction", start),
|
|
||||||
*self._find_matching_title("Conclusion", start),
|
|
||||||
*self._find_matching_title("Outlook", start),
|
|
||||||
*self._find_matching_meta("Annex", "annex_start", start)[0],
|
|
||||||
]
|
|
||||||
|
|
||||||
candidates = sorted(candidates, key=lambda c: c.document_order)
|
|
||||||
candidates = unique(candidates, key=lambda c: c.document_order)
|
|
||||||
candidates = unique(candidates, key=lambda c: c.title)
|
|
||||||
|
|
||||||
return candidates
|
|
||||||
|
|
||||||
@cached_property
|
|
||||||
def abstract_sentences(self) -> List[Text]:
|
|
||||||
try:
|
|
||||||
abstract_start = next(
|
|
||||||
i
|
|
||||||
for i, m in enumerate(self.content)
|
|
||||||
if isinstance(m, Meta) and m.meta_type == "abstract_start"
|
|
||||||
)
|
|
||||||
abstract_end = next(
|
|
||||||
i
|
|
||||||
for i, m in enumerate(self.content)
|
|
||||||
if isinstance(m, Meta) and m.meta_type == "abstract_end"
|
|
||||||
)
|
|
||||||
return [
|
|
||||||
s
|
|
||||||
for p in self.content[abstract_start:abstract_end]
|
|
||||||
if isinstance(p, Paragraph)
|
|
||||||
for s in p.sentences
|
|
||||||
]
|
|
||||||
except StopIteration:
|
|
||||||
pass # let's try another way
|
|
||||||
|
|
||||||
try:
|
|
||||||
abstract_start = next(
|
|
||||||
m.document_order for m in self.bookmarks if m.title == "Abstract"
|
|
||||||
)
|
|
||||||
abstract_sentences: List[Text] = []
|
|
||||||
for c in self.content:
|
|
||||||
if isinstance(c, Paragraph):
|
|
||||||
abstract_sentences.extend(
|
|
||||||
s for s in c.sentences if s.document_order > abstract_start
|
|
||||||
)
|
|
||||||
elif len(abstract_sentences) >= 5:
|
|
||||||
break
|
|
||||||
return abstract_sentences
|
|
||||||
except StopIteration:
|
|
||||||
pass # let's try another way
|
|
||||||
|
|
||||||
return self.sentences[:10]
|
|
||||||
|
|
||||||
@cached_property
|
|
||||||
def introduction_sentences(self) -> List[Text]:
|
|
||||||
"""Includes abstract"""
|
|
||||||
introduction_end = 4
|
|
||||||
|
|
||||||
try:
|
|
||||||
introduction_start = [
|
|
||||||
m.document_order for m in self.bookmarks if m.title == "Introduction"
|
|
||||||
][-1]
|
|
||||||
|
|
||||||
for m in self.bookmarks:
|
|
||||||
if m.title != "Introduction" and m.document_order > introduction_start:
|
|
||||||
introduction_end = m.document_order
|
|
||||||
break
|
|
||||||
except IndexError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
try:
|
|
||||||
introduction_end = max(
|
|
||||||
next(
|
|
||||||
i
|
|
||||||
for i, m in enumerate(self.content)
|
|
||||||
if isinstance(m, Meta) and m.meta_type == "abstract_end"
|
|
||||||
),
|
|
||||||
introduction_end,
|
|
||||||
)
|
|
||||||
except StopIteration:
|
|
||||||
pass
|
|
||||||
|
|
||||||
introduction_sentences: List[Text] = []
|
|
||||||
for c in self.content:
|
|
||||||
if isinstance(c, Paragraph):
|
|
||||||
introduction_sentences.extend(
|
|
||||||
s for s in c.sentences if s.document_order < introduction_end
|
|
||||||
)
|
|
||||||
|
|
||||||
return introduction_sentences
|
|
||||||
|
|
||||||
@cached_property
|
|
||||||
def conclusion_sentences(self) -> List[Text]:
|
|
||||||
try:
|
|
||||||
conclusion_start = next(
|
|
||||||
m.document_order for m in self.bookmarks if m.title == "Conclusion"
|
|
||||||
)
|
|
||||||
conclusion_sentences: List[Text] = []
|
|
||||||
for c in self.content:
|
|
||||||
if isinstance(c, Paragraph):
|
|
||||||
conclusion_sentences.extend(
|
|
||||||
s for s in c.sentences if s.document_order > conclusion_start
|
|
||||||
)
|
|
||||||
elif len(conclusion_sentences) >= 8:
|
|
||||||
break
|
|
||||||
return conclusion_sentences
|
|
||||||
except StopIteration:
|
|
||||||
return self.sentences[-10:]
|
|
||||||
|
|
||||||
def _parse_author(self, raw: Tag) -> Author:
|
|
||||||
return Author(
|
|
||||||
name=(
|
|
||||||
clean(" ".join(name.get_text() for name in raw.persName))
|
|
||||||
if raw.persName
|
|
||||||
else None
|
|
||||||
),
|
|
||||||
orcid=self._element_to_text(raw.find(attrs={"type": "ORCID"})),
|
|
||||||
email=self._element_to_text(raw.email),
|
|
||||||
corresponding=raw.get("role") == "corresp",
|
|
||||||
affiliations=[
|
|
||||||
self._parse_affiliation(aff) for aff in raw.find_all("affiliation")
|
|
||||||
],
|
|
||||||
coordinates=raw.persName.get("coords") if raw.persName else None,
|
|
||||||
)
|
|
||||||
|
|
||||||
def _parse_affiliation(self, raw: Tag) -> Affiliation:
|
|
||||||
return Affiliation(
|
|
||||||
institutions=[
|
|
||||||
self._element_to_text(v)
|
|
||||||
for v in raw.find_all("orgName", attrs={"type": "institution"})
|
|
||||||
],
|
|
||||||
departments=[
|
|
||||||
self._element_to_text(v)
|
|
||||||
for v in raw.find_all("orgName", attrs={"type": "department"})
|
|
||||||
],
|
|
||||||
laboratories=[
|
|
||||||
self._element_to_text(v)
|
|
||||||
for v in raw.find_all("orgName", attrs={"type": "laboratory"})
|
|
||||||
],
|
|
||||||
country=self._element_to_text(raw.address.country)
|
|
||||||
if raw.address and raw.address.country
|
|
||||||
else None,
|
|
||||||
settlement=self._element_to_text(raw.address.settlement)
|
|
||||||
if raw.address and raw.address.settlement
|
|
||||||
else None,
|
|
||||||
)
|
|
||||||
|
|
||||||
def _get_elements(self, raw: Tag) -> List[Element]:
|
|
||||||
results: List[Element] = []
|
|
||||||
|
|
||||||
for r in raw.find_all(["abstract", "div", "head", "p"]):
|
|
||||||
if r.name == "abstract":
|
|
||||||
results.append(Meta(meta_type="abstract_start"))
|
|
||||||
results.extend(self._get_primitives(r))
|
|
||||||
results.append(Meta(meta_type="abstract_end"))
|
|
||||||
elif r.name == "div" and r.get("type") == "acknowledgement":
|
|
||||||
results.append(Meta(meta_type="acknowledgements_start"))
|
|
||||||
results.extend(self._get_primitives(r))
|
|
||||||
results.append(Meta(meta_type="acknowledgements_end"))
|
|
||||||
elif r.name == "div" and r.get("type") == "annex":
|
|
||||||
results.append(Meta(meta_type="annex_start"))
|
|
||||||
results.extend(self._get_primitives(r))
|
|
||||||
results.append(Meta(meta_type="annex_end"))
|
|
||||||
elif not r.find_parents(
|
|
||||||
["abstract", "div", "figure"]
|
|
||||||
): # figures are omitted as well
|
|
||||||
results.extend(self._get_primitives(r))
|
|
||||||
|
|
||||||
return results
|
|
||||||
|
|
||||||
def _get_primitives(self, raw: Tag) -> List[Element]:
|
|
||||||
results: List[Element] = []
|
|
||||||
|
|
||||||
for r in raw.find_all(["head", "p"]):
|
|
||||||
if r.name == "head" and r.get("coords") and r.get_text():
|
|
||||||
text = self._parse_text(r)
|
|
||||||
if text:
|
|
||||||
results.append(Title(text=text))
|
|
||||||
elif r.name == "p" and r.find_all("s"):
|
|
||||||
results.append(
|
|
||||||
Paragraph(
|
|
||||||
sentences=[
|
|
||||||
t
|
|
||||||
for t in [
|
|
||||||
self._parse_text(sentence, ignore_partial=True)
|
|
||||||
for sentence in r.find_all("s")
|
|
||||||
if sentence.get_text()
|
|
||||||
]
|
|
||||||
if t
|
|
||||||
]
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
return results
|
|
||||||
|
|
||||||
def _element_to_text(
|
|
||||||
self, element: Optional[NavigableString], default: Any = None
|
|
||||||
) -> Union[str, Any]:
|
|
||||||
return (
|
|
||||||
clean(element.get_text(separator=" ", strip=True)) if element else default
|
|
||||||
)
|
|
||||||
|
|
||||||
def _parse_text(self, raw: Tag, ignore_partial: bool = False) -> Optional[Text]:
|
|
||||||
text = raw.get_text()
|
|
||||||
if text is None:
|
|
||||||
return None
|
|
||||||
|
|
||||||
if self.aggressive_cleaning_enabled:
|
|
||||||
filtered = filter_sentences(
|
|
||||||
text,
|
|
||||||
THIS_FOLDER / "templates.yaml",
|
|
||||||
inverse=True,
|
|
||||||
ignore_partial=ignore_partial,
|
|
||||||
)
|
|
||||||
text = " ".join(filtered)
|
|
||||||
|
|
||||||
if not text.strip():
|
|
||||||
return None
|
|
||||||
|
|
||||||
return Text(
|
|
||||||
content=text,
|
|
||||||
document_order=self._generate_document_order_id(),
|
|
||||||
coordinates=raw.get("coords"),
|
|
||||||
)
|
|
||||||
|
|
||||||
def _generate_document_order_id(self) -> int:
|
|
||||||
value = self._document_order_counter
|
|
||||||
self._document_order_counter += 1
|
|
||||||
return value
|
|
||||||
|
|
||||||
def _find_matching_meta(
|
|
||||||
self,
|
|
||||||
bookmark_title: Optional[BookmarkTitle],
|
|
||||||
meta_type: MetaType,
|
|
||||||
start: int = 0,
|
|
||||||
) -> Tuple[List[Bookmark], int]:
|
|
||||||
for i, e in enumerate(self.content[start:], start=start):
|
|
||||||
if not isinstance(e, Meta):
|
|
||||||
continue
|
|
||||||
|
|
||||||
if e.meta_type == meta_type:
|
|
||||||
for e_next in self.content[i + 1 :]:
|
|
||||||
if isinstance(e_next, Title):
|
|
||||||
return [
|
|
||||||
Bookmark(
|
|
||||||
title=bookmark_title,
|
|
||||||
original_title=e_next.text.content,
|
|
||||||
document_order=e_next.text.document_order,
|
|
||||||
coordinates=e_next.text.coordinates,
|
|
||||||
)
|
|
||||||
] if bookmark_title else [], i
|
|
||||||
if isinstance(e_next, Paragraph) and e_next.sentences:
|
|
||||||
return [
|
|
||||||
Bookmark(
|
|
||||||
title=bookmark_title,
|
|
||||||
original_title="",
|
|
||||||
document_order=e_next.sentences[0].document_order,
|
|
||||||
coordinates=e_next.sentences[0].coordinates,
|
|
||||||
)
|
|
||||||
] if bookmark_title else [], i
|
|
||||||
|
|
||||||
return [], 0
|
|
||||||
|
|
||||||
def _find_matching_title(
|
|
||||||
self,
|
|
||||||
bookmark_title: BookmarkTitle,
|
|
||||||
start: int = 0,
|
|
||||||
) -> List[Bookmark]:
|
|
||||||
return [
|
|
||||||
Bookmark(
|
|
||||||
title=bookmark_title,
|
|
||||||
original_title=e.text.content,
|
|
||||||
document_order=e.text.document_order,
|
|
||||||
coordinates=e.text.coordinates,
|
|
||||||
)
|
|
||||||
for e in self.content[start:]
|
|
||||||
if isinstance(e, Title)
|
|
||||||
and self._match_title(e.text.content, titles_of_interest[bookmark_title])
|
|
||||||
]
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _match_title(title: str, keywords: Tuple[Union[Pattern, str], ...]) -> bool:
|
|
||||||
title = PublicationTEI._process_section_title(title)
|
|
||||||
|
|
||||||
if any(
|
|
||||||
PublicationTEI._process_section_title(k) in title
|
|
||||||
for k in keywords
|
|
||||||
if isinstance(k, str)
|
|
||||||
):
|
|
||||||
return True
|
|
||||||
|
|
||||||
return any(k.match(title) for k in keywords if isinstance(k, Pattern))
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
@lru_cache(maxsize=2000)
|
|
||||||
def _process_section_title(title: str) -> str:
|
|
||||||
title = re.sub(r"^\s*[ivx]+[.,)]? ", "", title) # Remove leading Roman-numerals
|
|
||||||
title = clean(title, convert_to_ascii=True)
|
|
||||||
title = re.sub(
|
|
||||||
r"[^a-zA-Z ]", "", title
|
|
||||||
) # Remove everything but letters and spaces (hypens are also removed)
|
|
||||||
title_tokens = lemmatize_text(title)
|
|
||||||
title = " ".join(t for t in title_tokens if t.strip())
|
|
||||||
return title
|
|
||||||
|
|
@ -1,837 +0,0 @@
|
||||||
included in the article's creative commons: 3827
|
|
||||||
is a open access article distributed: 3822
|
|
||||||
a open access article distributed under: 3761
|
|
||||||
open access article distributed under the: 3761
|
|
||||||
access article distributed under the terms: 3761
|
|
||||||
the terms and conditions of the: 3688
|
|
||||||
this article is a open access: 3678
|
|
||||||
article is a open access article: 3678
|
|
||||||
article distributed under the terms and: 3678
|
|
||||||
distributed under the terms and conditions: 3678
|
|
||||||
under the terms and conditions of: 3678
|
|
||||||
terms and conditions of the creative: 3678
|
|
||||||
and conditions of the creative commons: 3678
|
|
||||||
conditions of the creative commons attribution: 3678
|
|
||||||
in the article's creative commons licence: 3503
|
|
||||||
authors declare that they have no: 3315
|
|
||||||
https :// doi.org / NUMBER /: 3189
|
|
||||||
the authors declare that they have: 3015
|
|
||||||
remains neutral with regard to jurisdictional: 2464
|
|
||||||
neutral with regard to jurisdictional claims: 2464
|
|
||||||
with regard to jurisdictional claims in: 2464
|
|
||||||
regard to jurisdictional claims in published: 2462
|
|
||||||
to jurisdictional claims in published maps: 2462
|
|
||||||
the original author ( s ): 2461
|
|
||||||
jurisdictional claims in published maps and: 2461
|
|
||||||
claims in published maps and institutional: 2461
|
|
||||||
author ( s ) and the: 2460
|
|
||||||
original author ( s ) and: 2456
|
|
||||||
in published maps and institutional affiliations: 2415
|
|
||||||
nature remains neutral with regard to: 2399
|
|
||||||
springer nature remains neutral with regard: 2392
|
|
||||||
( s ) and the source: 2353
|
|
||||||
of the creative commons attribution CITATION: 2302
|
|
||||||
", provide a link to the": 2299
|
|
||||||
the source , provide a link: 2293
|
|
||||||
source , provide a link to: 2293
|
|
||||||
you give appropriate credit to the: 2292
|
|
||||||
give appropriate credit to the original: 2292
|
|
||||||
appropriate credit to the original author: 2292
|
|
||||||
and indicate if changes were made: 2289
|
|
||||||
credit to the original author (: 2281
|
|
||||||
to the original author ( s: 2281
|
|
||||||
provide a link to the creative: 2280
|
|
||||||
a link to the creative commons: 2280
|
|
||||||
s ) and the source ,: 2279
|
|
||||||
) and the source , provide: 2277
|
|
||||||
and the source , provide a: 2277
|
|
||||||
", and indicate if changes were": 2271
|
|
||||||
at https :// doi.org / NUMBER: 2195
|
|
||||||
creative commons attribution NUMBER international license: 2158
|
|
||||||
", distribution and reproduction in any": 2158
|
|
||||||
distribution and reproduction in any medium: 2157
|
|
||||||
can be found online at https: 2106
|
|
||||||
be found online at https ://: 2106
|
|
||||||
article can be found online at: 2069
|
|
||||||
license , which permits use ,: 2063
|
|
||||||
this article can be found online: 2010
|
|
||||||
online version contains supplementary material available: 1980
|
|
||||||
supplementary material available at https ://: 1979
|
|
||||||
version contains supplementary material available at: 1978
|
|
||||||
contains supplementary material available at https: 1973
|
|
||||||
the online version contains supplementary material: 1957
|
|
||||||
note springer nature remains neutral with: 1956
|
|
||||||
other third party material in this: 1945
|
|
||||||
to view a copy of this: 1945
|
|
||||||
or other third party material in: 1944
|
|
||||||
need to obtain permission directly from: 1944
|
|
||||||
to obtain permission directly from the: 1944
|
|
||||||
images or other third party material: 1943
|
|
||||||
regulation or exceeds the permitted use: 1943
|
|
||||||
or exceeds the permitted use ,: 1943
|
|
||||||
obtain permission directly from the copyright: 1943
|
|
||||||
the permitted use , you will: 1942
|
|
||||||
by statutory regulation or exceeds the: 1941
|
|
||||||
statutory regulation or exceeds the permitted: 1941
|
|
||||||
exceeds the permitted use , you: 1941
|
|
||||||
use , you will need to: 1941
|
|
||||||
", you will need to obtain": 1941
|
|
||||||
you will need to obtain permission: 1941
|
|
||||||
permitted use , you will need: 1940
|
|
||||||
will need to obtain permission directly: 1940
|
|
||||||
permitted by statutory regulation or exceeds: 1939
|
|
||||||
publisher's note springer nature remains neutral: 1939
|
|
||||||
and your intended use is not: 1938
|
|
||||||
your intended use is not permitted: 1938
|
|
||||||
not permitted by statutory regulation or: 1938
|
|
||||||
intended use is not permitted by: 1937
|
|
||||||
is not permitted by statutory regulation: 1937
|
|
||||||
permission directly from the copyright holder: 1937
|
|
||||||
use is not permitted by statutory: 1936
|
|
||||||
material is not included in the: 1933
|
|
||||||
the images or other third party: 1932
|
|
||||||
if material is not included in: 1926
|
|
||||||
in a credit line to the: 1924
|
|
||||||
otherwise in a credit line to: 1923
|
|
||||||
this article is included in the: 1922
|
|
||||||
third party material in this article: 1921
|
|
||||||
party material in this article is: 1921
|
|
||||||
material in this article is included: 1921
|
|
||||||
in this article is included in: 1921
|
|
||||||
", unless indicated otherwise in a": 1920
|
|
||||||
indicated otherwise in a credit line: 1920
|
|
||||||
is included in the article's creative: 1919
|
|
||||||
unless indicated otherwise in a credit: 1919
|
|
||||||
article is included in the article's: 1918
|
|
||||||
a credit line to the material: 1914
|
|
||||||
not included in the article's creative: 1907
|
|
||||||
is not included in the article's: 1906
|
|
||||||
is licensed under a creative commons: 1898
|
|
||||||
authors declare no conflict of interest: 1883
|
|
||||||
and reproduction in any medium or: 1870
|
|
||||||
reproduction in any medium or format: 1870
|
|
||||||
in any medium or format ,: 1870
|
|
||||||
any medium or format , as: 1870
|
|
||||||
medium or format , as long: 1870
|
|
||||||
or format , as long as: 1870
|
|
||||||
format , as long as you: 1869
|
|
||||||
", as long as you give": 1869
|
|
||||||
as long as you give appropriate: 1869
|
|
||||||
long as you give appropriate credit: 1869
|
|
||||||
as you give appropriate credit to: 1869
|
|
||||||
", adaptation , distribution and reproduction": 1866
|
|
||||||
adaptation , distribution and reproduction in: 1866
|
|
||||||
use , sharing , adaptation ,: 1861
|
|
||||||
", sharing , adaptation , distribution": 1861
|
|
||||||
sharing , adaptation , distribution and: 1861
|
|
||||||
NUMBER international license , which permits: 1846
|
|
||||||
article is licensed under a creative: 1839
|
|
||||||
this article is licensed under a: 1838
|
|
||||||
under a creative commons attribution NUMBER: 1817
|
|
||||||
licensed under a creative commons attribution: 1811
|
|
||||||
a creative commons attribution NUMBER international: 1807
|
|
||||||
view a copy of this licence: 1799
|
|
||||||
a copy of this licence ,: 1799
|
|
||||||
copy of this licence , visit: 1799
|
|
||||||
of this licence , visit http: 1796
|
|
||||||
this licence , visit http ://: 1796
|
|
||||||
access this article is licensed under: 1794
|
|
||||||
the authors would like to thank: 1793
|
|
||||||
which permits use , sharing ,: 1791
|
|
||||||
permits use , sharing , adaptation: 1790
|
|
||||||
", which permits use , sharing": 1789
|
|
||||||
open access this article is licensed: 1782
|
|
||||||
international license , which permits use: 1776
|
|
||||||
attribution NUMBER international license , which: 1773
|
|
||||||
commons attribution NUMBER international license ,: 1772
|
|
||||||
creative commons licence and your intended: 1769
|
|
||||||
commons licence and your intended use: 1768
|
|
||||||
licence and your intended use is: 1768
|
|
||||||
the article's creative commons licence and: 1762
|
|
||||||
article's creative commons licence and your: 1762
|
|
||||||
creative commons licence , unless indicated: 1750
|
|
||||||
commons licence , unless indicated otherwise: 1749
|
|
||||||
licence , unless indicated otherwise in: 1749
|
|
||||||
article's creative commons licence , unless: 1743
|
|
||||||
the article's creative commons licence ,: 1742
|
|
||||||
the authors declare no conflict of: 1742
|
|
||||||
to this article can be found: 1714
|
|
||||||
licence , and indicate if changes: 1698
|
|
||||||
link to the creative commons licence: 1696
|
|
||||||
commons licence , and indicate if: 1695
|
|
||||||
to the creative commons licence ,: 1694
|
|
||||||
the creative commons licence , and: 1694
|
|
||||||
creative commons licence , and indicate: 1694
|
|
||||||
the authors declare no competing interests: 1564
|
|
||||||
and / or publication of this: 1530
|
|
||||||
/ or publication of this article: 1523
|
|
||||||
the research , authorship , and: 1410
|
|
||||||
data to this article can be: 1395
|
|
||||||
supplementary data to this article can: 1392
|
|
||||||
competing financial interests or personal relationships: 1388
|
|
||||||
research , authorship , and /: 1385
|
|
||||||
", authorship , and / or": 1385
|
|
||||||
authorship , and / or publication: 1385
|
|
||||||
is available from the corresponding author: 1385
|
|
||||||
", and / or publication of": 1384
|
|
||||||
no known competing financial interests or: 1382
|
|
||||||
known competing financial interests or personal: 1382
|
|
||||||
declare that they have no known: 1377
|
|
||||||
that they have no known competing: 1376
|
|
||||||
of the creative commons attribution (: 1372
|
|
||||||
have no known competing financial interests: 1371
|
|
||||||
financial interests or personal relationships that: 1371
|
|
||||||
the creative commons attribution ( cc: 1370
|
|
||||||
creative commons attribution ( cc by: 1370
|
|
||||||
they have no known competing financial: 1370
|
|
||||||
to influence the work reported in: 1369
|
|
||||||
influence the work reported in this: 1369
|
|
||||||
interests or personal relationships that could: 1368
|
|
||||||
appeared to influence the work reported: 1368
|
|
||||||
or personal relationships that could have: 1367
|
|
||||||
have appeared to influence the work: 1367
|
|
||||||
could have appeared to influence the: 1364
|
|
||||||
relationships that could have appeared to: 1363
|
|
||||||
that could have appeared to influence: 1363
|
|
||||||
personal relationships that could have appeared: 1362
|
|
||||||
is distributed under the terms of: 1265
|
|
||||||
relationships that could be construed as: 1259
|
|
||||||
/ licenses / by / NUMBER: 1243
|
|
||||||
in the absence of any commercial: 1242
|
|
||||||
the absence of any commercial or: 1241
|
|
||||||
commercial or financial relationships that could: 1241
|
|
||||||
or financial relationships that could be: 1240
|
|
||||||
absence of any commercial or financial: 1239
|
|
||||||
any commercial or financial relationships that: 1239
|
|
||||||
as a potential conflict of interest: 1239
|
|
||||||
was conducted in the absence of: 1238
|
|
||||||
of any commercial or financial relationships: 1238
|
|
||||||
conducted in the absence of any: 1237
|
|
||||||
financial relationships that could be construed: 1236
|
|
||||||
be construed as a potential conflict: 1236
|
|
||||||
the research was conducted in the: 1235
|
|
||||||
construed as a potential conflict of: 1235
|
|
||||||
that the research was conducted in: 1233
|
|
||||||
research was conducted in the absence: 1233
|
|
||||||
:// doi.org / NUMBER / s: 1220
|
|
||||||
doi.org / NUMBER / s NUMBER: 1220
|
|
||||||
declare that the research was conducted: 1212
|
|
||||||
authors declare that the research was: 1204
|
|
||||||
read and approved the final manuscript: 1196
|
|
||||||
study is available from the corresponding: 1163
|
|
||||||
creativecommons.org / licenses / by /: 1148
|
|
||||||
:// creativecommons.org / licenses / by: 1147
|
|
||||||
material available at https :// doi: 1135
|
|
||||||
authors read and approved the final: 1074
|
|
||||||
http :// creativecommons.org / licenses /: 1063
|
|
||||||
that they have no conflict of: 1061
|
|
||||||
all authors read and approved the: 1055
|
|
||||||
requests for materials should be addressed: 1054
|
|
||||||
for materials should be addressed to: 1054
|
|
||||||
correspondence and requests for materials should: 1053
|
|
||||||
and requests for materials should be: 1052
|
|
||||||
findings of this study is available: 1046
|
|
||||||
they have no conflict of interest: 1044
|
|
||||||
declare that they have no conflict: 1020
|
|
||||||
", visit http :// creat iveco": 1016
|
|
||||||
licence , visit http :// creat: 1015
|
|
||||||
the authors declare that the research: 993
|
|
||||||
", on the other hand ,": 989
|
|
||||||
available at https :// doi.org /: 925
|
|
||||||
informed consent was obtained from all: 904
|
|
||||||
this article is protected by copyright: 882
|
|
||||||
found online at https :// doi: 871
|
|
||||||
additional supporting information may be found: 866
|
|
||||||
and moral rights for the publications: 865
|
|
||||||
moral rights for the publications made: 865
|
|
||||||
rights for the publications made accessible: 865
|
|
||||||
for the publications made accessible in: 865
|
|
||||||
is retained by the authors and: 865
|
|
||||||
retained by the authors and /: 865
|
|
||||||
by the authors and / or: 865
|
|
||||||
the authors and / or other: 865
|
|
||||||
authors and / or other copyright: 865
|
|
||||||
and / or other copyright owners: 865
|
|
||||||
/ or other copyright owners and: 865
|
|
||||||
or other copyright owners and it: 865
|
|
||||||
other copyright owners and it is: 865
|
|
||||||
copyright owners and it is a: 865
|
|
||||||
owners and it is a condition: 865
|
|
||||||
and it is a condition of: 865
|
|
||||||
it is a condition of accessing: 865
|
|
||||||
is a condition of accessing publications: 865
|
|
||||||
a condition of accessing publications that: 865
|
|
||||||
condition of accessing publications that users: 865
|
|
||||||
of accessing publications that users recognise: 865
|
|
||||||
accessing publications that users recognise and: 865
|
|
||||||
publications that users recognise and abide: 865
|
|
||||||
that users recognise and abide by: 865
|
|
||||||
users recognise and abide by the: 865
|
|
||||||
recognise and abide by the legal: 865
|
|
||||||
and abide by the legal requirements: 865
|
|
||||||
abide by the legal requirements associated: 865
|
|
||||||
by the legal requirements associated with: 865
|
|
||||||
the legal requirements associated with this: 865
|
|
||||||
copyright and moral rights for the: 864
|
|
||||||
the publications made accessible in the: 864
|
|
||||||
publications made accessible in the public: 864
|
|
||||||
made accessible in the public portal: 864
|
|
||||||
accessible in the public portal is: 864
|
|
||||||
in the public portal is retained: 864
|
|
||||||
the public portal is retained by: 864
|
|
||||||
public portal is retained by the: 864
|
|
||||||
portal is retained by the authors: 864
|
|
||||||
legal requirements associated with this rights: 863
|
|
||||||
the terms of the creative commons: 839
|
|
||||||
and reproduction in any medium ,: 834
|
|
||||||
reproduction in any medium , provided: 833
|
|
||||||
declare that they have no competing: 823
|
|
||||||
conflicts of interest with respect to: 811
|
|
||||||
this work was supported by the: 804
|
|
||||||
that they have no competing interests: 783
|
|
||||||
respect to the research , authorship: 781
|
|
||||||
with respect to the research ,: 780
|
|
||||||
potential conflicts of interest with respect: 779
|
|
||||||
interest with respect to the research: 776
|
|
||||||
of article NUMBER fa of the: 775
|
|
||||||
article NUMBER fa of the dutch: 775
|
|
||||||
NUMBER fa of the dutch copyright: 775
|
|
||||||
fa of the dutch copyright act: 775
|
|
||||||
potential conflict of interest was reported: 775
|
|
||||||
conflict of interest was reported by: 775
|
|
||||||
no potential conflict of interest was: 773
|
|
||||||
of interest was reported by the: 770
|
|
||||||
in the present study , we: 767
|
|
||||||
under the terms of article NUMBER: 766
|
|
||||||
the terms of article NUMBER fa: 766
|
|
||||||
terms of article NUMBER fa of: 766
|
|
||||||
this study is available from the: 765
|
|
||||||
support for the research , authorship: 762
|
|
||||||
financial support for the research ,: 760
|
|
||||||
of the dutch copyright act ,: 758
|
|
||||||
distributed under the terms of the: 743
|
|
||||||
publication is distributed under the terms: 742
|
|
||||||
distributed under the terms of article: 742
|
|
||||||
the original work is properly cited: 738
|
|
||||||
if the publication is distributed under: 734
|
|
||||||
the publication is distributed under the: 734
|
|
||||||
the dutch copyright act , indicated: 734
|
|
||||||
dutch copyright act , indicated by: 734
|
|
||||||
copyright act , indicated by the: 734
|
|
||||||
act , indicated by the taverne: 734
|
|
||||||
", indicated by the taverne license": 734
|
|
||||||
indicated by the taverne license above: 734
|
|
||||||
by the taverne license above ,: 734
|
|
||||||
the taverne license above , please: 734
|
|
||||||
taverne license above , please follow: 734
|
|
||||||
license above , please follow below: 734
|
|
||||||
above , please follow below link: 734
|
|
||||||
", please follow below link for": 734
|
|
||||||
please follow below link for the: 734
|
|
||||||
follow below link for the end: 734
|
|
||||||
online at https :// doi.org /: 733
|
|
||||||
of this study is available from: 731
|
|
||||||
to the research , authorship ,: 728
|
|
||||||
from NUMBER % to NUMBER %: 728
|
|
||||||
CITATION , NUMBER CITATION , NUMBER: 723
|
|
||||||
reprints and permission information is available: 722
|
|
||||||
information is available at http ://: 722
|
|
||||||
available at http :// www.nature.com /: 722
|
|
||||||
is available at http :// www.nature.com: 721
|
|
||||||
at http :// www.nature.com / reprints: 721
|
|
||||||
terms of the creative commons attribution: 717
|
|
||||||
and permission information is available at: 710
|
|
||||||
permission information is available at http: 710
|
|
||||||
provided the original work is properly: 707
|
|
||||||
", provided the original work is": 706
|
|
||||||
http :// www.nature.com / reprints publisher's: 706
|
|
||||||
:// www.nature.com / reprints publisher's note: 705
|
|
||||||
www.nature.com / reprints publisher's note springer: 705
|
|
||||||
/ reprints publisher's note springer nature: 705
|
|
||||||
reprints publisher's note springer nature remains: 705
|
|
||||||
associated with this article can be: 700
|
|
||||||
with this article can be found: 695
|
|
||||||
available from the corresponding author upon: 691
|
|
||||||
no potential conflicts of interest with: 685
|
|
||||||
for the research , authorship ,: 685
|
|
||||||
", visit http :// creativecommons.org /": 683
|
|
||||||
visit http :// creativecommons.org / licenses: 683
|
|
||||||
conflict of interest the authors declare: 678
|
|
||||||
declared no potential conflicts of interest: 666
|
|
||||||
visit http :// creat iveco mmons: 665
|
|
||||||
for this article can be found: 658
|
|
||||||
had no role in the design: 651
|
|
||||||
material for this article can be: 645
|
|
||||||
this is a open access article: 644
|
|
||||||
the corresponding author upon reasonable request: 637
|
|
||||||
supplementary material for this article can: 633
|
|
||||||
from the corresponding author upon reasonable: 633
|
|
||||||
the final version of the manuscript: 624
|
|
||||||
the author ( s ) declared: 618
|
|
||||||
the supplementary material for this article: 617
|
|
||||||
in the public , commercial ,: 613
|
|
||||||
the public , commercial , or: 612
|
|
||||||
by the author ( s ): 599
|
|
||||||
of interest the authors declare that: 594
|
|
||||||
available from the corresponding author on: 589
|
|
||||||
this article can be found ,: 587
|
|
||||||
article can be found , in: 587
|
|
||||||
link to the creative commons license: 585
|
|
||||||
be found in the online version: 584
|
|
||||||
", the aim of this study": 583
|
|
||||||
be found online in the supporting: 583
|
|
||||||
found online in the supporting information: 583
|
|
||||||
", it has been shown that": 582
|
|
||||||
we would like to thank the: 580
|
|
||||||
supporting information may be found online: 579
|
|
||||||
information may be found online in: 577
|
|
||||||
may be found online in the: 577
|
|
||||||
be found , in the online: 577
|
|
||||||
found , in the online version: 577
|
|
||||||
license , and indicate if changes: 576
|
|
||||||
", in the online version ,": 574
|
|
||||||
in the online version , at: 574
|
|
||||||
for their contribution to the peer: 574
|
|
||||||
their contribution to the peer review: 574
|
|
||||||
this research did not receive any: 572
|
|
||||||
reprints and permissions information is available: 572
|
|
||||||
to the creative commons license ,: 571
|
|
||||||
the creative commons license , and: 571
|
|
||||||
creative commons license , and indicate: 571
|
|
||||||
commons license , and indicate if: 571
|
|
||||||
the funders had no role in: 570
|
|
||||||
and permissions information is available at: 570
|
|
||||||
did not receive any specific grant: 566
|
|
||||||
agencies in the public , commercial: 566
|
|
||||||
from the corresponding author on reasonable: 564
|
|
||||||
in any medium , provided the: 563
|
|
||||||
any medium , provided the original: 562
|
|
||||||
to the peer review of this: 562
|
|
||||||
the peer review of this work: 562
|
|
||||||
not receive any specific grant from: 561
|
|
||||||
from funding agencies in the public: 561
|
|
||||||
the corresponding author on reasonable request: 561
|
|
||||||
is available at www.nature.com / reprints: 559
|
|
||||||
funding agencies in the public ,: 558
|
|
||||||
permissions information is available at www.nature.com: 558
|
|
||||||
information is available at www.nature.com /: 558
|
|
||||||
research did not receive any specific: 552
|
|
||||||
/ NUMBER / s NUMBER correspondence: 552
|
|
||||||
NUMBER correspondence and requests for materials: 552
|
|
||||||
below link for the end user: 551
|
|
||||||
NUMBER / s NUMBER correspondence and: 551
|
|
||||||
/ s NUMBER correspondence and requests: 551
|
|
||||||
s NUMBER correspondence and requests for: 551
|
|
||||||
online in the supporting information section: 550
|
|
||||||
public , commercial , or not-for-profit: 549
|
|
||||||
found online at https :// www.frontiersin.org: 549
|
|
||||||
online at https :// www.frontiersin.org /: 549
|
|
||||||
at https :// www.frontiersin.org / articles: 548
|
|
||||||
https :// www.frontiersin.org / articles /: 548
|
|
||||||
", distribution , and reproduction in": 546
|
|
||||||
distribution , and reproduction in any: 546
|
|
||||||
", and reproduction in any medium": 546
|
|
||||||
:// www.frontiersin.org / articles / NUMBER: 545
|
|
||||||
www.frontiersin.org / articles / NUMBER /: 544
|
|
||||||
contribution to the peer review of: 542
|
|
||||||
grant from funding agencies in the: 541
|
|
||||||
use , distribution , and reproduction: 537
|
|
||||||
", commercial , or not-for-profit sectors": 535
|
|
||||||
received no financial support for the: 535
|
|
||||||
no financial support for the research: 534
|
|
||||||
receive any specific grant from funding: 529
|
|
||||||
specific grant from funding agencies in: 529
|
|
||||||
approved the final version of the: 529
|
|
||||||
any specific grant from funding agencies: 528
|
|
||||||
author ( s ) declared no: 526
|
|
||||||
is available online at https ://: 525
|
|
||||||
( s ) declared no potential: 525
|
|
||||||
s ) declared no potential conflicts: 525
|
|
||||||
) declared no potential conflicts of: 525
|
|
||||||
article is distributed under the terms: 520
|
|
||||||
this article is distributed under the: 518
|
|
||||||
"% and NUMBER % , respectively": 507
|
|
||||||
access this article is distributed under: 501
|
|
||||||
open access this article is distributed: 498
|
|
||||||
authors declare that there is no: 487
|
|
||||||
the following is available online at: 485
|
|
||||||
NUMBER % , NUMBER % ,: 484
|
|
||||||
authors would like to thank the: 479
|
|
||||||
is a open access article under: 477
|
|
||||||
a open access article under the: 477
|
|
||||||
medium , provided the original work: 473
|
|
||||||
/ NUMBER /) , which permits: 471
|
|
||||||
the online version , at doi: 466
|
|
||||||
interest was reported by the author: 464
|
|
||||||
supplementary material associated with this article: 452
|
|
||||||
material associated with this article can: 452
|
|
||||||
we would like to thank all: 449
|
|
||||||
http :// creat iveco mmons.org /: 448
|
|
||||||
:// creat iveco mmons.org / licen: 446
|
|
||||||
creat iveco mmons.org / licen ses: 446
|
|
||||||
iveco mmons.org / licen ses /: 446
|
|
||||||
following is available online at https: 446
|
|
||||||
available online at https :// www.mdpi.com: 445
|
|
||||||
online at https :// www.mdpi.com /: 445
|
|
||||||
union's horizon NUMBER research and innovation: 443
|
|
||||||
during the current study is available: 443
|
|
||||||
european union's horizon NUMBER research and: 441
|
|
||||||
the netherlands organization for scientific research: 441
|
|
||||||
the authors declare that there is: 441
|
|
||||||
NUMBER international license ( http ://: 440
|
|
||||||
of the authors and do not: 439
|
|
||||||
www.mdpi.com / article / NUMBER /: 438
|
|
||||||
NUMBER CITATION , NUMBER CITATION ,: 436
|
|
||||||
access article under the terms of: 435
|
|
||||||
article under the terms of the: 435
|
|
||||||
/ by / NUMBER /) ,: 435
|
|
||||||
by / NUMBER /) , which: 435
|
|
||||||
/ licen ses / by /: 434
|
|
||||||
licen ses / by / NUMBER: 434
|
|
||||||
open access article under the terms: 434
|
|
||||||
current study is available from the: 434
|
|
||||||
the current study is available from: 433
|
|
||||||
", which permits unrestricted use ,": 431
|
|
||||||
mmons.org / licen ses / by: 429
|
|
||||||
have no conflicts of interest to: 427
|
|
||||||
which permits unrestricted use , distribution: 427
|
|
||||||
", there is a need for": 425
|
|
||||||
permits unrestricted use , distribution ,: 425
|
|
||||||
unrestricted use , distribution , and: 425
|
|
||||||
have read and agreed to the: 423
|
|
||||||
authors have read and agreed to: 422
|
|
||||||
any medium , provided you give: 422
|
|
||||||
medium , provided you give appropriate: 422
|
|
||||||
", provided you give appropriate credit": 422
|
|
||||||
provided you give appropriate credit to: 422
|
|
||||||
was reported by the author (: 422
|
|
||||||
reported by the author ( s: 422
|
|
||||||
NUMBER /) , which permits unrestricted: 421
|
|
||||||
in any medium , provided you: 421
|
|
||||||
peer review information nature communications thanks: 421
|
|
||||||
the european union's horizon NUMBER research: 420
|
|
||||||
and agreed to the published version: 419
|
|
||||||
agreed to the published version of: 419
|
|
||||||
the published version of the manuscript: 419
|
|
||||||
to the published version of the: 418
|
|
||||||
the authors and do not necessarily: 417
|
|
||||||
/) , which permits unrestricted use: 417
|
|
||||||
read and agreed to the published: 416
|
|
||||||
those of the authors and do: 414
|
|
||||||
all authors have read and agreed: 413
|
|
||||||
that there is no conflict of: 411
|
|
||||||
attribution NUMBER international license ( http: 410
|
|
||||||
author ( s ) received no: 405
|
|
||||||
consent was obtained from all subjects: 404
|
|
||||||
authors have no conflicts of interest: 403
|
|
||||||
( s ) received no financial: 403
|
|
||||||
s ) received no financial support: 403
|
|
||||||
) received no financial support for: 403
|
|
||||||
NUMBER % and NUMBER % of: 400
|
|
||||||
contributed to the article and approved: 400
|
|
||||||
to the article and approved the: 400
|
|
||||||
authors contributed to the article and: 397
|
|
||||||
the author ( s ) received: 396
|
|
||||||
the article and approved the submitted: 396
|
|
||||||
article and approved the submitted version: 395
|
|
||||||
informed consent statement informed consent was: 393
|
|
||||||
accountable for all aspects of the: 389
|
|
||||||
https :// www.mdpi.com / article /: 389
|
|
||||||
:// www.mdpi.com / article / NUMBER: 389
|
|
||||||
consent statement informed consent was obtained: 389
|
|
||||||
at https :// www.mdpi.com / article: 388
|
|
||||||
statement informed consent was obtained from: 388
|
|
||||||
no role in the design of: 385
|
|
||||||
) for their contribution to the: 383
|
|
||||||
", we would like to thank": 381
|
|
||||||
is available on request from the: 378
|
|
||||||
had no role in study design: 377
|
|
||||||
responsibility for the integrity of the: 375
|
|
||||||
of the creative commons attribution NUMBER: 375
|
|
||||||
in the supporting information section at: 374
|
|
||||||
the supporting information section at the: 374
|
|
||||||
supporting information section at the end: 374
|
|
||||||
reviewer ( s ) for their: 374
|
|
||||||
this study is available on request: 373
|
|
||||||
all authors contributed to the article: 373
|
|
||||||
( s ) for their contribution: 372
|
|
||||||
s ) for their contribution to: 372
|
|
||||||
", analysis and interpretation of data": 371
|
|
||||||
available on request from the corresponding: 370
|
|
||||||
in accordance with the ethical standards: 370
|
|
||||||
the manuscript for important intellectual content: 368
|
|
||||||
for all aspects of the work: 367
|
|
||||||
role in the design of the: 367
|
|
||||||
authors declare no conflicts of interest: 366
|
|
||||||
", analysis , and interpretation of": 365
|
|
||||||
on request from the corresponding author: 364
|
|
||||||
be accountable for all aspects of: 362
|
|
||||||
obtained from all subjects involved in: 361
|
|
||||||
funders had no role in the: 360
|
|
||||||
for scientific research ( nwo ): 359
|
|
||||||
the creative commons attribution NUMBER international: 359
|
|
||||||
was obtained from all subjects involved: 359
|
|
||||||
authors and do not necessarily represent: 358
|
|
||||||
to be accountable for all aspects: 356
|
|
||||||
original work is properly cited ,: 355
|
|
||||||
data in the writing of the: 354
|
|
||||||
no role in study design ,: 353
|
|
||||||
from all subjects involved in the: 353
|
|
||||||
all subjects involved in the study: 351
|
|
||||||
provenance and peer review not commissioned: 346
|
|
||||||
study is available on request from: 346
|
|
||||||
there is no conflict of interest: 346
|
|
||||||
wrote the first draft of the: 345
|
|
||||||
NUMBER / NUMBER / NUMBER /: 345
|
|
||||||
written informed consent to participate in: 341
|
|
||||||
commons attribution NUMBER international license (: 341
|
|
||||||
interpretation of data in the writing: 339
|
|
||||||
of data in the writing of: 339
|
|
||||||
ses / by / NUMBER /: 337
|
|
||||||
we would also like to thank: 336
|
|
||||||
the integrity of the data and: 335
|
|
||||||
and peer review not commissioned externally: 334
|
|
||||||
of this study was to assess: 334
|
|
||||||
peer review not commissioned externally peer: 332
|
|
||||||
review not commissioned externally peer reviewed: 332
|
|
||||||
declare that there is no conflict: 331
|
|
||||||
role in study design , data: 328
|
|
||||||
written informed consent was obtained from: 327
|
|
||||||
no conflicts of interest to declare: 326
|
|
||||||
consent to participate in this study: 326
|
|
||||||
in the article's creative commons license: 326
|
|
||||||
do not necessarily represent those of: 325
|
|
||||||
( http :// creativecommons.org / licenses: 324
|
|
||||||
the authors have no conflicts of: 322
|
|
||||||
informed consent to participate in this: 322
|
|
||||||
with the ethical standards of the: 322
|
|
||||||
license ( http :// creativecommons.org /: 322
|
|
||||||
solely those of the authors and: 322
|
|
||||||
and do not necessarily represent those: 322
|
|
||||||
in this article is solely those: 320
|
|
||||||
this article is solely those of: 320
|
|
||||||
article is solely those of the: 320
|
|
||||||
integrity of the data and the: 319
|
|
||||||
of the data and the accuracy: 319
|
|
||||||
the data and the accuracy of: 319
|
|
||||||
accordance with the ethical standards of: 319
|
|
||||||
data and the accuracy of the: 318
|
|
||||||
during the conduct of the study: 317
|
|
||||||
the creative commons attribution CITATION ,: 317
|
|
||||||
expressed in this article is solely: 316
|
|
||||||
and approved the final version of: 315
|
|
||||||
publisher's note all claims expressed in: 315
|
|
||||||
note all claims expressed in this: 315
|
|
||||||
all claims expressed in this article: 315
|
|
||||||
claims expressed in this article is: 315
|
|
||||||
not necessarily represent those of their: 315
|
|
||||||
necessarily represent those of their affiliated: 315
|
|
||||||
represent those of their affiliated organizations: 315
|
|
||||||
or those of the publisher ,: 315
|
|
||||||
those of the publisher , the: 314
|
|
||||||
of the publisher , the editors: 314
|
|
||||||
in the writing of the manuscript: 313
|
|
||||||
affiliated organizations , or those of: 312
|
|
||||||
organizations , or those of the: 312
|
|
||||||
or claim that may be made: 312
|
|
||||||
claim that may be made by: 312
|
|
||||||
that may be made by its: 312
|
|
||||||
is not guaranteed or endorsed by: 312
|
|
||||||
not guaranteed or endorsed by the: 312
|
|
||||||
guaranteed or endorsed by the publisher: 312
|
|
||||||
authors would like to thank all: 311
|
|
||||||
related to this article can be: 311
|
|
||||||
those of their affiliated organizations ,: 311
|
|
||||||
of their affiliated organizations , or: 311
|
|
||||||
their affiliated organizations , or those: 311
|
|
||||||
", or those of the publisher": 311
|
|
||||||
any product that may be evaluated: 311
|
|
||||||
this article , or claim that: 311
|
|
||||||
article , or claim that may: 311
|
|
||||||
", or claim that may be": 311
|
|
||||||
product that may be evaluated in: 310
|
|
||||||
may be made by its manufacturer: 310
|
|
||||||
the accuracy of the data analysis: 309
|
|
||||||
", there is a need to": 309
|
|
||||||
of the creative commons attribution license: 309
|
|
||||||
that may be evaluated in this: 309
|
|
||||||
may be evaluated in this article: 309
|
|
||||||
be made by its manufacturer ,: 309
|
|
||||||
made by its manufacturer , is: 309
|
|
||||||
by its manufacturer , is not: 309
|
|
||||||
its manufacturer , is not guaranteed: 309
|
|
||||||
manufacturer , is not guaranteed or: 309
|
|
||||||
", is not guaranteed or endorsed": 309
|
|
||||||
", there is a lack of": 308
|
|
||||||
the authors declare no conflicts of: 308
|
|
||||||
be evaluated in this article ,: 308
|
|
||||||
evaluated in this article , or: 308
|
|
||||||
in this article , or claim: 308
|
|
||||||
the data in the study and: 306
|
|
||||||
in this paper , we present: 306
|
|
||||||
information may be found in the: 305
|
|
||||||
this study was to investigate the: 305
|
|
||||||
", on the one hand ,": 305
|
|
||||||
may be found in the online: 304
|
|
||||||
is not publicly available due to: 304
|
|
||||||
authors report no conflicts of interest: 304
|
|
||||||
online version , at doi NUMBER: 303
|
|
||||||
version , at doi NUMBER /: 303
|
|
||||||
supporting information may be found in: 302
|
|
||||||
in the public , commercial or: 301
|
|
||||||
from any funding agency in the: 300
|
|
||||||
any funding agency in the public: 300
|
|
||||||
were in accordance with the ethical: 300
|
|
||||||
the publisher , the editors and: 300
|
|
||||||
publisher , the editors and the: 300
|
|
||||||
", the editors and the reviewers": 300
|
|
||||||
funding agency in the public ,: 299
|
|
||||||
agency in the public , commercial: 299
|
|
||||||
found in the online version of: 297
|
|
||||||
there is no conflicts of interest: 295
|
|
||||||
can be found in the online: 295
|
|
||||||
article can be found in the: 291
|
|
||||||
use , distribution and reproduction in: 289
|
|
||||||
the online version of this article: 287
|
|
||||||
before it is published in its: 287
|
|
||||||
it is published in its final: 287
|
|
||||||
to submit the manuscript for publication: 286
|
|
||||||
international license ( http :// creativecommons.org: 285
|
|
||||||
is published in its final form: 285
|
|
||||||
decision to submit the manuscript for: 284
|
|
||||||
", which permits use , distribution": 284
|
|
||||||
which permits use , distribution and: 284
|
|
||||||
permits use , distribution and reproduction: 284
|
|
||||||
", as shown in fig. NUMBER": 284
|
|
||||||
errors may be discovered which could: 284
|
|
||||||
may be discovered which could affect: 284
|
|
||||||
be discovered which could affect the: 284
|
|
||||||
discovered which could affect the content: 284
|
|
||||||
which could affect the content ,: 284
|
|
||||||
could affect the content , and: 284
|
|
||||||
affect the content , and all: 284
|
|
||||||
the content , and all legal: 284
|
|
||||||
content , and all legal disclaimers: 284
|
|
||||||
all legal disclaimers that apply to: 283
|
|
||||||
legal disclaimers that apply to the: 283
|
|
||||||
disclaimers that apply to the journal: 283
|
|
||||||
design , data collection and analysis: 282
|
|
||||||
", and all legal disclaimers that": 282
|
|
||||||
and all legal disclaimers that apply: 282
|
|
||||||
from the european union's horizon NUMBER: 280
|
|
||||||
and the other , anonymous ,: 280
|
|
||||||
is a pdf file of a: 278
|
|
||||||
this article can be found in: 277
|
|
||||||
helsinki declaration and its later amendments: 276
|
|
||||||
NUMBER helsinki declaration and its later: 275
|
|
||||||
the NUMBER helsinki declaration and its: 274
|
|
||||||
during the production process , errors: 274
|
|
||||||
the production process , errors may: 274
|
|
||||||
production process , errors may be: 274
|
|
||||||
process , errors may be discovered: 274
|
|
||||||
this is a pdf file of: 273
|
|
||||||
review before it is published in: 273
|
|
||||||
", errors may be discovered which": 273
|
|
||||||
will undergo additional copyediting , typesetting: 271
|
|
||||||
and review before it is published: 271
|
|
||||||
note that , during the production: 271
|
|
||||||
that , during the production process: 271
|
|
||||||
", during the production process ,": 271
|
|
||||||
with the NUMBER helsinki declaration and: 270
|
|
||||||
this version will undergo additional copyediting: 270
|
|
||||||
version will undergo additional copyediting ,: 270
|
|
||||||
undergo additional copyediting , typesetting and: 270
|
|
||||||
additional copyediting , typesetting and review: 270
|
|
||||||
copyediting , typesetting and review before: 270
|
|
||||||
", typesetting and review before it": 270
|
|
||||||
typesetting and review before it is: 270
|
|
||||||
published in its final form ,: 270
|
|
||||||
in its final form , but: 270
|
|
||||||
its final form , but we: 270
|
|
||||||
final form , but we is: 270
|
|
||||||
form , but we is providing: 270
|
|
||||||
", but we is providing this": 270
|
|
||||||
but we is providing this version: 270
|
|
||||||
we is providing this version to: 270
|
|
||||||
is providing this version to give: 270
|
|
||||||
providing this version to give early: 270
|
|
||||||
this version to give early visibility: 270
|
|
||||||
version to give early visibility of: 270
|
|
||||||
to give early visibility of the: 270
|
|
||||||
give early visibility of the article: 270
|
|
||||||
please note that , during the: 270
|
|
||||||
netherlands organization for scientific research (: 269
|
|
||||||
acute respiratory syndrome coronavirus NUMBER CITATION: 268
|
|
||||||
a pdf file of a article: 268
|
|
||||||
pdf file of a article that: 268
|
|
||||||
file of a article that has: 268
|
|
||||||
of a article that has undergone: 268
|
|
||||||
a article that has undergone enhancements: 268
|
|
||||||
article that has undergone enhancements after: 268
|
|
||||||
that has undergone enhancements after acceptance: 268
|
|
||||||
has undergone enhancements after acceptance ,: 268
|
|
||||||
undergone enhancements after acceptance , such: 268
|
|
||||||
enhancements after acceptance , such as: 268
|
|
||||||
after acceptance , such as the: 268
|
|
||||||
acceptance , such as the addition: 268
|
|
||||||
as the addition of a cover: 268
|
|
||||||
the addition of a cover page: 268
|
|
||||||
addition of a cover page and: 268
|
|
||||||
of a cover page and metadata: 268
|
|
||||||
a cover page and metadata ,: 268
|
|
||||||
cover page and metadata , and: 268
|
|
||||||
page and metadata , and formatting: 268
|
|
||||||
and metadata , and formatting for: 268
|
|
||||||
metadata , and formatting for readability: 268
|
|
||||||
", and formatting for readability ,": 268
|
|
||||||
and formatting for readability , but: 268
|
|
||||||
formatting for readability , but it: 268
|
|
||||||
for readability , but it is: 268
|
|
||||||
readability , but it is not: 268
|
|
||||||
but it is not yet the: 268
|
|
||||||
it is not yet the definitive: 268
|
|
||||||
is not yet the definitive version: 268
|
|
||||||
not yet the definitive version of: 268
|
|
||||||
yet the definitive version of record: 268
|
|
||||||
and its later amendments or comparable: 267
|
|
||||||
its later amendments or comparable ethical: 267
|
|
||||||
the netherlands organisation for scientific research: 265
|
|
||||||
analysis and interpretation of data ,: 265
|
|
||||||
NUMBER https :// doi.org / NUMBER: 265
|
|
||||||
later amendments or comparable ethical standards: 265
|
|
||||||
the public , commercial or not-for-profit: 265
|
|
||||||
the aim of this paper is: 263
|
|
||||||
"% , NUMBER % and NUMBER": 262
|
|
||||||
the other , anonymous , reviewer: 261
|
|
||||||
and with the NUMBER helsinki declaration: 260
|
|
||||||
human participants were reviewed and approved: 259
|
|
||||||
participants were reviewed and approved by: 259
|
|
||||||
involving human participants were reviewed and: 258
|
|
||||||
studies involving human participants were reviewed: 257
|
|
||||||
and analysis , decision to publish: 257
|
|
||||||
it should be noted that the: 257
|
|
||||||
in the decision to publish the: 256
|
|
||||||
study design , data collection ,: 256
|
|
||||||
found in the online version at: 256
|
|
||||||
the authors report no conflicts of: 255
|
|
||||||
under the terms of the creat: 255
|
|
||||||
the terms of the creat ive: 255
|
|
||||||
terms of the creat ive commo: 255
|
|
||||||
of the creat ive commo ns: 255
|
|
||||||
the creat ive commo ns attri: 255
|
|
||||||
they have no conflicts of interest: 255
|
|
||||||
data presented in this study is: 255
|
|
||||||
participants provided their written informed consent: 254
|
|
||||||
their written informed consent to participate: 254
|
|
||||||
the decision to publish the results: 254
|
|
||||||
the data presented in this study: 254
|
|
||||||
provided their written informed consent to: 253
|
|
||||||
CITATION license , which permits others: 252
|
|
||||||
license , which permits others to: 252
|
|
||||||
all authors approved the final version: 251
|
|
||||||
that apply to the journal pertain: 251
|
|
||||||
anonymous , reviewer ( s ): 251
|
|
||||||
study is included in the article: 250
|
|
||||||
other , anonymous , reviewer (: 250
|
|
||||||
", anonymous , reviewer ( s": 250
|
|
||||||
", reviewer ( s ) for": 250
|
|
||||||
|
|
@ -1,145 +0,0 @@
|
||||||
import re
|
|
||||||
from typing import Dict, Pattern, Tuple, Union
|
|
||||||
|
|
||||||
from .models import BookmarkTitle
|
|
||||||
|
|
||||||
# partly inspired by https://github.com/KMCS-NII/AASC/blob/master/section_classify/section_classify.pl
|
|
||||||
titles_of_interest: Dict[BookmarkTitle, Tuple[Union[str, Pattern], ...]] = {
|
|
||||||
"Abstract": ("abstract",),
|
|
||||||
"Author contribution": (
|
|
||||||
"author contribution",
|
|
||||||
"credit authorship contribution statement",
|
|
||||||
"credit author statement",
|
|
||||||
"corresponding author",
|
|
||||||
),
|
|
||||||
"Introduction": (
|
|
||||||
"introduction",
|
|
||||||
"preliminaries",
|
|
||||||
"research question",
|
|
||||||
"objective",
|
|
||||||
"purpose",
|
|
||||||
re.compile(r"^aim$"),
|
|
||||||
"proposition",
|
|
||||||
"what this study adds",
|
|
||||||
"what does this study add",
|
|
||||||
),
|
|
||||||
"Background": (
|
|
||||||
"background",
|
|
||||||
"previous",
|
|
||||||
"prior",
|
|
||||||
"relevant work",
|
|
||||||
"state of the art",
|
|
||||||
"related work",
|
|
||||||
"research in context",
|
|
||||||
"earlier work",
|
|
||||||
"related work",
|
|
||||||
"problem",
|
|
||||||
"challenge",
|
|
||||||
"goal",
|
|
||||||
"literature review",
|
|
||||||
re.compile(r"^review$"),
|
|
||||||
re.compile(r"^existing"),
|
|
||||||
"comparison with existing literature",
|
|
||||||
"literature search",
|
|
||||||
"comparison with other studies",
|
|
||||||
"study area",
|
|
||||||
),
|
|
||||||
"Methods": (
|
|
||||||
"method",
|
|
||||||
"approach",
|
|
||||||
re.compile(r"^our"),
|
|
||||||
"proposed",
|
|
||||||
"study design",
|
|
||||||
"procedure",
|
|
||||||
"formulation",
|
|
||||||
"methodology",
|
|
||||||
"research design",
|
|
||||||
"study protocol",
|
|
||||||
"design",
|
|
||||||
re.compile(r"experimental (?!result)"),
|
|
||||||
"overview",
|
|
||||||
"approach",
|
|
||||||
"system",
|
|
||||||
"definition",
|
|
||||||
"algorithm",
|
|
||||||
"model",
|
|
||||||
"setup",
|
|
||||||
"implementation",
|
|
||||||
"evaluation metric",
|
|
||||||
"scheme",
|
|
||||||
),
|
|
||||||
"Results": (
|
|
||||||
"result",
|
|
||||||
"data",
|
|
||||||
"outcome",
|
|
||||||
"statistic",
|
|
||||||
"statistics",
|
|
||||||
"statistical analysis",
|
|
||||||
"analysis",
|
|
||||||
"measure",
|
|
||||||
"experiment",
|
|
||||||
"finding",
|
|
||||||
"hypothesis testing",
|
|
||||||
"proof of proposition",
|
|
||||||
"proof of concept",
|
|
||||||
"report",
|
|
||||||
"implication of all the available evidence",
|
|
||||||
"contribution",
|
|
||||||
"evaluation",
|
|
||||||
"performance",
|
|
||||||
"demonstration",
|
|
||||||
"demonstrating",
|
|
||||||
"example",
|
|
||||||
"study",
|
|
||||||
"studies",
|
|
||||||
"qualitative comparison",
|
|
||||||
"quantitative comparison",
|
|
||||||
"validation",
|
|
||||||
),
|
|
||||||
"Discussion": (
|
|
||||||
"discussion",
|
|
||||||
"recommendation",
|
|
||||||
"implication",
|
|
||||||
"findings",
|
|
||||||
"impact",
|
|
||||||
"limitation",
|
|
||||||
"assessment",
|
|
||||||
"quality assessment",
|
|
||||||
"interpretation",
|
|
||||||
"contribution",
|
|
||||||
"explanation",
|
|
||||||
),
|
|
||||||
"Outlook": (
|
|
||||||
"outlook",
|
|
||||||
"future research",
|
|
||||||
"future",
|
|
||||||
"follow up",
|
|
||||||
"remaining",
|
|
||||||
"prospect",
|
|
||||||
"perspective",
|
|
||||||
),
|
|
||||||
"Conclusion": (
|
|
||||||
"conclusion",
|
|
||||||
"conclude",
|
|
||||||
"concluding",
|
|
||||||
"concluding",
|
|
||||||
"final remark",
|
|
||||||
"remark",
|
|
||||||
"conclusie",
|
|
||||||
),
|
|
||||||
"Conflict of interest": (
|
|
||||||
"compete interest",
|
|
||||||
"funding support and author disclosure",
|
|
||||||
"conflict of interest",
|
|
||||||
"disclosure statement",
|
|
||||||
"role of the funding source",
|
|
||||||
"disclaimer",
|
|
||||||
"conflicting interest",
|
|
||||||
"declaration of interest",
|
|
||||||
"risk of bias",
|
|
||||||
"funding",
|
|
||||||
"disclosure",
|
|
||||||
"declaration",
|
|
||||||
),
|
|
||||||
"Acknowledgement": ("acknowledgement", "acknowledgment"),
|
|
||||||
}
|
|
||||||
|
|
@ -1,139 +0,0 @@
|
||||||
import unittest
|
|
||||||
|
|
||||||
from src.great_ai.utilities import lemmatize_text
|
|
||||||
|
|
||||||
|
|
||||||
class TestLemmatizeText(unittest.TestCase):
|
|
||||||
def test_simple(self) -> None:
|
|
||||||
text = "The state-of-the-art could not be improved, however we managed to create a less resource-intensive implementation of it."
|
|
||||||
|
|
||||||
lemmatized = [
|
|
||||||
"the",
|
|
||||||
"state",
|
|
||||||
"-",
|
|
||||||
"of",
|
|
||||||
"-",
|
|
||||||
"the",
|
|
||||||
"-",
|
|
||||||
"art",
|
|
||||||
"could",
|
|
||||||
"not",
|
|
||||||
"be",
|
|
||||||
"improve",
|
|
||||||
",",
|
|
||||||
"however",
|
|
||||||
"we",
|
|
||||||
"manage",
|
|
||||||
"to",
|
|
||||||
"create",
|
|
||||||
"a",
|
|
||||||
"less",
|
|
||||||
"resource",
|
|
||||||
"-",
|
|
||||||
"intensive",
|
|
||||||
"implementation",
|
|
||||||
"of",
|
|
||||||
"it",
|
|
||||||
".",
|
|
||||||
]
|
|
||||||
|
|
||||||
lemmatized_pos = [
|
|
||||||
"the_DET",
|
|
||||||
"state_NOUN",
|
|
||||||
"-_PUNCT",
|
|
||||||
"of_ADP",
|
|
||||||
"-_PUNCT",
|
|
||||||
"the_DET",
|
|
||||||
"-_PUNCT",
|
|
||||||
"art_NOUN",
|
|
||||||
"could_AUX",
|
|
||||||
"not_PART",
|
|
||||||
"be_AUX",
|
|
||||||
"improve_VERB",
|
|
||||||
",_PUNCT",
|
|
||||||
"however_ADV",
|
|
||||||
"we_PRON",
|
|
||||||
"manage_VERB",
|
|
||||||
"to_PART",
|
|
||||||
"create_VERB",
|
|
||||||
"a_DET",
|
|
||||||
"less_ADV",
|
|
||||||
"resource_NOUN",
|
|
||||||
"-_PUNCT",
|
|
||||||
"intensive_ADJ",
|
|
||||||
"implementation_NOUN",
|
|
||||||
"of_ADP",
|
|
||||||
"it_PRON",
|
|
||||||
"._PUNCT",
|
|
||||||
]
|
|
||||||
|
|
||||||
lemmatized_neg = [
|
|
||||||
"the",
|
|
||||||
"state",
|
|
||||||
"-",
|
|
||||||
"of",
|
|
||||||
"-",
|
|
||||||
"the",
|
|
||||||
"-",
|
|
||||||
"art",
|
|
||||||
"could",
|
|
||||||
"not",
|
|
||||||
"NOT_be",
|
|
||||||
"NOT_improve",
|
|
||||||
"NOT_,",
|
|
||||||
"however",
|
|
||||||
"we",
|
|
||||||
"manage",
|
|
||||||
"to",
|
|
||||||
"create",
|
|
||||||
"a",
|
|
||||||
"less",
|
|
||||||
"resource",
|
|
||||||
"-",
|
|
||||||
"intensive",
|
|
||||||
"implementation",
|
|
||||||
"of",
|
|
||||||
"it",
|
|
||||||
".",
|
|
||||||
]
|
|
||||||
|
|
||||||
lemmatized_pos_neg = [
|
|
||||||
"the_DET",
|
|
||||||
"state_NOUN",
|
|
||||||
"-_PUNCT",
|
|
||||||
"of_ADP",
|
|
||||||
"-_PUNCT",
|
|
||||||
"the_DET",
|
|
||||||
"-_PUNCT",
|
|
||||||
"art_NOUN",
|
|
||||||
"could_AUX",
|
|
||||||
"not_PART",
|
|
||||||
"NOT_be_AUX",
|
|
||||||
"NOT_improve_VERB",
|
|
||||||
"NOT_,_PUNCT",
|
|
||||||
"however_ADV",
|
|
||||||
"we_PRON",
|
|
||||||
"manage_VERB",
|
|
||||||
"to_PART",
|
|
||||||
"create_VERB",
|
|
||||||
"a_DET",
|
|
||||||
"less_ADV",
|
|
||||||
"resource_NOUN",
|
|
||||||
"-_PUNCT",
|
|
||||||
"intensive_ADJ",
|
|
||||||
"implementation_NOUN",
|
|
||||||
"of_ADP",
|
|
||||||
"it_PRON",
|
|
||||||
"._PUNCT",
|
|
||||||
]
|
|
||||||
|
|
||||||
assert lemmatize_text(text) == lemmatized
|
|
||||||
assert lemmatize_text(text, add_part_of_speech=True) == lemmatized_pos
|
|
||||||
assert lemmatize_text(text, add_negation=True) == lemmatized_neg
|
|
||||||
self.assertEqual(
|
|
||||||
lemmatize_text(text, add_negation=True, add_part_of_speech=True),
|
|
||||||
lemmatized_pos_neg,
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_empty(self) -> None:
|
|
||||||
assert lemmatize_text("") == []
|
|
||||||
|
|
@ -1,19 +0,0 @@
|
||||||
import unittest
|
|
||||||
|
|
||||||
from src.great_ai.utilities import lemmatize_token, nlp
|
|
||||||
|
|
||||||
|
|
||||||
class TestLemmatizeToken(unittest.TestCase):
|
|
||||||
def test_simple(self) -> None:
|
|
||||||
token = nlp("Center")[0]
|
|
||||||
|
|
||||||
assert lemmatize_token(token) == "centre"
|
|
||||||
assert lemmatize_token(token, add_negation=True) == "centre"
|
|
||||||
assert lemmatize_token(token, add_part_of_speech=True) == "centre_NOUN"
|
|
||||||
|
|
||||||
def test_punctuation(self) -> None:
|
|
||||||
token = nlp("This.")[1]
|
|
||||||
|
|
||||||
assert lemmatize_token(token) == "."
|
|
||||||
assert lemmatize_token(token, add_negation=True) == "."
|
|
||||||
assert lemmatize_token(token, add_part_of_speech=True) == "._PUNCT"
|
|
||||||
|
|
@ -1,67 +0,0 @@
|
||||||
import unittest
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from src.great_ai.utilities import PublicationTEI
|
|
||||||
|
|
||||||
from .data.parsed import (
|
|
||||||
abstract,
|
|
||||||
authors,
|
|
||||||
bookmarks,
|
|
||||||
conclusion,
|
|
||||||
content,
|
|
||||||
introduction,
|
|
||||||
metadata,
|
|
||||||
sentences,
|
|
||||||
)
|
|
||||||
|
|
||||||
DATA_PATH = Path(__file__).parent.resolve() / "data"
|
|
||||||
|
|
||||||
|
|
||||||
class TestPublicationTEI(unittest.TestCase):
|
|
||||||
test_xml: str
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def setUpClass(cls) -> None:
|
|
||||||
with open(
|
|
||||||
DATA_PATH / "10.1136_bmjspcare-2021-003026.pdf.tei.xml", encoding="utf-8"
|
|
||||||
) as f:
|
|
||||||
cls.test_xml = f.read()
|
|
||||||
|
|
||||||
def test_metadata_extraction(self) -> None:
|
|
||||||
assert PublicationTEI(self.test_xml).publication_metadata == metadata
|
|
||||||
|
|
||||||
def test_authors(self) -> None:
|
|
||||||
assert PublicationTEI(self.test_xml).authors == authors
|
|
||||||
|
|
||||||
def test_content(self) -> None:
|
|
||||||
assert PublicationTEI(self.test_xml).content == content
|
|
||||||
|
|
||||||
def test_sentences(self) -> None:
|
|
||||||
assert PublicationTEI(self.test_xml).sentences == sentences
|
|
||||||
|
|
||||||
def test_bookmarks(self) -> None:
|
|
||||||
assert PublicationTEI(self.test_xml).bookmarks == bookmarks
|
|
||||||
|
|
||||||
def test_abstract(self) -> None:
|
|
||||||
assert PublicationTEI(self.test_xml).abstract_sentences == abstract
|
|
||||||
|
|
||||||
def test_introduction(self) -> None:
|
|
||||||
self.assertEqual(
|
|
||||||
PublicationTEI(self.test_xml).introduction_sentences, introduction
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_conclusion(self) -> None:
|
|
||||||
assert PublicationTEI(self.test_xml).conclusion_sentences == conclusion
|
|
||||||
|
|
||||||
def test_empty1(self) -> None:
|
|
||||||
tei = PublicationTEI("<TEI/>")
|
|
||||||
tei.publication_metadata, tei.publication_metadata, tei.authors, tei.content, tei.sentences
|
|
||||||
|
|
||||||
def test_empty2(self) -> None:
|
|
||||||
tei = PublicationTEI("")
|
|
||||||
tei.publication_metadata, tei.publication_metadata, tei.authors, tei.content, tei.sentences
|
|
||||||
|
|
||||||
def test_missing_fields(self) -> None:
|
|
||||||
with open(DATA_PATH / "bad.tei.xml", encoding="utf-8") as f:
|
|
||||||
tei = PublicationTEI(f.read())
|
|
||||||
tei.publication_metadata, tei.publication_metadata, tei.authors, tei.content, tei.sentences
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue