Remove Spacy and Publication TEI
This commit is contained in:
parent
e16db17db0
commit
46c99a06f9
30 changed files with 5 additions and 2412 deletions
|
|
@ -19,9 +19,6 @@ RUN python3 -m pip --no-cache-dir install --upgrade pip &&\
|
|||
pip install --no-cache-dir ./great_ai &&\
|
||||
rm -rf great_ai
|
||||
|
||||
# great_ai.utilities.nlp depends on this
|
||||
RUN pip3 install --no-cache-dir en-core-web-sm@https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl
|
||||
|
||||
HEALTHCHECK \
|
||||
--interval=60s \
|
||||
--timeout=60s \
|
||||
|
|
|
|||
|
|
@ -1,32 +1,14 @@
|
|||
# **S**coutinScience **U**tilitie**S** for text processing [](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml)
|
||||
|
||||
> amogus
|
||||
|
||||
## Exports
|
||||
|
||||
- [clean](src/sus/clean.py)
|
||||
- [unique](src/sus/unique.py)
|
||||
- [parallel_map](src/sus/parallel_map.py)
|
||||
- [match_names](src/sus/match_names/match_names.py)
|
||||
- [lemmatize](src/sus/lemmatize.py)
|
||||
- [evaluate_ranking](src/sus/evaluate_ranking/evaluate_ranking.py)
|
||||
- [get_sentences](src/sus/get_sentences.py)
|
||||
|
||||
### Requires loading spacy model
|
||||
|
||||
> This is automatic but will require some time.
|
||||
|
||||
> Add this to the Dockerfile for caching the spaCy model:
|
||||
>
|
||||
> ```docker
|
||||
> RUN pip install --no-cache-dir en-core-web-sm@https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl
|
||||
> ```
|
||||
|
||||
- [publication TEI](src/sus/publication_tei/publication_tei.py)
|
||||
- [lemmatize_text](src/sus/lemmatize_text.py)
|
||||
- [lemmatize_token](src/sus/lemmatize_token.py)
|
||||
- [spacy model (nlp)](src/sus/nlp.py)
|
||||
- [filter_sentences](src/sus/matcher/filter_sentences.py)
|
||||
|
||||
## Development
|
||||
|
||||
- Optional booleans must have a default value of `False`.
|
||||
|
|
|
|||
|
|
@ -23,9 +23,6 @@ install_requires =
|
|||
unidecode >= 1.3.0
|
||||
multiprocess >= 0.70.0.0
|
||||
tqdm >= 4.0.0
|
||||
beautifulsoup4 >= 4.10.0
|
||||
lxml >= 4.6.0
|
||||
spacy >= 3.3.0
|
||||
scikit-learn == 1.1.1
|
||||
matplotlib >= 3.5.0
|
||||
numpy >= 1.22.0
|
||||
|
|
@ -34,7 +31,6 @@ install_requires =
|
|||
langdetect >= 1.0.9
|
||||
tinydb >= 4.7.0
|
||||
pandas >= 1.4.0
|
||||
pyaml >= 21.0.0
|
||||
boto3 >= 1.23.0
|
||||
fastapi >= 0.70.0
|
||||
plotly >= 5.8.0
|
||||
|
|
|
|||
|
|
@ -1,14 +1,10 @@
|
|||
from .clean import clean
|
||||
from .parallel_map import parallel_map
|
||||
from .unique import unique
|
||||
from .config_file import ConfigFile, ParseError
|
||||
from .evaluate_ranking import evaluate_ranking
|
||||
from .get_sentences import get_sentences
|
||||
from .language import english_name_of_language, is_english, predict_language
|
||||
from .lemmatize_text import lemmatize_text
|
||||
from .lemmatize_token import lemmatize_token
|
||||
from .lemmatize import lemmatize
|
||||
from .logger import get_logger
|
||||
from .evaluate_ranking import evaluate_ranking
|
||||
from .match_names import match_names
|
||||
from .matcher import fast_tokenize, filter_sentences, normalize
|
||||
from .nlp import nlp
|
||||
from .parallel_map import parallel_map
|
||||
from .publication_tei import PublicationTEI
|
||||
from .unique import unique
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
https://github.com/jenojp/negspacy
|
||||
222
src/great_ai/utilities/external/negspacy/negation.py
vendored
222
src/great_ai/utilities/external/negspacy/negation.py
vendored
|
|
@ -1,222 +0,0 @@
|
|||
import logging
|
||||
|
||||
from spacy.language import Language
|
||||
from spacy.matcher import PhraseMatcher
|
||||
from spacy.tokens import Token
|
||||
|
||||
from .termsets import termset
|
||||
|
||||
default_ts = termset("en").get_patterns()
|
||||
|
||||
|
||||
@Language.factory(
|
||||
"negex",
|
||||
default_config={
|
||||
"neg_termset": default_ts,
|
||||
"extension_name": "negex",
|
||||
"chunk_prefix": list(),
|
||||
},
|
||||
)
|
||||
class Negex:
|
||||
"""
|
||||
A spaCy pipeline component which identifies negated tokens in text.
|
||||
|
||||
Based on: NegEx - A Simple Algorithm for Identifying Negated Findings and Diseasesin Discharge Summaries
|
||||
Chapman, Bridewell, Hanbury, Cooper, Buchanan
|
||||
|
||||
Parameters
|
||||
----------
|
||||
nlp: object
|
||||
spaCy language object
|
||||
termset_lang: str
|
||||
language code, if using default termsets (e.g. "en" for english)
|
||||
extension_name: str
|
||||
defaults to "negex"; whether entity is negated is then available as ent._.negex
|
||||
pseudo_negations: list
|
||||
list of phrases that cancel out a negation, if empty, defaults are used
|
||||
preceding_negations: list
|
||||
negations that appear before an entity, if empty, defaults are used
|
||||
following_negations: list
|
||||
negations that appear after an entity, if empty, defaults are used
|
||||
termination: list
|
||||
phrases that "terminate" a sentence for processing purposes such as "but". If empty, defaults are used
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
nlp: Language,
|
||||
name: str,
|
||||
neg_termset: dict,
|
||||
extension_name: str,
|
||||
chunk_prefix: list,
|
||||
):
|
||||
if not Token.has_extension(extension_name):
|
||||
Token.set_extension(extension_name, default=False, force=True)
|
||||
|
||||
ts = neg_termset
|
||||
expected_keys = [
|
||||
"pseudo_negations",
|
||||
"preceding_negations",
|
||||
"following_negations",
|
||||
"termination",
|
||||
]
|
||||
if not set(ts.keys()) == set(expected_keys):
|
||||
raise KeyError(
|
||||
f"Unexpected or missing keys in 'neg_termset', expected: {expected_keys}, instead got: {list(ts.keys())}"
|
||||
)
|
||||
|
||||
self.pseudo_negations = ts["pseudo_negations"]
|
||||
self.preceding_negations = ts["preceding_negations"]
|
||||
self.following_negations = ts["following_negations"]
|
||||
self.termination = ts["termination"]
|
||||
|
||||
self.nlp = nlp
|
||||
self.extension_name = extension_name
|
||||
self.build_patterns()
|
||||
self.chunk_prefix = list(nlp.tokenizer.pipe(chunk_prefix))
|
||||
|
||||
def build_patterns(self):
|
||||
# efficiently build spaCy matcher patterns
|
||||
self.matcher = PhraseMatcher(self.nlp.vocab, attr="LOWER")
|
||||
|
||||
self.pseudo_patterns = list(self.nlp.tokenizer.pipe(self.pseudo_negations))
|
||||
self.matcher.add("pseudo", None, *self.pseudo_patterns)
|
||||
|
||||
self.preceding_patterns = list(
|
||||
self.nlp.tokenizer.pipe(self.preceding_negations)
|
||||
)
|
||||
self.matcher.add("Preceding", None, *self.preceding_patterns)
|
||||
|
||||
self.following_patterns = list(
|
||||
self.nlp.tokenizer.pipe(self.following_negations)
|
||||
)
|
||||
self.matcher.add("Following", None, *self.following_patterns)
|
||||
|
||||
self.termination_patterns = list(self.nlp.tokenizer.pipe(self.termination))
|
||||
self.matcher.add("Termination", None, *self.termination_patterns)
|
||||
|
||||
def process_negations(self, doc):
|
||||
"""
|
||||
Find negations in doc and clean candidate negations to remove pseudo negations
|
||||
|
||||
Parameters
|
||||
----------
|
||||
doc: object
|
||||
spaCy Doc object
|
||||
|
||||
Returns
|
||||
-------
|
||||
preceding: list
|
||||
list of tuples for preceding negations
|
||||
following: list
|
||||
list of tuples for following negations
|
||||
terminating: list
|
||||
list of tuples of terminating phrases
|
||||
|
||||
"""
|
||||
###
|
||||
# does not work properly in spacy 2.1.8. Will incorporate after 2.2.
|
||||
# Relying on user to use NER in meantime
|
||||
# see https://github.com/jenojp/negspacy/issues/7
|
||||
###
|
||||
# if not doc.is_nered:
|
||||
# raise ValueError(
|
||||
# "Negations are evaluated for Named Entities found in text. "
|
||||
# "Your SpaCy pipeline does not included Named Entity resolution. "
|
||||
# "Please ensure it is enabled or choose a different language model that includes it."
|
||||
# )
|
||||
preceding = list()
|
||||
following = list()
|
||||
terminating = list()
|
||||
|
||||
matches = self.matcher(doc)
|
||||
pseudo = [
|
||||
(match_id, start, end)
|
||||
for match_id, start, end in matches
|
||||
if self.nlp.vocab.strings[match_id] == "pseudo"
|
||||
]
|
||||
|
||||
for match_id, start, end in matches:
|
||||
if self.nlp.vocab.strings[match_id] == "pseudo":
|
||||
continue
|
||||
pseudo_flag = False
|
||||
for p in pseudo:
|
||||
if start >= p[1] and start <= p[2]:
|
||||
pseudo_flag = True
|
||||
continue
|
||||
if not pseudo_flag:
|
||||
if self.nlp.vocab.strings[match_id] == "Preceding":
|
||||
preceding.append((match_id, start, end))
|
||||
elif self.nlp.vocab.strings[match_id] == "Following":
|
||||
following.append((match_id, start, end))
|
||||
elif self.nlp.vocab.strings[match_id] == "Termination":
|
||||
terminating.append((match_id, start, end))
|
||||
else:
|
||||
logging.warnings(
|
||||
f"phrase {doc[start:end].text} not in one of the expected matcher types."
|
||||
)
|
||||
return preceding, following, terminating
|
||||
|
||||
def termination_boundaries(self, doc, terminating):
|
||||
"""
|
||||
Create sub sentences based on terminations found in text.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
doc: object
|
||||
spaCy Doc object
|
||||
terminating: list
|
||||
list of tuples with (match_id, start, end)
|
||||
|
||||
returns
|
||||
-------
|
||||
boundaries: list
|
||||
list of tuples with (start, end) of spans
|
||||
|
||||
"""
|
||||
sent_starts = [sent.start for sent in doc.sents]
|
||||
terminating_starts = [t[1] for t in terminating]
|
||||
starts = sent_starts + terminating_starts + [len(doc)]
|
||||
starts.sort()
|
||||
boundaries = list()
|
||||
index = 0
|
||||
for i, start in enumerate(starts):
|
||||
if not i == 0:
|
||||
boundaries.append((index, start))
|
||||
index = start
|
||||
return boundaries
|
||||
|
||||
def negex(self, doc):
|
||||
"""
|
||||
Negates entities of interest
|
||||
|
||||
Parameters
|
||||
----------
|
||||
doc: object
|
||||
spaCy Doc object
|
||||
|
||||
"""
|
||||
preceding, following, terminating = self.process_negations(doc)
|
||||
boundaries = self.termination_boundaries(doc, terminating)
|
||||
for b in boundaries:
|
||||
sub_preceding = [i for i in preceding if b[0] <= i[1] < b[1]]
|
||||
sub_following = [i for i in following if b[0] <= i[1] < b[1]]
|
||||
|
||||
for e in doc[b[0] : b[1]]:
|
||||
if any(pre < e.i for pre in [i[1] for i in sub_preceding]):
|
||||
e._.set(self.extension_name, True)
|
||||
continue
|
||||
if any(fol > e.i for fol in [i[2] for i in sub_following]):
|
||||
e._.set(self.extension_name, True)
|
||||
continue
|
||||
if self.chunk_prefix:
|
||||
if any(
|
||||
e.text.lower().startswith(c.text.lower())
|
||||
for c in self.chunk_prefix
|
||||
):
|
||||
e._.set(self.extension_name, True)
|
||||
return doc
|
||||
|
||||
def __call__(self, doc):
|
||||
return self.negex(doc)
|
||||
229
src/great_ai/utilities/external/negspacy/termsets.py
vendored
229
src/great_ai/utilities/external/negspacy/termsets.py
vendored
|
|
@ -1,229 +0,0 @@
|
|||
"""
|
||||
Default termsets for various languages
|
||||
"""
|
||||
|
||||
LANGUAGES = dict()
|
||||
|
||||
# english termset dictionary
|
||||
en = dict()
|
||||
pseudo = [
|
||||
"no further",
|
||||
"not able to be",
|
||||
"not certain if",
|
||||
"not certain whether",
|
||||
"not necessarily",
|
||||
"without any further",
|
||||
"without difficulty",
|
||||
"without further",
|
||||
"might not",
|
||||
"not only",
|
||||
"no increase",
|
||||
"no significant change",
|
||||
"no change",
|
||||
"no definite change",
|
||||
"not extend",
|
||||
"not cause",
|
||||
]
|
||||
en["pseudo_negations"] = pseudo
|
||||
|
||||
preceding = [
|
||||
"absence of",
|
||||
"declined",
|
||||
"denied",
|
||||
"denies",
|
||||
"denying",
|
||||
"no sign of",
|
||||
"no signs of",
|
||||
"not",
|
||||
"not demonstrate",
|
||||
"symptoms atypical",
|
||||
"doubt",
|
||||
"negative for",
|
||||
"no",
|
||||
"versus",
|
||||
"without",
|
||||
"doesn't",
|
||||
"doesnt",
|
||||
"don't",
|
||||
"dont",
|
||||
"didn't",
|
||||
"didnt",
|
||||
"wasn't",
|
||||
"wasnt",
|
||||
"weren't",
|
||||
"werent",
|
||||
"isn't",
|
||||
"isnt",
|
||||
"aren't",
|
||||
"arent",
|
||||
"cannot",
|
||||
"can't",
|
||||
"cant",
|
||||
"couldn't",
|
||||
"couldnt",
|
||||
"never",
|
||||
]
|
||||
en["preceding_negations"] = preceding
|
||||
|
||||
following = [
|
||||
"declined",
|
||||
"unlikely",
|
||||
"was not",
|
||||
"were not",
|
||||
"wasn't",
|
||||
"wasnt",
|
||||
"weren't",
|
||||
"werent",
|
||||
]
|
||||
en["following_negations"] = following
|
||||
|
||||
termination = [
|
||||
"although",
|
||||
"apart from",
|
||||
"as there are",
|
||||
"aside from",
|
||||
"but",
|
||||
"except",
|
||||
"however",
|
||||
"involving",
|
||||
"nevertheless",
|
||||
"still",
|
||||
"though",
|
||||
"which",
|
||||
"yet",
|
||||
]
|
||||
en["termination"] = termination
|
||||
|
||||
LANGUAGES["en"] = en
|
||||
|
||||
# en_clinical builds upon en
|
||||
en_clinical = dict()
|
||||
pseudo_clinical = pseudo + [
|
||||
"gram negative",
|
||||
"not rule out",
|
||||
"not ruled out",
|
||||
"not been ruled out",
|
||||
"not drain",
|
||||
"no suspicious change",
|
||||
"no interval change",
|
||||
"no significant interval change",
|
||||
]
|
||||
en_clinical["pseudo_negations"] = pseudo_clinical
|
||||
|
||||
preceding_clinical = preceding + [
|
||||
"patient was not",
|
||||
"without indication of",
|
||||
"without sign of",
|
||||
"without signs of",
|
||||
"without any reactions or signs of",
|
||||
"no complaints of",
|
||||
"no evidence of",
|
||||
"no cause of",
|
||||
"evaluate for",
|
||||
"fails to reveal",
|
||||
"free of",
|
||||
"never developed",
|
||||
"never had",
|
||||
"did not exhibit",
|
||||
"rules out",
|
||||
"rule out",
|
||||
"rule him out",
|
||||
"rule her out",
|
||||
"rule patient out",
|
||||
"rule the patient out",
|
||||
"ruled out",
|
||||
"ruled him out",
|
||||
"ruled her out",
|
||||
"ruled patient out",
|
||||
"ruled the patient out",
|
||||
"r/o",
|
||||
"ro",
|
||||
]
|
||||
en_clinical["preceding_negations"] = preceding_clinical
|
||||
|
||||
following_clinical = following + ["was ruled out", "were ruled out", "free"]
|
||||
en_clinical["following_negations"] = following_clinical
|
||||
|
||||
termination_clinical = termination + [
|
||||
"cause for",
|
||||
"cause of",
|
||||
"causes for",
|
||||
"causes of",
|
||||
"etiology for",
|
||||
"etiology of",
|
||||
"origin for",
|
||||
"origin of",
|
||||
"origins for",
|
||||
"origins of",
|
||||
"other possibilities of",
|
||||
"reason for",
|
||||
"reason of",
|
||||
"reasons for",
|
||||
"reasons of",
|
||||
"secondary to",
|
||||
"source for",
|
||||
"source of",
|
||||
"sources for",
|
||||
"sources of",
|
||||
"trigger event for",
|
||||
]
|
||||
en_clinical["termination"] = termination_clinical
|
||||
LANGUAGES["en_clinical"] = en_clinical
|
||||
|
||||
en_clinical_sensitive = dict()
|
||||
|
||||
preceding_clinical_sensitive = preceding_clinical + [
|
||||
"concern for",
|
||||
"supposed",
|
||||
"which causes",
|
||||
"leads to",
|
||||
"h/o",
|
||||
"history of",
|
||||
"instead of",
|
||||
"if you experience",
|
||||
"if you get",
|
||||
"teaching the patient",
|
||||
"taught the patient",
|
||||
"teach the patient",
|
||||
"educated the patient",
|
||||
"educate the patient",
|
||||
"educating the patient",
|
||||
"monitored for",
|
||||
"monitor for",
|
||||
"test for",
|
||||
"tested for",
|
||||
]
|
||||
en_clinical_sensitive["pseudo_negations"] = pseudo_clinical
|
||||
en_clinical_sensitive["preceding_negations"] = preceding_clinical_sensitive
|
||||
en_clinical_sensitive["following_negations"] = following_clinical
|
||||
en_clinical_sensitive["termination"] = termination_clinical
|
||||
|
||||
LANGUAGES["en_clinical_sensitive"] = en_clinical_sensitive
|
||||
|
||||
|
||||
class termset:
|
||||
def __init__(self, termset_lang):
|
||||
self.pattern_types = [
|
||||
"pseudo_negations",
|
||||
"preceding_negations",
|
||||
"following_negations",
|
||||
"termination",
|
||||
]
|
||||
self.terms = LANGUAGES[termset_lang]
|
||||
|
||||
def get_patterns(self):
|
||||
return self.terms
|
||||
|
||||
def remove_patterns(self, pattern_dict):
|
||||
for key, value in pattern_dict.items():
|
||||
if key in self.pattern_types:
|
||||
self.terms[key] = [i for i in self.terms[key] if i not in value]
|
||||
else:
|
||||
raise ValueError(f"Unexpected key: {key} not in {self.pattern_types}")
|
||||
|
||||
def add_patterns(self, pattern_dict):
|
||||
for key, value in pattern_dict.items():
|
||||
if key in self.pattern_types:
|
||||
self.terms[key] = list(set(self.terms[key] + value))
|
||||
else:
|
||||
raise ValueError(f"Unexpected key: {key} not in {self.pattern_types}")
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
from typing import List
|
||||
|
||||
from .lemmatize_token import lemmatize_token
|
||||
from .nlp import nlp
|
||||
|
||||
|
||||
def lemmatize_text(
|
||||
text: str,
|
||||
add_negation: bool = False,
|
||||
add_part_of_speech: bool = False,
|
||||
) -> List[str]:
|
||||
doc = nlp(text)
|
||||
|
||||
return [
|
||||
lemmatize_token(
|
||||
t,
|
||||
add_negation=add_negation,
|
||||
add_part_of_speech=add_part_of_speech,
|
||||
)
|
||||
for t in doc
|
||||
]
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
from spacy.tokens import Token
|
||||
|
||||
from .data import american_spellings
|
||||
|
||||
|
||||
def lemmatize_token(
|
||||
token: Token,
|
||||
add_negation: bool = False,
|
||||
add_part_of_speech: bool = False,
|
||||
) -> str:
|
||||
lemma = token.lemma_.lower()
|
||||
|
||||
lemma = american_spellings.get(lemma, lemma)
|
||||
|
||||
if add_part_of_speech:
|
||||
lemma = f"{lemma}_{token.pos_}"
|
||||
if add_negation and token._.negex:
|
||||
lemma = f"NOT_{lemma}"
|
||||
|
||||
return lemma
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
from .fast_tokenize import fast_tokenize
|
||||
from .filter_sentences import filter_sentences
|
||||
from .normalize import normalize
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
import re
|
||||
from typing import List, Union
|
||||
|
||||
from segtok.tokenizer import word_tokenizer
|
||||
|
||||
from ..get_sentences import get_sentences
|
||||
from .normalize import normalize
|
||||
|
||||
|
||||
def fast_tokenize(
|
||||
text: Union[List[str], str], ignore_partial: bool = False
|
||||
) -> List[List[str]]:
|
||||
if isinstance(text, str):
|
||||
text = normalize(text)
|
||||
text = get_sentences(text, ignore_partial=ignore_partial)
|
||||
|
||||
results: List[List[str]] = []
|
||||
|
||||
for sentence in text:
|
||||
sentence = re.sub(r"\bare\b", "is", sentence)
|
||||
sentence = re.sub(r"\ban\b", "a", sentence)
|
||||
sentence = re.sub(r"\bthese\b", "this", sentence)
|
||||
results.append(
|
||||
[
|
||||
token.lower() if token not in {"CITATION", "NUMBER"} else token
|
||||
for token in word_tokenizer(sentence)
|
||||
]
|
||||
)
|
||||
|
||||
return results
|
||||
|
|
@ -1,66 +0,0 @@
|
|||
from pathlib import Path
|
||||
from typing import Dict, List, Union
|
||||
|
||||
import yaml
|
||||
from spacy.matcher import Matcher
|
||||
|
||||
from ..get_sentences import get_sentences
|
||||
from ..nlp import nlp
|
||||
from .fast_tokenize import fast_tokenize
|
||||
from .normalize import normalize
|
||||
|
||||
rules_cache: Dict[str, Matcher] = {}
|
||||
|
||||
|
||||
def filter_sentences(
|
||||
sentences: str,
|
||||
rules_file: Path,
|
||||
inverse: bool = False,
|
||||
ignore_partial: bool = False,
|
||||
) -> List[str]:
|
||||
if str(rules_file) not in rules_cache:
|
||||
with open(rules_file, encoding="utf-8") as f:
|
||||
rule_patterns = yaml.safe_load(f).keys()
|
||||
|
||||
matcher = Matcher(nlp.vocab)
|
||||
rules = [_pattern_to_rule(p) for p in rule_patterns]
|
||||
matcher.add("", rules)
|
||||
rules_cache[str(rules_file)] = matcher
|
||||
|
||||
matcher = rules_cache[str(rules_file)]
|
||||
|
||||
original_sentences = get_sentences(sentences, ignore_partial=ignore_partial)
|
||||
|
||||
tokenized = fast_tokenize(original_sentences, ignore_partial=ignore_partial)
|
||||
|
||||
results: List[str] = []
|
||||
for original_sentence, sentence in zip(original_sentences, tokenized):
|
||||
doc = nlp(normalize(" ".join(sentence)))
|
||||
matches = matcher(doc)
|
||||
if matches:
|
||||
# _, start, end = max(
|
||||
# matches,
|
||||
# key=lambda v: v[2] - v[1],
|
||||
# )
|
||||
# print(str(doc[start:end]))
|
||||
|
||||
if not inverse:
|
||||
results.append(original_sentence)
|
||||
elif inverse:
|
||||
results.append(original_sentence)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def _pattern_to_rule(pattern: str) -> List[Dict[str, Union[bool, str]]]:
|
||||
result: List[Dict[str, Union[bool, str]]] = []
|
||||
for t in pattern.split():
|
||||
if t == "*":
|
||||
result.extend([{"OP": "?"}, {"OP": "?"}])
|
||||
elif t == "CITATION":
|
||||
result.append({"ORTH": "CITATION"})
|
||||
elif t == "NUMBER":
|
||||
result.append({"ORTH": "NUMBER"})
|
||||
else:
|
||||
result.append({"LOWER": t})
|
||||
return result
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
import re
|
||||
|
||||
from ..clean import clean
|
||||
|
||||
|
||||
def normalize(text: str) -> str:
|
||||
text = re.sub(
|
||||
r"""
|
||||
([A-Z]\w+\W+(et\ al.)) # inline reference: Bank et al.
|
||||
| (\[[0-9-, ]+\]) # IEEE style: [1], [2-4], [3, 5]
|
||||
| (\(.*?,?\W+?\d+\)) # APA style: (Bank, 2020)
|
||||
| ([A-Z]\w+ \(?\d+\)) # APA style: Bank (2020)
|
||||
""",
|
||||
" CITATION ",
|
||||
text,
|
||||
flags=re.VERBOSE,
|
||||
)
|
||||
text = re.sub(r"\d[\d,. -]*(st|nd|rd|th)?", " NUMBER ", text)
|
||||
text = clean(text, convert_to_ascii=True)
|
||||
text = re.sub(r"[^a-zA-Z.?!,:;'\" -]", "", text)
|
||||
|
||||
return text
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
try:
|
||||
import en_core_web_sm
|
||||
except ImportError:
|
||||
import subprocess
|
||||
|
||||
print("Spacy model en_core_web_sm not found locally, downloading...")
|
||||
|
||||
subprocess.call(
|
||||
[
|
||||
"pip",
|
||||
"install",
|
||||
"https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl",
|
||||
]
|
||||
)
|
||||
import en_core_web_sm
|
||||
|
||||
from .external.negspacy import negation # noqa: F401 it's important to import this
|
||||
|
||||
nlp = en_core_web_sm.load()
|
||||
|
||||
nlp.add_pipe("negex")
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
from .models import *
|
||||
from .publication_tei import PublicationTEI
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
from .affiliation import Affiliation
|
||||
from .author import Author
|
||||
from .bookmark import Bookmark
|
||||
from .bookmark_title import BookmarkTitle
|
||||
from .element import Element, Meta, MetaType, Paragraph, Title
|
||||
from .publication_metadata import PublicationMetadata
|
||||
from .text import Text
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
from typing import Optional, Tuple
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class Affiliation(BaseModel):
|
||||
institutions: Tuple[str, ...]
|
||||
departments: Tuple[str, ...]
|
||||
laboratories: Tuple[str, ...]
|
||||
country: Optional[str]
|
||||
settlement: Optional[str]
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
from typing import List, Optional
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from .affiliation import Affiliation
|
||||
|
||||
|
||||
class Author(BaseModel):
|
||||
name: Optional[str]
|
||||
orcid: Optional[str]
|
||||
email: Optional[str]
|
||||
corresponding: bool
|
||||
affiliations: List[Affiliation]
|
||||
coordinates: Optional[str]
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
from pydantic import BaseModel
|
||||
|
||||
from .bookmark_title import BookmarkTitle
|
||||
|
||||
|
||||
class Bookmark(BaseModel):
|
||||
title: BookmarkTitle
|
||||
original_title: str
|
||||
document_order: int
|
||||
coordinates: str
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
from typing import Literal
|
||||
|
||||
BookmarkTitle = Literal[
|
||||
"Abstract",
|
||||
"Author contribution",
|
||||
"Introduction",
|
||||
"Background",
|
||||
"Methods",
|
||||
"Results",
|
||||
"Discussion",
|
||||
"Outlook",
|
||||
"Conclusion",
|
||||
"Conflict of interest",
|
||||
"Acknowledgement",
|
||||
"Annex",
|
||||
]
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
from typing import List, Literal, Union
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from .text import Text
|
||||
|
||||
|
||||
class Title(BaseModel):
|
||||
text: Text
|
||||
|
||||
|
||||
class Paragraph(BaseModel):
|
||||
sentences: List[Text]
|
||||
|
||||
|
||||
MetaType = Literal[
|
||||
"abstract_start",
|
||||
"abstract_end",
|
||||
"acknowledgements_start",
|
||||
"acknowledgements_end",
|
||||
"annex_start",
|
||||
"annex_end",
|
||||
]
|
||||
|
||||
|
||||
class Meta(BaseModel):
|
||||
meta_type: MetaType
|
||||
|
||||
|
||||
Element = Union[Title, Paragraph, Meta]
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
from typing import List, Optional
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class PublicationMetadata(BaseModel):
|
||||
language: Optional[str]
|
||||
title: Optional[str]
|
||||
publisher: Optional[str]
|
||||
doi: Optional[str]
|
||||
md5: Optional[str]
|
||||
publication_date: Optional[str]
|
||||
keywords: List[str]
|
||||
reference_count: int
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
from typing import Optional
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class Text(BaseModel):
|
||||
content: str
|
||||
document_order: int
|
||||
coordinates: Optional[str]
|
||||
|
|
@ -1,423 +0,0 @@
|
|||
import os
|
||||
import re
|
||||
from functools import cached_property, lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Any, List, Optional, Pattern, Tuple, Union
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
from bs4.element import NavigableString, Tag
|
||||
|
||||
from ..clean import clean
|
||||
from ..lemmatize_text import lemmatize_text
|
||||
from ..matcher import filter_sentences
|
||||
from ..unique import unique
|
||||
from .models import (
|
||||
Affiliation,
|
||||
Author,
|
||||
Bookmark,
|
||||
BookmarkTitle,
|
||||
Element,
|
||||
Meta,
|
||||
MetaType,
|
||||
Paragraph,
|
||||
PublicationMetadata,
|
||||
Text,
|
||||
Title,
|
||||
)
|
||||
from .titles_of_interest import titles_of_interest
|
||||
|
||||
THIS_FOLDER = Path(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
|
||||
class PublicationTEI:
|
||||
# remove template sentences, such as copyright notices
|
||||
aggressive_cleaning_enabled = True
|
||||
|
||||
def __init__(self, tei: str):
|
||||
self._document_order_counter = 0
|
||||
|
||||
if tei:
|
||||
self.soup = BeautifulSoup(tei, "xml")
|
||||
else:
|
||||
self.soup = BeautifulSoup()
|
||||
|
||||
@cached_property
|
||||
def publication_metadata(self) -> PublicationMetadata:
|
||||
publication_date = (
|
||||
self.soup.publicationStmt.date.get("when")
|
||||
if self.soup.publicationStmt and self.soup.publicationStmt.date
|
||||
else None
|
||||
)
|
||||
|
||||
keywords = (
|
||||
[self._element_to_text(k) for k in self.soup.keywords.find_all("term")]
|
||||
if self.soup.keywords
|
||||
else []
|
||||
)
|
||||
|
||||
return PublicationMetadata(
|
||||
language=self.soup.teiHeader.get("xml:lang")
|
||||
if self.soup.teiHeader
|
||||
else None,
|
||||
title=self._element_to_text(self.soup.title),
|
||||
publisher=self._element_to_text(self.soup.publisher),
|
||||
doi=self._element_to_text(self.soup.find("idno", type="DOI")),
|
||||
md5=self._element_to_text(self.soup.find("idno", type="MD5")),
|
||||
publication_date=publication_date,
|
||||
keywords=keywords,
|
||||
reference_count=self.get_reference_count(),
|
||||
)
|
||||
|
||||
def get_reference_count(self) -> int:
|
||||
references = self.soup.find("div", {"type": "references"})
|
||||
|
||||
if not references:
|
||||
return 0
|
||||
|
||||
return len(references.findAll("biblStruct"))
|
||||
|
||||
@cached_property
|
||||
def authors(self) -> List[Author]:
|
||||
if not self.soup.analytic:
|
||||
return []
|
||||
|
||||
return [
|
||||
self._parse_author(author)
|
||||
for author in self.soup.analytic.find_all("author")
|
||||
]
|
||||
|
||||
@cached_property
|
||||
def content(self) -> List[Element]:
|
||||
self._document_order_counter = 0
|
||||
return self._get_elements(self.soup)
|
||||
|
||||
@cached_property
|
||||
def sentences(self) -> List[Text]:
|
||||
return [
|
||||
sentence
|
||||
for element in self.content
|
||||
if isinstance(element, Paragraph)
|
||||
for sentence in element.sentences
|
||||
]
|
||||
|
||||
@cached_property
|
||||
def bookmarks(self) -> List[Bookmark]:
|
||||
candidates: List[Bookmark] = [
|
||||
*self._find_matching_meta("Abstract", "abstract_start")[0],
|
||||
*self._find_matching_title("Abstract"),
|
||||
*self._find_matching_title("Author contribution"),
|
||||
*self._find_matching_meta("Acknowledgement", "acknowledgements_start")[0],
|
||||
*self._find_matching_title("Acknowledgement"),
|
||||
*self._find_matching_title("Conflict of interest"),
|
||||
]
|
||||
|
||||
_, start = self._find_matching_meta(None, "abstract_end")
|
||||
|
||||
candidates += [
|
||||
*self._find_matching_title("Background", start),
|
||||
*self._find_matching_title("Methods", start),
|
||||
*self._find_matching_title("Results", start),
|
||||
*self._find_matching_title("Discussion", start),
|
||||
*self._find_matching_title("Introduction", start),
|
||||
*self._find_matching_title("Conclusion", start),
|
||||
*self._find_matching_title("Outlook", start),
|
||||
*self._find_matching_meta("Annex", "annex_start", start)[0],
|
||||
]
|
||||
|
||||
candidates = sorted(candidates, key=lambda c: c.document_order)
|
||||
candidates = unique(candidates, key=lambda c: c.document_order)
|
||||
candidates = unique(candidates, key=lambda c: c.title)
|
||||
|
||||
return candidates
|
||||
|
||||
@cached_property
|
||||
def abstract_sentences(self) -> List[Text]:
|
||||
try:
|
||||
abstract_start = next(
|
||||
i
|
||||
for i, m in enumerate(self.content)
|
||||
if isinstance(m, Meta) and m.meta_type == "abstract_start"
|
||||
)
|
||||
abstract_end = next(
|
||||
i
|
||||
for i, m in enumerate(self.content)
|
||||
if isinstance(m, Meta) and m.meta_type == "abstract_end"
|
||||
)
|
||||
return [
|
||||
s
|
||||
for p in self.content[abstract_start:abstract_end]
|
||||
if isinstance(p, Paragraph)
|
||||
for s in p.sentences
|
||||
]
|
||||
except StopIteration:
|
||||
pass # let's try another way
|
||||
|
||||
try:
|
||||
abstract_start = next(
|
||||
m.document_order for m in self.bookmarks if m.title == "Abstract"
|
||||
)
|
||||
abstract_sentences: List[Text] = []
|
||||
for c in self.content:
|
||||
if isinstance(c, Paragraph):
|
||||
abstract_sentences.extend(
|
||||
s for s in c.sentences if s.document_order > abstract_start
|
||||
)
|
||||
elif len(abstract_sentences) >= 5:
|
||||
break
|
||||
return abstract_sentences
|
||||
except StopIteration:
|
||||
pass # let's try another way
|
||||
|
||||
return self.sentences[:10]
|
||||
|
||||
@cached_property
|
||||
def introduction_sentences(self) -> List[Text]:
|
||||
"""Includes abstract"""
|
||||
introduction_end = 4
|
||||
|
||||
try:
|
||||
introduction_start = [
|
||||
m.document_order for m in self.bookmarks if m.title == "Introduction"
|
||||
][-1]
|
||||
|
||||
for m in self.bookmarks:
|
||||
if m.title != "Introduction" and m.document_order > introduction_start:
|
||||
introduction_end = m.document_order
|
||||
break
|
||||
except IndexError:
|
||||
pass
|
||||
|
||||
try:
|
||||
introduction_end = max(
|
||||
next(
|
||||
i
|
||||
for i, m in enumerate(self.content)
|
||||
if isinstance(m, Meta) and m.meta_type == "abstract_end"
|
||||
),
|
||||
introduction_end,
|
||||
)
|
||||
except StopIteration:
|
||||
pass
|
||||
|
||||
introduction_sentences: List[Text] = []
|
||||
for c in self.content:
|
||||
if isinstance(c, Paragraph):
|
||||
introduction_sentences.extend(
|
||||
s for s in c.sentences if s.document_order < introduction_end
|
||||
)
|
||||
|
||||
return introduction_sentences
|
||||
|
||||
@cached_property
|
||||
def conclusion_sentences(self) -> List[Text]:
|
||||
try:
|
||||
conclusion_start = next(
|
||||
m.document_order for m in self.bookmarks if m.title == "Conclusion"
|
||||
)
|
||||
conclusion_sentences: List[Text] = []
|
||||
for c in self.content:
|
||||
if isinstance(c, Paragraph):
|
||||
conclusion_sentences.extend(
|
||||
s for s in c.sentences if s.document_order > conclusion_start
|
||||
)
|
||||
elif len(conclusion_sentences) >= 8:
|
||||
break
|
||||
return conclusion_sentences
|
||||
except StopIteration:
|
||||
return self.sentences[-10:]
|
||||
|
||||
def _parse_author(self, raw: Tag) -> Author:
|
||||
return Author(
|
||||
name=(
|
||||
clean(" ".join(name.get_text() for name in raw.persName))
|
||||
if raw.persName
|
||||
else None
|
||||
),
|
||||
orcid=self._element_to_text(raw.find(attrs={"type": "ORCID"})),
|
||||
email=self._element_to_text(raw.email),
|
||||
corresponding=raw.get("role") == "corresp",
|
||||
affiliations=[
|
||||
self._parse_affiliation(aff) for aff in raw.find_all("affiliation")
|
||||
],
|
||||
coordinates=raw.persName.get("coords") if raw.persName else None,
|
||||
)
|
||||
|
||||
def _parse_affiliation(self, raw: Tag) -> Affiliation:
|
||||
return Affiliation(
|
||||
institutions=[
|
||||
self._element_to_text(v)
|
||||
for v in raw.find_all("orgName", attrs={"type": "institution"})
|
||||
],
|
||||
departments=[
|
||||
self._element_to_text(v)
|
||||
for v in raw.find_all("orgName", attrs={"type": "department"})
|
||||
],
|
||||
laboratories=[
|
||||
self._element_to_text(v)
|
||||
for v in raw.find_all("orgName", attrs={"type": "laboratory"})
|
||||
],
|
||||
country=self._element_to_text(raw.address.country)
|
||||
if raw.address and raw.address.country
|
||||
else None,
|
||||
settlement=self._element_to_text(raw.address.settlement)
|
||||
if raw.address and raw.address.settlement
|
||||
else None,
|
||||
)
|
||||
|
||||
def _get_elements(self, raw: Tag) -> List[Element]:
|
||||
results: List[Element] = []
|
||||
|
||||
for r in raw.find_all(["abstract", "div", "head", "p"]):
|
||||
if r.name == "abstract":
|
||||
results.append(Meta(meta_type="abstract_start"))
|
||||
results.extend(self._get_primitives(r))
|
||||
results.append(Meta(meta_type="abstract_end"))
|
||||
elif r.name == "div" and r.get("type") == "acknowledgement":
|
||||
results.append(Meta(meta_type="acknowledgements_start"))
|
||||
results.extend(self._get_primitives(r))
|
||||
results.append(Meta(meta_type="acknowledgements_end"))
|
||||
elif r.name == "div" and r.get("type") == "annex":
|
||||
results.append(Meta(meta_type="annex_start"))
|
||||
results.extend(self._get_primitives(r))
|
||||
results.append(Meta(meta_type="annex_end"))
|
||||
elif not r.find_parents(
|
||||
["abstract", "div", "figure"]
|
||||
): # figures are omitted as well
|
||||
results.extend(self._get_primitives(r))
|
||||
|
||||
return results
|
||||
|
||||
def _get_primitives(self, raw: Tag) -> List[Element]:
|
||||
results: List[Element] = []
|
||||
|
||||
for r in raw.find_all(["head", "p"]):
|
||||
if r.name == "head" and r.get("coords") and r.get_text():
|
||||
text = self._parse_text(r)
|
||||
if text:
|
||||
results.append(Title(text=text))
|
||||
elif r.name == "p" and r.find_all("s"):
|
||||
results.append(
|
||||
Paragraph(
|
||||
sentences=[
|
||||
t
|
||||
for t in [
|
||||
self._parse_text(sentence, ignore_partial=True)
|
||||
for sentence in r.find_all("s")
|
||||
if sentence.get_text()
|
||||
]
|
||||
if t
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
def _element_to_text(
|
||||
self, element: Optional[NavigableString], default: Any = None
|
||||
) -> Union[str, Any]:
|
||||
return (
|
||||
clean(element.get_text(separator=" ", strip=True)) if element else default
|
||||
)
|
||||
|
||||
def _parse_text(self, raw: Tag, ignore_partial: bool = False) -> Optional[Text]:
|
||||
text = raw.get_text()
|
||||
if text is None:
|
||||
return None
|
||||
|
||||
if self.aggressive_cleaning_enabled:
|
||||
filtered = filter_sentences(
|
||||
text,
|
||||
THIS_FOLDER / "templates.yaml",
|
||||
inverse=True,
|
||||
ignore_partial=ignore_partial,
|
||||
)
|
||||
text = " ".join(filtered)
|
||||
|
||||
if not text.strip():
|
||||
return None
|
||||
|
||||
return Text(
|
||||
content=text,
|
||||
document_order=self._generate_document_order_id(),
|
||||
coordinates=raw.get("coords"),
|
||||
)
|
||||
|
||||
def _generate_document_order_id(self) -> int:
|
||||
value = self._document_order_counter
|
||||
self._document_order_counter += 1
|
||||
return value
|
||||
|
||||
def _find_matching_meta(
|
||||
self,
|
||||
bookmark_title: Optional[BookmarkTitle],
|
||||
meta_type: MetaType,
|
||||
start: int = 0,
|
||||
) -> Tuple[List[Bookmark], int]:
|
||||
for i, e in enumerate(self.content[start:], start=start):
|
||||
if not isinstance(e, Meta):
|
||||
continue
|
||||
|
||||
if e.meta_type == meta_type:
|
||||
for e_next in self.content[i + 1 :]:
|
||||
if isinstance(e_next, Title):
|
||||
return [
|
||||
Bookmark(
|
||||
title=bookmark_title,
|
||||
original_title=e_next.text.content,
|
||||
document_order=e_next.text.document_order,
|
||||
coordinates=e_next.text.coordinates,
|
||||
)
|
||||
] if bookmark_title else [], i
|
||||
if isinstance(e_next, Paragraph) and e_next.sentences:
|
||||
return [
|
||||
Bookmark(
|
||||
title=bookmark_title,
|
||||
original_title="",
|
||||
document_order=e_next.sentences[0].document_order,
|
||||
coordinates=e_next.sentences[0].coordinates,
|
||||
)
|
||||
] if bookmark_title else [], i
|
||||
|
||||
return [], 0
|
||||
|
||||
def _find_matching_title(
|
||||
self,
|
||||
bookmark_title: BookmarkTitle,
|
||||
start: int = 0,
|
||||
) -> List[Bookmark]:
|
||||
return [
|
||||
Bookmark(
|
||||
title=bookmark_title,
|
||||
original_title=e.text.content,
|
||||
document_order=e.text.document_order,
|
||||
coordinates=e.text.coordinates,
|
||||
)
|
||||
for e in self.content[start:]
|
||||
if isinstance(e, Title)
|
||||
and self._match_title(e.text.content, titles_of_interest[bookmark_title])
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _match_title(title: str, keywords: Tuple[Union[Pattern, str], ...]) -> bool:
|
||||
title = PublicationTEI._process_section_title(title)
|
||||
|
||||
if any(
|
||||
PublicationTEI._process_section_title(k) in title
|
||||
for k in keywords
|
||||
if isinstance(k, str)
|
||||
):
|
||||
return True
|
||||
|
||||
return any(k.match(title) for k in keywords if isinstance(k, Pattern))
|
||||
|
||||
@staticmethod
|
||||
@lru_cache(maxsize=2000)
|
||||
def _process_section_title(title: str) -> str:
|
||||
title = re.sub(r"^\s*[ivx]+[.,)]? ", "", title) # Remove leading Roman-numerals
|
||||
title = clean(title, convert_to_ascii=True)
|
||||
title = re.sub(
|
||||
r"[^a-zA-Z ]", "", title
|
||||
) # Remove everything but letters and spaces (hypens are also removed)
|
||||
title_tokens = lemmatize_text(title)
|
||||
title = " ".join(t for t in title_tokens if t.strip())
|
||||
return title
|
||||
|
|
@ -1,837 +0,0 @@
|
|||
included in the article's creative commons: 3827
|
||||
is a open access article distributed: 3822
|
||||
a open access article distributed under: 3761
|
||||
open access article distributed under the: 3761
|
||||
access article distributed under the terms: 3761
|
||||
the terms and conditions of the: 3688
|
||||
this article is a open access: 3678
|
||||
article is a open access article: 3678
|
||||
article distributed under the terms and: 3678
|
||||
distributed under the terms and conditions: 3678
|
||||
under the terms and conditions of: 3678
|
||||
terms and conditions of the creative: 3678
|
||||
and conditions of the creative commons: 3678
|
||||
conditions of the creative commons attribution: 3678
|
||||
in the article's creative commons licence: 3503
|
||||
authors declare that they have no: 3315
|
||||
https :// doi.org / NUMBER /: 3189
|
||||
the authors declare that they have: 3015
|
||||
remains neutral with regard to jurisdictional: 2464
|
||||
neutral with regard to jurisdictional claims: 2464
|
||||
with regard to jurisdictional claims in: 2464
|
||||
regard to jurisdictional claims in published: 2462
|
||||
to jurisdictional claims in published maps: 2462
|
||||
the original author ( s ): 2461
|
||||
jurisdictional claims in published maps and: 2461
|
||||
claims in published maps and institutional: 2461
|
||||
author ( s ) and the: 2460
|
||||
original author ( s ) and: 2456
|
||||
in published maps and institutional affiliations: 2415
|
||||
nature remains neutral with regard to: 2399
|
||||
springer nature remains neutral with regard: 2392
|
||||
( s ) and the source: 2353
|
||||
of the creative commons attribution CITATION: 2302
|
||||
", provide a link to the": 2299
|
||||
the source , provide a link: 2293
|
||||
source , provide a link to: 2293
|
||||
you give appropriate credit to the: 2292
|
||||
give appropriate credit to the original: 2292
|
||||
appropriate credit to the original author: 2292
|
||||
and indicate if changes were made: 2289
|
||||
credit to the original author (: 2281
|
||||
to the original author ( s: 2281
|
||||
provide a link to the creative: 2280
|
||||
a link to the creative commons: 2280
|
||||
s ) and the source ,: 2279
|
||||
) and the source , provide: 2277
|
||||
and the source , provide a: 2277
|
||||
", and indicate if changes were": 2271
|
||||
at https :// doi.org / NUMBER: 2195
|
||||
creative commons attribution NUMBER international license: 2158
|
||||
", distribution and reproduction in any": 2158
|
||||
distribution and reproduction in any medium: 2157
|
||||
can be found online at https: 2106
|
||||
be found online at https ://: 2106
|
||||
article can be found online at: 2069
|
||||
license , which permits use ,: 2063
|
||||
this article can be found online: 2010
|
||||
online version contains supplementary material available: 1980
|
||||
supplementary material available at https ://: 1979
|
||||
version contains supplementary material available at: 1978
|
||||
contains supplementary material available at https: 1973
|
||||
the online version contains supplementary material: 1957
|
||||
note springer nature remains neutral with: 1956
|
||||
other third party material in this: 1945
|
||||
to view a copy of this: 1945
|
||||
or other third party material in: 1944
|
||||
need to obtain permission directly from: 1944
|
||||
to obtain permission directly from the: 1944
|
||||
images or other third party material: 1943
|
||||
regulation or exceeds the permitted use: 1943
|
||||
or exceeds the permitted use ,: 1943
|
||||
obtain permission directly from the copyright: 1943
|
||||
the permitted use , you will: 1942
|
||||
by statutory regulation or exceeds the: 1941
|
||||
statutory regulation or exceeds the permitted: 1941
|
||||
exceeds the permitted use , you: 1941
|
||||
use , you will need to: 1941
|
||||
", you will need to obtain": 1941
|
||||
you will need to obtain permission: 1941
|
||||
permitted use , you will need: 1940
|
||||
will need to obtain permission directly: 1940
|
||||
permitted by statutory regulation or exceeds: 1939
|
||||
publisher's note springer nature remains neutral: 1939
|
||||
and your intended use is not: 1938
|
||||
your intended use is not permitted: 1938
|
||||
not permitted by statutory regulation or: 1938
|
||||
intended use is not permitted by: 1937
|
||||
is not permitted by statutory regulation: 1937
|
||||
permission directly from the copyright holder: 1937
|
||||
use is not permitted by statutory: 1936
|
||||
material is not included in the: 1933
|
||||
the images or other third party: 1932
|
||||
if material is not included in: 1926
|
||||
in a credit line to the: 1924
|
||||
otherwise in a credit line to: 1923
|
||||
this article is included in the: 1922
|
||||
third party material in this article: 1921
|
||||
party material in this article is: 1921
|
||||
material in this article is included: 1921
|
||||
in this article is included in: 1921
|
||||
", unless indicated otherwise in a": 1920
|
||||
indicated otherwise in a credit line: 1920
|
||||
is included in the article's creative: 1919
|
||||
unless indicated otherwise in a credit: 1919
|
||||
article is included in the article's: 1918
|
||||
a credit line to the material: 1914
|
||||
not included in the article's creative: 1907
|
||||
is not included in the article's: 1906
|
||||
is licensed under a creative commons: 1898
|
||||
authors declare no conflict of interest: 1883
|
||||
and reproduction in any medium or: 1870
|
||||
reproduction in any medium or format: 1870
|
||||
in any medium or format ,: 1870
|
||||
any medium or format , as: 1870
|
||||
medium or format , as long: 1870
|
||||
or format , as long as: 1870
|
||||
format , as long as you: 1869
|
||||
", as long as you give": 1869
|
||||
as long as you give appropriate: 1869
|
||||
long as you give appropriate credit: 1869
|
||||
as you give appropriate credit to: 1869
|
||||
", adaptation , distribution and reproduction": 1866
|
||||
adaptation , distribution and reproduction in: 1866
|
||||
use , sharing , adaptation ,: 1861
|
||||
", sharing , adaptation , distribution": 1861
|
||||
sharing , adaptation , distribution and: 1861
|
||||
NUMBER international license , which permits: 1846
|
||||
article is licensed under a creative: 1839
|
||||
this article is licensed under a: 1838
|
||||
under a creative commons attribution NUMBER: 1817
|
||||
licensed under a creative commons attribution: 1811
|
||||
a creative commons attribution NUMBER international: 1807
|
||||
view a copy of this licence: 1799
|
||||
a copy of this licence ,: 1799
|
||||
copy of this licence , visit: 1799
|
||||
of this licence , visit http: 1796
|
||||
this licence , visit http ://: 1796
|
||||
access this article is licensed under: 1794
|
||||
the authors would like to thank: 1793
|
||||
which permits use , sharing ,: 1791
|
||||
permits use , sharing , adaptation: 1790
|
||||
", which permits use , sharing": 1789
|
||||
open access this article is licensed: 1782
|
||||
international license , which permits use: 1776
|
||||
attribution NUMBER international license , which: 1773
|
||||
commons attribution NUMBER international license ,: 1772
|
||||
creative commons licence and your intended: 1769
|
||||
commons licence and your intended use: 1768
|
||||
licence and your intended use is: 1768
|
||||
the article's creative commons licence and: 1762
|
||||
article's creative commons licence and your: 1762
|
||||
creative commons licence , unless indicated: 1750
|
||||
commons licence , unless indicated otherwise: 1749
|
||||
licence , unless indicated otherwise in: 1749
|
||||
article's creative commons licence , unless: 1743
|
||||
the article's creative commons licence ,: 1742
|
||||
the authors declare no conflict of: 1742
|
||||
to this article can be found: 1714
|
||||
licence , and indicate if changes: 1698
|
||||
link to the creative commons licence: 1696
|
||||
commons licence , and indicate if: 1695
|
||||
to the creative commons licence ,: 1694
|
||||
the creative commons licence , and: 1694
|
||||
creative commons licence , and indicate: 1694
|
||||
the authors declare no competing interests: 1564
|
||||
and / or publication of this: 1530
|
||||
/ or publication of this article: 1523
|
||||
the research , authorship , and: 1410
|
||||
data to this article can be: 1395
|
||||
supplementary data to this article can: 1392
|
||||
competing financial interests or personal relationships: 1388
|
||||
research , authorship , and /: 1385
|
||||
", authorship , and / or": 1385
|
||||
authorship , and / or publication: 1385
|
||||
is available from the corresponding author: 1385
|
||||
", and / or publication of": 1384
|
||||
no known competing financial interests or: 1382
|
||||
known competing financial interests or personal: 1382
|
||||
declare that they have no known: 1377
|
||||
that they have no known competing: 1376
|
||||
of the creative commons attribution (: 1372
|
||||
have no known competing financial interests: 1371
|
||||
financial interests or personal relationships that: 1371
|
||||
the creative commons attribution ( cc: 1370
|
||||
creative commons attribution ( cc by: 1370
|
||||
they have no known competing financial: 1370
|
||||
to influence the work reported in: 1369
|
||||
influence the work reported in this: 1369
|
||||
interests or personal relationships that could: 1368
|
||||
appeared to influence the work reported: 1368
|
||||
or personal relationships that could have: 1367
|
||||
have appeared to influence the work: 1367
|
||||
could have appeared to influence the: 1364
|
||||
relationships that could have appeared to: 1363
|
||||
that could have appeared to influence: 1363
|
||||
personal relationships that could have appeared: 1362
|
||||
is distributed under the terms of: 1265
|
||||
relationships that could be construed as: 1259
|
||||
/ licenses / by / NUMBER: 1243
|
||||
in the absence of any commercial: 1242
|
||||
the absence of any commercial or: 1241
|
||||
commercial or financial relationships that could: 1241
|
||||
or financial relationships that could be: 1240
|
||||
absence of any commercial or financial: 1239
|
||||
any commercial or financial relationships that: 1239
|
||||
as a potential conflict of interest: 1239
|
||||
was conducted in the absence of: 1238
|
||||
of any commercial or financial relationships: 1238
|
||||
conducted in the absence of any: 1237
|
||||
financial relationships that could be construed: 1236
|
||||
be construed as a potential conflict: 1236
|
||||
the research was conducted in the: 1235
|
||||
construed as a potential conflict of: 1235
|
||||
that the research was conducted in: 1233
|
||||
research was conducted in the absence: 1233
|
||||
:// doi.org / NUMBER / s: 1220
|
||||
doi.org / NUMBER / s NUMBER: 1220
|
||||
declare that the research was conducted: 1212
|
||||
authors declare that the research was: 1204
|
||||
read and approved the final manuscript: 1196
|
||||
study is available from the corresponding: 1163
|
||||
creativecommons.org / licenses / by /: 1148
|
||||
:// creativecommons.org / licenses / by: 1147
|
||||
material available at https :// doi: 1135
|
||||
authors read and approved the final: 1074
|
||||
http :// creativecommons.org / licenses /: 1063
|
||||
that they have no conflict of: 1061
|
||||
all authors read and approved the: 1055
|
||||
requests for materials should be addressed: 1054
|
||||
for materials should be addressed to: 1054
|
||||
correspondence and requests for materials should: 1053
|
||||
and requests for materials should be: 1052
|
||||
findings of this study is available: 1046
|
||||
they have no conflict of interest: 1044
|
||||
declare that they have no conflict: 1020
|
||||
", visit http :// creat iveco": 1016
|
||||
licence , visit http :// creat: 1015
|
||||
the authors declare that the research: 993
|
||||
", on the other hand ,": 989
|
||||
available at https :// doi.org /: 925
|
||||
informed consent was obtained from all: 904
|
||||
this article is protected by copyright: 882
|
||||
found online at https :// doi: 871
|
||||
additional supporting information may be found: 866
|
||||
and moral rights for the publications: 865
|
||||
moral rights for the publications made: 865
|
||||
rights for the publications made accessible: 865
|
||||
for the publications made accessible in: 865
|
||||
is retained by the authors and: 865
|
||||
retained by the authors and /: 865
|
||||
by the authors and / or: 865
|
||||
the authors and / or other: 865
|
||||
authors and / or other copyright: 865
|
||||
and / or other copyright owners: 865
|
||||
/ or other copyright owners and: 865
|
||||
or other copyright owners and it: 865
|
||||
other copyright owners and it is: 865
|
||||
copyright owners and it is a: 865
|
||||
owners and it is a condition: 865
|
||||
and it is a condition of: 865
|
||||
it is a condition of accessing: 865
|
||||
is a condition of accessing publications: 865
|
||||
a condition of accessing publications that: 865
|
||||
condition of accessing publications that users: 865
|
||||
of accessing publications that users recognise: 865
|
||||
accessing publications that users recognise and: 865
|
||||
publications that users recognise and abide: 865
|
||||
that users recognise and abide by: 865
|
||||
users recognise and abide by the: 865
|
||||
recognise and abide by the legal: 865
|
||||
and abide by the legal requirements: 865
|
||||
abide by the legal requirements associated: 865
|
||||
by the legal requirements associated with: 865
|
||||
the legal requirements associated with this: 865
|
||||
copyright and moral rights for the: 864
|
||||
the publications made accessible in the: 864
|
||||
publications made accessible in the public: 864
|
||||
made accessible in the public portal: 864
|
||||
accessible in the public portal is: 864
|
||||
in the public portal is retained: 864
|
||||
the public portal is retained by: 864
|
||||
public portal is retained by the: 864
|
||||
portal is retained by the authors: 864
|
||||
legal requirements associated with this rights: 863
|
||||
the terms of the creative commons: 839
|
||||
and reproduction in any medium ,: 834
|
||||
reproduction in any medium , provided: 833
|
||||
declare that they have no competing: 823
|
||||
conflicts of interest with respect to: 811
|
||||
this work was supported by the: 804
|
||||
that they have no competing interests: 783
|
||||
respect to the research , authorship: 781
|
||||
with respect to the research ,: 780
|
||||
potential conflicts of interest with respect: 779
|
||||
interest with respect to the research: 776
|
||||
of article NUMBER fa of the: 775
|
||||
article NUMBER fa of the dutch: 775
|
||||
NUMBER fa of the dutch copyright: 775
|
||||
fa of the dutch copyright act: 775
|
||||
potential conflict of interest was reported: 775
|
||||
conflict of interest was reported by: 775
|
||||
no potential conflict of interest was: 773
|
||||
of interest was reported by the: 770
|
||||
in the present study , we: 767
|
||||
under the terms of article NUMBER: 766
|
||||
the terms of article NUMBER fa: 766
|
||||
terms of article NUMBER fa of: 766
|
||||
this study is available from the: 765
|
||||
support for the research , authorship: 762
|
||||
financial support for the research ,: 760
|
||||
of the dutch copyright act ,: 758
|
||||
distributed under the terms of the: 743
|
||||
publication is distributed under the terms: 742
|
||||
distributed under the terms of article: 742
|
||||
the original work is properly cited: 738
|
||||
if the publication is distributed under: 734
|
||||
the publication is distributed under the: 734
|
||||
the dutch copyright act , indicated: 734
|
||||
dutch copyright act , indicated by: 734
|
||||
copyright act , indicated by the: 734
|
||||
act , indicated by the taverne: 734
|
||||
", indicated by the taverne license": 734
|
||||
indicated by the taverne license above: 734
|
||||
by the taverne license above ,: 734
|
||||
the taverne license above , please: 734
|
||||
taverne license above , please follow: 734
|
||||
license above , please follow below: 734
|
||||
above , please follow below link: 734
|
||||
", please follow below link for": 734
|
||||
please follow below link for the: 734
|
||||
follow below link for the end: 734
|
||||
online at https :// doi.org /: 733
|
||||
of this study is available from: 731
|
||||
to the research , authorship ,: 728
|
||||
from NUMBER % to NUMBER %: 728
|
||||
CITATION , NUMBER CITATION , NUMBER: 723
|
||||
reprints and permission information is available: 722
|
||||
information is available at http ://: 722
|
||||
available at http :// www.nature.com /: 722
|
||||
is available at http :// www.nature.com: 721
|
||||
at http :// www.nature.com / reprints: 721
|
||||
terms of the creative commons attribution: 717
|
||||
and permission information is available at: 710
|
||||
permission information is available at http: 710
|
||||
provided the original work is properly: 707
|
||||
", provided the original work is": 706
|
||||
http :// www.nature.com / reprints publisher's: 706
|
||||
:// www.nature.com / reprints publisher's note: 705
|
||||
www.nature.com / reprints publisher's note springer: 705
|
||||
/ reprints publisher's note springer nature: 705
|
||||
reprints publisher's note springer nature remains: 705
|
||||
associated with this article can be: 700
|
||||
with this article can be found: 695
|
||||
available from the corresponding author upon: 691
|
||||
no potential conflicts of interest with: 685
|
||||
for the research , authorship ,: 685
|
||||
", visit http :// creativecommons.org /": 683
|
||||
visit http :// creativecommons.org / licenses: 683
|
||||
conflict of interest the authors declare: 678
|
||||
declared no potential conflicts of interest: 666
|
||||
visit http :// creat iveco mmons: 665
|
||||
for this article can be found: 658
|
||||
had no role in the design: 651
|
||||
material for this article can be: 645
|
||||
this is a open access article: 644
|
||||
the corresponding author upon reasonable request: 637
|
||||
supplementary material for this article can: 633
|
||||
from the corresponding author upon reasonable: 633
|
||||
the final version of the manuscript: 624
|
||||
the author ( s ) declared: 618
|
||||
the supplementary material for this article: 617
|
||||
in the public , commercial ,: 613
|
||||
the public , commercial , or: 612
|
||||
by the author ( s ): 599
|
||||
of interest the authors declare that: 594
|
||||
available from the corresponding author on: 589
|
||||
this article can be found ,: 587
|
||||
article can be found , in: 587
|
||||
link to the creative commons license: 585
|
||||
be found in the online version: 584
|
||||
", the aim of this study": 583
|
||||
be found online in the supporting: 583
|
||||
found online in the supporting information: 583
|
||||
", it has been shown that": 582
|
||||
we would like to thank the: 580
|
||||
supporting information may be found online: 579
|
||||
information may be found online in: 577
|
||||
may be found online in the: 577
|
||||
be found , in the online: 577
|
||||
found , in the online version: 577
|
||||
license , and indicate if changes: 576
|
||||
", in the online version ,": 574
|
||||
in the online version , at: 574
|
||||
for their contribution to the peer: 574
|
||||
their contribution to the peer review: 574
|
||||
this research did not receive any: 572
|
||||
reprints and permissions information is available: 572
|
||||
to the creative commons license ,: 571
|
||||
the creative commons license , and: 571
|
||||
creative commons license , and indicate: 571
|
||||
commons license , and indicate if: 571
|
||||
the funders had no role in: 570
|
||||
and permissions information is available at: 570
|
||||
did not receive any specific grant: 566
|
||||
agencies in the public , commercial: 566
|
||||
from the corresponding author on reasonable: 564
|
||||
in any medium , provided the: 563
|
||||
any medium , provided the original: 562
|
||||
to the peer review of this: 562
|
||||
the peer review of this work: 562
|
||||
not receive any specific grant from: 561
|
||||
from funding agencies in the public: 561
|
||||
the corresponding author on reasonable request: 561
|
||||
is available at www.nature.com / reprints: 559
|
||||
funding agencies in the public ,: 558
|
||||
permissions information is available at www.nature.com: 558
|
||||
information is available at www.nature.com /: 558
|
||||
research did not receive any specific: 552
|
||||
/ NUMBER / s NUMBER correspondence: 552
|
||||
NUMBER correspondence and requests for materials: 552
|
||||
below link for the end user: 551
|
||||
NUMBER / s NUMBER correspondence and: 551
|
||||
/ s NUMBER correspondence and requests: 551
|
||||
s NUMBER correspondence and requests for: 551
|
||||
online in the supporting information section: 550
|
||||
public , commercial , or not-for-profit: 549
|
||||
found online at https :// www.frontiersin.org: 549
|
||||
online at https :// www.frontiersin.org /: 549
|
||||
at https :// www.frontiersin.org / articles: 548
|
||||
https :// www.frontiersin.org / articles /: 548
|
||||
", distribution , and reproduction in": 546
|
||||
distribution , and reproduction in any: 546
|
||||
", and reproduction in any medium": 546
|
||||
:// www.frontiersin.org / articles / NUMBER: 545
|
||||
www.frontiersin.org / articles / NUMBER /: 544
|
||||
contribution to the peer review of: 542
|
||||
grant from funding agencies in the: 541
|
||||
use , distribution , and reproduction: 537
|
||||
", commercial , or not-for-profit sectors": 535
|
||||
received no financial support for the: 535
|
||||
no financial support for the research: 534
|
||||
receive any specific grant from funding: 529
|
||||
specific grant from funding agencies in: 529
|
||||
approved the final version of the: 529
|
||||
any specific grant from funding agencies: 528
|
||||
author ( s ) declared no: 526
|
||||
is available online at https ://: 525
|
||||
( s ) declared no potential: 525
|
||||
s ) declared no potential conflicts: 525
|
||||
) declared no potential conflicts of: 525
|
||||
article is distributed under the terms: 520
|
||||
this article is distributed under the: 518
|
||||
"% and NUMBER % , respectively": 507
|
||||
access this article is distributed under: 501
|
||||
open access this article is distributed: 498
|
||||
authors declare that there is no: 487
|
||||
the following is available online at: 485
|
||||
NUMBER % , NUMBER % ,: 484
|
||||
authors would like to thank the: 479
|
||||
is a open access article under: 477
|
||||
a open access article under the: 477
|
||||
medium , provided the original work: 473
|
||||
/ NUMBER /) , which permits: 471
|
||||
the online version , at doi: 466
|
||||
interest was reported by the author: 464
|
||||
supplementary material associated with this article: 452
|
||||
material associated with this article can: 452
|
||||
we would like to thank all: 449
|
||||
http :// creat iveco mmons.org /: 448
|
||||
:// creat iveco mmons.org / licen: 446
|
||||
creat iveco mmons.org / licen ses: 446
|
||||
iveco mmons.org / licen ses /: 446
|
||||
following is available online at https: 446
|
||||
available online at https :// www.mdpi.com: 445
|
||||
online at https :// www.mdpi.com /: 445
|
||||
union's horizon NUMBER research and innovation: 443
|
||||
during the current study is available: 443
|
||||
european union's horizon NUMBER research and: 441
|
||||
the netherlands organization for scientific research: 441
|
||||
the authors declare that there is: 441
|
||||
NUMBER international license ( http ://: 440
|
||||
of the authors and do not: 439
|
||||
www.mdpi.com / article / NUMBER /: 438
|
||||
NUMBER CITATION , NUMBER CITATION ,: 436
|
||||
access article under the terms of: 435
|
||||
article under the terms of the: 435
|
||||
/ by / NUMBER /) ,: 435
|
||||
by / NUMBER /) , which: 435
|
||||
/ licen ses / by /: 434
|
||||
licen ses / by / NUMBER: 434
|
||||
open access article under the terms: 434
|
||||
current study is available from the: 434
|
||||
the current study is available from: 433
|
||||
", which permits unrestricted use ,": 431
|
||||
mmons.org / licen ses / by: 429
|
||||
have no conflicts of interest to: 427
|
||||
which permits unrestricted use , distribution: 427
|
||||
", there is a need for": 425
|
||||
permits unrestricted use , distribution ,: 425
|
||||
unrestricted use , distribution , and: 425
|
||||
have read and agreed to the: 423
|
||||
authors have read and agreed to: 422
|
||||
any medium , provided you give: 422
|
||||
medium , provided you give appropriate: 422
|
||||
", provided you give appropriate credit": 422
|
||||
provided you give appropriate credit to: 422
|
||||
was reported by the author (: 422
|
||||
reported by the author ( s: 422
|
||||
NUMBER /) , which permits unrestricted: 421
|
||||
in any medium , provided you: 421
|
||||
peer review information nature communications thanks: 421
|
||||
the european union's horizon NUMBER research: 420
|
||||
and agreed to the published version: 419
|
||||
agreed to the published version of: 419
|
||||
the published version of the manuscript: 419
|
||||
to the published version of the: 418
|
||||
the authors and do not necessarily: 417
|
||||
/) , which permits unrestricted use: 417
|
||||
read and agreed to the published: 416
|
||||
those of the authors and do: 414
|
||||
all authors have read and agreed: 413
|
||||
that there is no conflict of: 411
|
||||
attribution NUMBER international license ( http: 410
|
||||
author ( s ) received no: 405
|
||||
consent was obtained from all subjects: 404
|
||||
authors have no conflicts of interest: 403
|
||||
( s ) received no financial: 403
|
||||
s ) received no financial support: 403
|
||||
) received no financial support for: 403
|
||||
NUMBER % and NUMBER % of: 400
|
||||
contributed to the article and approved: 400
|
||||
to the article and approved the: 400
|
||||
authors contributed to the article and: 397
|
||||
the author ( s ) received: 396
|
||||
the article and approved the submitted: 396
|
||||
article and approved the submitted version: 395
|
||||
informed consent statement informed consent was: 393
|
||||
accountable for all aspects of the: 389
|
||||
https :// www.mdpi.com / article /: 389
|
||||
:// www.mdpi.com / article / NUMBER: 389
|
||||
consent statement informed consent was obtained: 389
|
||||
at https :// www.mdpi.com / article: 388
|
||||
statement informed consent was obtained from: 388
|
||||
no role in the design of: 385
|
||||
) for their contribution to the: 383
|
||||
", we would like to thank": 381
|
||||
is available on request from the: 378
|
||||
had no role in study design: 377
|
||||
responsibility for the integrity of the: 375
|
||||
of the creative commons attribution NUMBER: 375
|
||||
in the supporting information section at: 374
|
||||
the supporting information section at the: 374
|
||||
supporting information section at the end: 374
|
||||
reviewer ( s ) for their: 374
|
||||
this study is available on request: 373
|
||||
all authors contributed to the article: 373
|
||||
( s ) for their contribution: 372
|
||||
s ) for their contribution to: 372
|
||||
", analysis and interpretation of data": 371
|
||||
available on request from the corresponding: 370
|
||||
in accordance with the ethical standards: 370
|
||||
the manuscript for important intellectual content: 368
|
||||
for all aspects of the work: 367
|
||||
role in the design of the: 367
|
||||
authors declare no conflicts of interest: 366
|
||||
", analysis , and interpretation of": 365
|
||||
on request from the corresponding author: 364
|
||||
be accountable for all aspects of: 362
|
||||
obtained from all subjects involved in: 361
|
||||
funders had no role in the: 360
|
||||
for scientific research ( nwo ): 359
|
||||
the creative commons attribution NUMBER international: 359
|
||||
was obtained from all subjects involved: 359
|
||||
authors and do not necessarily represent: 358
|
||||
to be accountable for all aspects: 356
|
||||
original work is properly cited ,: 355
|
||||
data in the writing of the: 354
|
||||
no role in study design ,: 353
|
||||
from all subjects involved in the: 353
|
||||
all subjects involved in the study: 351
|
||||
provenance and peer review not commissioned: 346
|
||||
study is available on request from: 346
|
||||
there is no conflict of interest: 346
|
||||
wrote the first draft of the: 345
|
||||
NUMBER / NUMBER / NUMBER /: 345
|
||||
written informed consent to participate in: 341
|
||||
commons attribution NUMBER international license (: 341
|
||||
interpretation of data in the writing: 339
|
||||
of data in the writing of: 339
|
||||
ses / by / NUMBER /: 337
|
||||
we would also like to thank: 336
|
||||
the integrity of the data and: 335
|
||||
and peer review not commissioned externally: 334
|
||||
of this study was to assess: 334
|
||||
peer review not commissioned externally peer: 332
|
||||
review not commissioned externally peer reviewed: 332
|
||||
declare that there is no conflict: 331
|
||||
role in study design , data: 328
|
||||
written informed consent was obtained from: 327
|
||||
no conflicts of interest to declare: 326
|
||||
consent to participate in this study: 326
|
||||
in the article's creative commons license: 326
|
||||
do not necessarily represent those of: 325
|
||||
( http :// creativecommons.org / licenses: 324
|
||||
the authors have no conflicts of: 322
|
||||
informed consent to participate in this: 322
|
||||
with the ethical standards of the: 322
|
||||
license ( http :// creativecommons.org /: 322
|
||||
solely those of the authors and: 322
|
||||
and do not necessarily represent those: 322
|
||||
in this article is solely those: 320
|
||||
this article is solely those of: 320
|
||||
article is solely those of the: 320
|
||||
integrity of the data and the: 319
|
||||
of the data and the accuracy: 319
|
||||
the data and the accuracy of: 319
|
||||
accordance with the ethical standards of: 319
|
||||
data and the accuracy of the: 318
|
||||
during the conduct of the study: 317
|
||||
the creative commons attribution CITATION ,: 317
|
||||
expressed in this article is solely: 316
|
||||
and approved the final version of: 315
|
||||
publisher's note all claims expressed in: 315
|
||||
note all claims expressed in this: 315
|
||||
all claims expressed in this article: 315
|
||||
claims expressed in this article is: 315
|
||||
not necessarily represent those of their: 315
|
||||
necessarily represent those of their affiliated: 315
|
||||
represent those of their affiliated organizations: 315
|
||||
or those of the publisher ,: 315
|
||||
those of the publisher , the: 314
|
||||
of the publisher , the editors: 314
|
||||
in the writing of the manuscript: 313
|
||||
affiliated organizations , or those of: 312
|
||||
organizations , or those of the: 312
|
||||
or claim that may be made: 312
|
||||
claim that may be made by: 312
|
||||
that may be made by its: 312
|
||||
is not guaranteed or endorsed by: 312
|
||||
not guaranteed or endorsed by the: 312
|
||||
guaranteed or endorsed by the publisher: 312
|
||||
authors would like to thank all: 311
|
||||
related to this article can be: 311
|
||||
those of their affiliated organizations ,: 311
|
||||
of their affiliated organizations , or: 311
|
||||
their affiliated organizations , or those: 311
|
||||
", or those of the publisher": 311
|
||||
any product that may be evaluated: 311
|
||||
this article , or claim that: 311
|
||||
article , or claim that may: 311
|
||||
", or claim that may be": 311
|
||||
product that may be evaluated in: 310
|
||||
may be made by its manufacturer: 310
|
||||
the accuracy of the data analysis: 309
|
||||
", there is a need to": 309
|
||||
of the creative commons attribution license: 309
|
||||
that may be evaluated in this: 309
|
||||
may be evaluated in this article: 309
|
||||
be made by its manufacturer ,: 309
|
||||
made by its manufacturer , is: 309
|
||||
by its manufacturer , is not: 309
|
||||
its manufacturer , is not guaranteed: 309
|
||||
manufacturer , is not guaranteed or: 309
|
||||
", is not guaranteed or endorsed": 309
|
||||
", there is a lack of": 308
|
||||
the authors declare no conflicts of: 308
|
||||
be evaluated in this article ,: 308
|
||||
evaluated in this article , or: 308
|
||||
in this article , or claim: 308
|
||||
the data in the study and: 306
|
||||
in this paper , we present: 306
|
||||
information may be found in the: 305
|
||||
this study was to investigate the: 305
|
||||
", on the one hand ,": 305
|
||||
may be found in the online: 304
|
||||
is not publicly available due to: 304
|
||||
authors report no conflicts of interest: 304
|
||||
online version , at doi NUMBER: 303
|
||||
version , at doi NUMBER /: 303
|
||||
supporting information may be found in: 302
|
||||
in the public , commercial or: 301
|
||||
from any funding agency in the: 300
|
||||
any funding agency in the public: 300
|
||||
were in accordance with the ethical: 300
|
||||
the publisher , the editors and: 300
|
||||
publisher , the editors and the: 300
|
||||
", the editors and the reviewers": 300
|
||||
funding agency in the public ,: 299
|
||||
agency in the public , commercial: 299
|
||||
found in the online version of: 297
|
||||
there is no conflicts of interest: 295
|
||||
can be found in the online: 295
|
||||
article can be found in the: 291
|
||||
use , distribution and reproduction in: 289
|
||||
the online version of this article: 287
|
||||
before it is published in its: 287
|
||||
it is published in its final: 287
|
||||
to submit the manuscript for publication: 286
|
||||
international license ( http :// creativecommons.org: 285
|
||||
is published in its final form: 285
|
||||
decision to submit the manuscript for: 284
|
||||
", which permits use , distribution": 284
|
||||
which permits use , distribution and: 284
|
||||
permits use , distribution and reproduction: 284
|
||||
", as shown in fig. NUMBER": 284
|
||||
errors may be discovered which could: 284
|
||||
may be discovered which could affect: 284
|
||||
be discovered which could affect the: 284
|
||||
discovered which could affect the content: 284
|
||||
which could affect the content ,: 284
|
||||
could affect the content , and: 284
|
||||
affect the content , and all: 284
|
||||
the content , and all legal: 284
|
||||
content , and all legal disclaimers: 284
|
||||
all legal disclaimers that apply to: 283
|
||||
legal disclaimers that apply to the: 283
|
||||
disclaimers that apply to the journal: 283
|
||||
design , data collection and analysis: 282
|
||||
", and all legal disclaimers that": 282
|
||||
and all legal disclaimers that apply: 282
|
||||
from the european union's horizon NUMBER: 280
|
||||
and the other , anonymous ,: 280
|
||||
is a pdf file of a: 278
|
||||
this article can be found in: 277
|
||||
helsinki declaration and its later amendments: 276
|
||||
NUMBER helsinki declaration and its later: 275
|
||||
the NUMBER helsinki declaration and its: 274
|
||||
during the production process , errors: 274
|
||||
the production process , errors may: 274
|
||||
production process , errors may be: 274
|
||||
process , errors may be discovered: 274
|
||||
this is a pdf file of: 273
|
||||
review before it is published in: 273
|
||||
", errors may be discovered which": 273
|
||||
will undergo additional copyediting , typesetting: 271
|
||||
and review before it is published: 271
|
||||
note that , during the production: 271
|
||||
that , during the production process: 271
|
||||
", during the production process ,": 271
|
||||
with the NUMBER helsinki declaration and: 270
|
||||
this version will undergo additional copyediting: 270
|
||||
version will undergo additional copyediting ,: 270
|
||||
undergo additional copyediting , typesetting and: 270
|
||||
additional copyediting , typesetting and review: 270
|
||||
copyediting , typesetting and review before: 270
|
||||
", typesetting and review before it": 270
|
||||
typesetting and review before it is: 270
|
||||
published in its final form ,: 270
|
||||
in its final form , but: 270
|
||||
its final form , but we: 270
|
||||
final form , but we is: 270
|
||||
form , but we is providing: 270
|
||||
", but we is providing this": 270
|
||||
but we is providing this version: 270
|
||||
we is providing this version to: 270
|
||||
is providing this version to give: 270
|
||||
providing this version to give early: 270
|
||||
this version to give early visibility: 270
|
||||
version to give early visibility of: 270
|
||||
to give early visibility of the: 270
|
||||
give early visibility of the article: 270
|
||||
please note that , during the: 270
|
||||
netherlands organization for scientific research (: 269
|
||||
acute respiratory syndrome coronavirus NUMBER CITATION: 268
|
||||
a pdf file of a article: 268
|
||||
pdf file of a article that: 268
|
||||
file of a article that has: 268
|
||||
of a article that has undergone: 268
|
||||
a article that has undergone enhancements: 268
|
||||
article that has undergone enhancements after: 268
|
||||
that has undergone enhancements after acceptance: 268
|
||||
has undergone enhancements after acceptance ,: 268
|
||||
undergone enhancements after acceptance , such: 268
|
||||
enhancements after acceptance , such as: 268
|
||||
after acceptance , such as the: 268
|
||||
acceptance , such as the addition: 268
|
||||
as the addition of a cover: 268
|
||||
the addition of a cover page: 268
|
||||
addition of a cover page and: 268
|
||||
of a cover page and metadata: 268
|
||||
a cover page and metadata ,: 268
|
||||
cover page and metadata , and: 268
|
||||
page and metadata , and formatting: 268
|
||||
and metadata , and formatting for: 268
|
||||
metadata , and formatting for readability: 268
|
||||
", and formatting for readability ,": 268
|
||||
and formatting for readability , but: 268
|
||||
formatting for readability , but it: 268
|
||||
for readability , but it is: 268
|
||||
readability , but it is not: 268
|
||||
but it is not yet the: 268
|
||||
it is not yet the definitive: 268
|
||||
is not yet the definitive version: 268
|
||||
not yet the definitive version of: 268
|
||||
yet the definitive version of record: 268
|
||||
and its later amendments or comparable: 267
|
||||
its later amendments or comparable ethical: 267
|
||||
the netherlands organisation for scientific research: 265
|
||||
analysis and interpretation of data ,: 265
|
||||
NUMBER https :// doi.org / NUMBER: 265
|
||||
later amendments or comparable ethical standards: 265
|
||||
the public , commercial or not-for-profit: 265
|
||||
the aim of this paper is: 263
|
||||
"% , NUMBER % and NUMBER": 262
|
||||
the other , anonymous , reviewer: 261
|
||||
and with the NUMBER helsinki declaration: 260
|
||||
human participants were reviewed and approved: 259
|
||||
participants were reviewed and approved by: 259
|
||||
involving human participants were reviewed and: 258
|
||||
studies involving human participants were reviewed: 257
|
||||
and analysis , decision to publish: 257
|
||||
it should be noted that the: 257
|
||||
in the decision to publish the: 256
|
||||
study design , data collection ,: 256
|
||||
found in the online version at: 256
|
||||
the authors report no conflicts of: 255
|
||||
under the terms of the creat: 255
|
||||
the terms of the creat ive: 255
|
||||
terms of the creat ive commo: 255
|
||||
of the creat ive commo ns: 255
|
||||
the creat ive commo ns attri: 255
|
||||
they have no conflicts of interest: 255
|
||||
data presented in this study is: 255
|
||||
participants provided their written informed consent: 254
|
||||
their written informed consent to participate: 254
|
||||
the decision to publish the results: 254
|
||||
the data presented in this study: 254
|
||||
provided their written informed consent to: 253
|
||||
CITATION license , which permits others: 252
|
||||
license , which permits others to: 252
|
||||
all authors approved the final version: 251
|
||||
that apply to the journal pertain: 251
|
||||
anonymous , reviewer ( s ): 251
|
||||
study is included in the article: 250
|
||||
other , anonymous , reviewer (: 250
|
||||
", anonymous , reviewer ( s": 250
|
||||
", reviewer ( s ) for": 250
|
||||
|
|
@ -1,145 +0,0 @@
|
|||
import re
|
||||
from typing import Dict, Pattern, Tuple, Union
|
||||
|
||||
from .models import BookmarkTitle
|
||||
|
||||
# partly inspired by https://github.com/KMCS-NII/AASC/blob/master/section_classify/section_classify.pl
|
||||
titles_of_interest: Dict[BookmarkTitle, Tuple[Union[str, Pattern], ...]] = {
|
||||
"Abstract": ("abstract",),
|
||||
"Author contribution": (
|
||||
"author contribution",
|
||||
"credit authorship contribution statement",
|
||||
"credit author statement",
|
||||
"corresponding author",
|
||||
),
|
||||
"Introduction": (
|
||||
"introduction",
|
||||
"preliminaries",
|
||||
"research question",
|
||||
"objective",
|
||||
"purpose",
|
||||
re.compile(r"^aim$"),
|
||||
"proposition",
|
||||
"what this study adds",
|
||||
"what does this study add",
|
||||
),
|
||||
"Background": (
|
||||
"background",
|
||||
"previous",
|
||||
"prior",
|
||||
"relevant work",
|
||||
"state of the art",
|
||||
"related work",
|
||||
"research in context",
|
||||
"earlier work",
|
||||
"related work",
|
||||
"problem",
|
||||
"challenge",
|
||||
"goal",
|
||||
"literature review",
|
||||
re.compile(r"^review$"),
|
||||
re.compile(r"^existing"),
|
||||
"comparison with existing literature",
|
||||
"literature search",
|
||||
"comparison with other studies",
|
||||
"study area",
|
||||
),
|
||||
"Methods": (
|
||||
"method",
|
||||
"approach",
|
||||
re.compile(r"^our"),
|
||||
"proposed",
|
||||
"study design",
|
||||
"procedure",
|
||||
"formulation",
|
||||
"methodology",
|
||||
"research design",
|
||||
"study protocol",
|
||||
"design",
|
||||
re.compile(r"experimental (?!result)"),
|
||||
"overview",
|
||||
"approach",
|
||||
"system",
|
||||
"definition",
|
||||
"algorithm",
|
||||
"model",
|
||||
"setup",
|
||||
"implementation",
|
||||
"evaluation metric",
|
||||
"scheme",
|
||||
),
|
||||
"Results": (
|
||||
"result",
|
||||
"data",
|
||||
"outcome",
|
||||
"statistic",
|
||||
"statistics",
|
||||
"statistical analysis",
|
||||
"analysis",
|
||||
"measure",
|
||||
"experiment",
|
||||
"finding",
|
||||
"hypothesis testing",
|
||||
"proof of proposition",
|
||||
"proof of concept",
|
||||
"report",
|
||||
"implication of all the available evidence",
|
||||
"contribution",
|
||||
"evaluation",
|
||||
"performance",
|
||||
"demonstration",
|
||||
"demonstrating",
|
||||
"example",
|
||||
"study",
|
||||
"studies",
|
||||
"qualitative comparison",
|
||||
"quantitative comparison",
|
||||
"validation",
|
||||
),
|
||||
"Discussion": (
|
||||
"discussion",
|
||||
"recommendation",
|
||||
"implication",
|
||||
"findings",
|
||||
"impact",
|
||||
"limitation",
|
||||
"assessment",
|
||||
"quality assessment",
|
||||
"interpretation",
|
||||
"contribution",
|
||||
"explanation",
|
||||
),
|
||||
"Outlook": (
|
||||
"outlook",
|
||||
"future research",
|
||||
"future",
|
||||
"follow up",
|
||||
"remaining",
|
||||
"prospect",
|
||||
"perspective",
|
||||
),
|
||||
"Conclusion": (
|
||||
"conclusion",
|
||||
"conclude",
|
||||
"concluding",
|
||||
"concluding",
|
||||
"final remark",
|
||||
"remark",
|
||||
"conclusie",
|
||||
),
|
||||
"Conflict of interest": (
|
||||
"compete interest",
|
||||
"funding support and author disclosure",
|
||||
"conflict of interest",
|
||||
"disclosure statement",
|
||||
"role of the funding source",
|
||||
"disclaimer",
|
||||
"conflicting interest",
|
||||
"declaration of interest",
|
||||
"risk of bias",
|
||||
"funding",
|
||||
"disclosure",
|
||||
"declaration",
|
||||
),
|
||||
"Acknowledgement": ("acknowledgement", "acknowledgment"),
|
||||
}
|
||||
|
|
@ -1,139 +0,0 @@
|
|||
import unittest
|
||||
|
||||
from src.great_ai.utilities import lemmatize_text
|
||||
|
||||
|
||||
class TestLemmatizeText(unittest.TestCase):
|
||||
def test_simple(self) -> None:
|
||||
text = "The state-of-the-art could not be improved, however we managed to create a less resource-intensive implementation of it."
|
||||
|
||||
lemmatized = [
|
||||
"the",
|
||||
"state",
|
||||
"-",
|
||||
"of",
|
||||
"-",
|
||||
"the",
|
||||
"-",
|
||||
"art",
|
||||
"could",
|
||||
"not",
|
||||
"be",
|
||||
"improve",
|
||||
",",
|
||||
"however",
|
||||
"we",
|
||||
"manage",
|
||||
"to",
|
||||
"create",
|
||||
"a",
|
||||
"less",
|
||||
"resource",
|
||||
"-",
|
||||
"intensive",
|
||||
"implementation",
|
||||
"of",
|
||||
"it",
|
||||
".",
|
||||
]
|
||||
|
||||
lemmatized_pos = [
|
||||
"the_DET",
|
||||
"state_NOUN",
|
||||
"-_PUNCT",
|
||||
"of_ADP",
|
||||
"-_PUNCT",
|
||||
"the_DET",
|
||||
"-_PUNCT",
|
||||
"art_NOUN",
|
||||
"could_AUX",
|
||||
"not_PART",
|
||||
"be_AUX",
|
||||
"improve_VERB",
|
||||
",_PUNCT",
|
||||
"however_ADV",
|
||||
"we_PRON",
|
||||
"manage_VERB",
|
||||
"to_PART",
|
||||
"create_VERB",
|
||||
"a_DET",
|
||||
"less_ADV",
|
||||
"resource_NOUN",
|
||||
"-_PUNCT",
|
||||
"intensive_ADJ",
|
||||
"implementation_NOUN",
|
||||
"of_ADP",
|
||||
"it_PRON",
|
||||
"._PUNCT",
|
||||
]
|
||||
|
||||
lemmatized_neg = [
|
||||
"the",
|
||||
"state",
|
||||
"-",
|
||||
"of",
|
||||
"-",
|
||||
"the",
|
||||
"-",
|
||||
"art",
|
||||
"could",
|
||||
"not",
|
||||
"NOT_be",
|
||||
"NOT_improve",
|
||||
"NOT_,",
|
||||
"however",
|
||||
"we",
|
||||
"manage",
|
||||
"to",
|
||||
"create",
|
||||
"a",
|
||||
"less",
|
||||
"resource",
|
||||
"-",
|
||||
"intensive",
|
||||
"implementation",
|
||||
"of",
|
||||
"it",
|
||||
".",
|
||||
]
|
||||
|
||||
lemmatized_pos_neg = [
|
||||
"the_DET",
|
||||
"state_NOUN",
|
||||
"-_PUNCT",
|
||||
"of_ADP",
|
||||
"-_PUNCT",
|
||||
"the_DET",
|
||||
"-_PUNCT",
|
||||
"art_NOUN",
|
||||
"could_AUX",
|
||||
"not_PART",
|
||||
"NOT_be_AUX",
|
||||
"NOT_improve_VERB",
|
||||
"NOT_,_PUNCT",
|
||||
"however_ADV",
|
||||
"we_PRON",
|
||||
"manage_VERB",
|
||||
"to_PART",
|
||||
"create_VERB",
|
||||
"a_DET",
|
||||
"less_ADV",
|
||||
"resource_NOUN",
|
||||
"-_PUNCT",
|
||||
"intensive_ADJ",
|
||||
"implementation_NOUN",
|
||||
"of_ADP",
|
||||
"it_PRON",
|
||||
"._PUNCT",
|
||||
]
|
||||
|
||||
assert lemmatize_text(text) == lemmatized
|
||||
assert lemmatize_text(text, add_part_of_speech=True) == lemmatized_pos
|
||||
assert lemmatize_text(text, add_negation=True) == lemmatized_neg
|
||||
self.assertEqual(
|
||||
lemmatize_text(text, add_negation=True, add_part_of_speech=True),
|
||||
lemmatized_pos_neg,
|
||||
)
|
||||
|
||||
def test_empty(self) -> None:
|
||||
assert lemmatize_text("") == []
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
import unittest
|
||||
|
||||
from src.great_ai.utilities import lemmatize_token, nlp
|
||||
|
||||
|
||||
class TestLemmatizeToken(unittest.TestCase):
|
||||
def test_simple(self) -> None:
|
||||
token = nlp("Center")[0]
|
||||
|
||||
assert lemmatize_token(token) == "centre"
|
||||
assert lemmatize_token(token, add_negation=True) == "centre"
|
||||
assert lemmatize_token(token, add_part_of_speech=True) == "centre_NOUN"
|
||||
|
||||
def test_punctuation(self) -> None:
|
||||
token = nlp("This.")[1]
|
||||
|
||||
assert lemmatize_token(token) == "."
|
||||
assert lemmatize_token(token, add_negation=True) == "."
|
||||
assert lemmatize_token(token, add_part_of_speech=True) == "._PUNCT"
|
||||
|
|
@ -1,67 +0,0 @@
|
|||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from src.great_ai.utilities import PublicationTEI
|
||||
|
||||
from .data.parsed import (
|
||||
abstract,
|
||||
authors,
|
||||
bookmarks,
|
||||
conclusion,
|
||||
content,
|
||||
introduction,
|
||||
metadata,
|
||||
sentences,
|
||||
)
|
||||
|
||||
DATA_PATH = Path(__file__).parent.resolve() / "data"
|
||||
|
||||
|
||||
class TestPublicationTEI(unittest.TestCase):
|
||||
test_xml: str
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
with open(
|
||||
DATA_PATH / "10.1136_bmjspcare-2021-003026.pdf.tei.xml", encoding="utf-8"
|
||||
) as f:
|
||||
cls.test_xml = f.read()
|
||||
|
||||
def test_metadata_extraction(self) -> None:
|
||||
assert PublicationTEI(self.test_xml).publication_metadata == metadata
|
||||
|
||||
def test_authors(self) -> None:
|
||||
assert PublicationTEI(self.test_xml).authors == authors
|
||||
|
||||
def test_content(self) -> None:
|
||||
assert PublicationTEI(self.test_xml).content == content
|
||||
|
||||
def test_sentences(self) -> None:
|
||||
assert PublicationTEI(self.test_xml).sentences == sentences
|
||||
|
||||
def test_bookmarks(self) -> None:
|
||||
assert PublicationTEI(self.test_xml).bookmarks == bookmarks
|
||||
|
||||
def test_abstract(self) -> None:
|
||||
assert PublicationTEI(self.test_xml).abstract_sentences == abstract
|
||||
|
||||
def test_introduction(self) -> None:
|
||||
self.assertEqual(
|
||||
PublicationTEI(self.test_xml).introduction_sentences, introduction
|
||||
)
|
||||
|
||||
def test_conclusion(self) -> None:
|
||||
assert PublicationTEI(self.test_xml).conclusion_sentences == conclusion
|
||||
|
||||
def test_empty1(self) -> None:
|
||||
tei = PublicationTEI("<TEI/>")
|
||||
tei.publication_metadata, tei.publication_metadata, tei.authors, tei.content, tei.sentences
|
||||
|
||||
def test_empty2(self) -> None:
|
||||
tei = PublicationTEI("")
|
||||
tei.publication_metadata, tei.publication_metadata, tei.authors, tei.content, tei.sentences
|
||||
|
||||
def test_missing_fields(self) -> None:
|
||||
with open(DATA_PATH / "bad.tei.xml", encoding="utf-8") as f:
|
||||
tei = PublicationTEI(f.read())
|
||||
tei.publication_metadata, tei.publication_metadata, tei.authors, tei.content, tei.sentences
|
||||
Loading…
Add table
Add a link
Reference in a new issue