Add sus updates
This commit is contained in:
parent
a7e3dc11fd
commit
3ef1a72418
31 changed files with 2502 additions and 970 deletions
|
|
@ -9,6 +9,7 @@
|
|||
- [parallel_map](src/sus/parallel_map.py)
|
||||
- [match_names](src/sus/match_names/match_names.py)
|
||||
- [evaluate_ranking](src/sus/evaluate_ranking/evaluate_ranking.py)
|
||||
- [get_sentences](src/sus/get_sentences.py)
|
||||
|
||||
### Requires loading spacy model
|
||||
|
||||
|
|
@ -17,14 +18,14 @@
|
|||
> Add this to the Dockerfile for caching the spaCy model:
|
||||
>
|
||||
> ```docker
|
||||
> RUN pip install --no-cache-dir en-core-web-lg@https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.2.0/en_core_web_lg-3.2.0-py3-none-any.whl
|
||||
> RUN pip install --no-cache-dir en-core-web-sm@https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl
|
||||
> ```
|
||||
|
||||
- [spacy model (nlp)](src/sus/nlp.py)
|
||||
- [get_sentences](src/sus/get_sentences.py)
|
||||
- [publication TEI](src/sus/publication_tei/publication_tei.py)
|
||||
- [lemmatize_text](src/sus/lemmatize_text.py)
|
||||
- [lemmatize_token](src/sus/lemmatize_token.py)
|
||||
- [publication TEI](src/sus/publication_tei/publication_tei.py)
|
||||
- [spacy model (nlp)](src/sus/nlp.py)
|
||||
- [filter_sentences](src/sus/matcher/filter_sentences.py)
|
||||
|
||||
## Development
|
||||
|
||||
|
|
|
|||
|
|
@ -1,14 +1,14 @@
|
|||
[metadata]
|
||||
name = sus
|
||||
version = 0.0.3
|
||||
author = ScoutinScience B.V.
|
||||
name = great-ai
|
||||
version = 0.0.1
|
||||
author = András Schmelczer
|
||||
author_email = andras@scoutinscience.com
|
||||
description = [S]coutinScience [u]tilitie[s]: reusable utilities for text processing
|
||||
description =
|
||||
long_description = file: README.md
|
||||
long_description_content_type = text/markdown
|
||||
url = https://github.com/ScoutinScience/platform
|
||||
url = https://github.com/ScoutinScience/great-ai
|
||||
project_urls =
|
||||
Bug Tracker = https://github.com/ScoutinScience/platform/issues
|
||||
Bug Tracker = https://github.com/ScoutinScience/great-ai/issues
|
||||
classifiers =
|
||||
Programming Language :: Python :: 3
|
||||
Operating System :: OS Independent
|
||||
|
|
@ -27,13 +27,17 @@ install_requires =
|
|||
psutil >= 5.9.0
|
||||
beautifulsoup4 >= 4.10.0
|
||||
lxml >= 4.6.0
|
||||
spacy >= 3.2.0
|
||||
spacy >= 3.3.0
|
||||
pydantic >= 1.8.0
|
||||
scikit-learn >= 1.0.0
|
||||
matplotlib >= 3.5.0
|
||||
numpy >= 1.22.0
|
||||
langcodes[data] >= 3.3.0
|
||||
segtok >= 1.5.11
|
||||
langdetect >= 1.0.9
|
||||
|
||||
[options.package_data]
|
||||
* = *.json, *.yaml, *.yml
|
||||
|
||||
[options.packages.find]
|
||||
where = src
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ from .external.pylatexenc.latex2text import LatexNodes2Text
|
|||
from .logger.create_logger import create_logger
|
||||
|
||||
logger = create_logger("clean")
|
||||
|
||||
latex = LatexNodes2Text()
|
||||
|
||||
|
||||
|
|
@ -46,6 +45,10 @@ def clean(
|
|||
text = "".join([c for c in text if not unicodedata.combining(c)])
|
||||
text = unidecode.unidecode(text)
|
||||
|
||||
text = re.sub(
|
||||
r"\b[a-zA-Z](?:[\t ]+[a-zA-Z]\b)+", lambda m: re.sub(r"[\t ]", "", m[0]), text
|
||||
) # A R T I C L E => ARTICLE
|
||||
|
||||
if remove_brackets:
|
||||
text = re.sub(r"\[[^\]]*\]", " ", text)
|
||||
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ preceding = [
|
|||
"doubt",
|
||||
"negative for",
|
||||
"no",
|
||||
"vergreat_ai.utilities",
|
||||
"versus",
|
||||
"without",
|
||||
"doesn't",
|
||||
"doesnt",
|
||||
|
|
@ -104,7 +104,7 @@ pseudo_clinical = pseudo + [
|
|||
"not ruled out",
|
||||
"not been ruled out",
|
||||
"not drain",
|
||||
"no great_ai.utilitiespicious change",
|
||||
"no suspicious change",
|
||||
"no interval change",
|
||||
"no significant interval change",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -287,7 +287,7 @@ def placeholder_node_formatter(placeholdertext, block=True):
|
|||
r"""
|
||||
This function returns a callable that can be used in
|
||||
:py:class:`MacroTextSpec`, :py:class:`EnvironmentTextSpec`, or
|
||||
:py:class:`SpecialsTextSpec` for latex nodes that do not have a great textual
|
||||
:py:class:`SpecialsTextSpec` for latex nodes that do not have a good textual
|
||||
representation, providing as text replacement the simple placeholder text
|
||||
``'< P L A C E H O L D E R T E X T >'``.
|
||||
|
||||
|
|
@ -316,7 +316,7 @@ def fmt_placeholder_node(node, l2tobj):
|
|||
r"""
|
||||
This function can be used as callable in :py:class:`MacroTextSpec`,
|
||||
:py:class:`EnvironmentTextSpec`, or :py:class:`SpecialsTextSpec` for latex
|
||||
nodes that do not have a great textual representation. The text replacement
|
||||
nodes that do not have a good textual representation. The text replacement
|
||||
is the placeholder text
|
||||
``'< N A M E O F T H E M A C R O O R E N V I R O N M E N T >'``.
|
||||
|
||||
|
|
|
|||
|
|
@ -266,7 +266,7 @@ _latex_specs_approximations = {
|
|||
("fcolorbox", "%(5)s"),
|
||||
("hspace", ""),
|
||||
("vspace", "\n"),
|
||||
# \\ is treated as an "approximation" because a great text renderer would
|
||||
# \\ is treated as an "approximation" because a good text renderer would
|
||||
# have to actually note that this is a end-of-line marker which is not
|
||||
# to be confused with other newlines in the paragraph (which can be
|
||||
# reflowed)
|
||||
|
|
|
|||
|
|
@ -342,7 +342,7 @@ class UnicodeToLatexEncoder(object):
|
|||
|
||||
Possible protection schemes are:
|
||||
|
||||
- 'braces' (the default): Any great_ai.utilitiespicious replacement text (that
|
||||
- 'braces' (the default): Any suspicious replacement text (that
|
||||
might look fragile) is placed in curly braces ``{...}``.
|
||||
|
||||
- 'braces-all': All replacement latex escapes are surrounded in
|
||||
|
|
|
|||
230
great_ai/src/great_ai/utilities/external/pylatexenc/latexwalker/__main__.py
vendored
Normal file
230
great_ai/src/great_ai/utilities/external/pylatexenc/latexwalker/__main__.py
vendored
Normal file
|
|
@ -0,0 +1,230 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# The MIT License (MIT)
|
||||
#
|
||||
# Copyright (c) 2018 Philippe Faist
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
#
|
||||
|
||||
import argparse
|
||||
import fileinput
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
|
||||
from ..latexwalker import LatexWalker, disp_node, make_json_encoder
|
||||
from ..version import version_str
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
|
||||
if argv is None:
|
||||
argv = sys.argv[1:]
|
||||
|
||||
parser = argparse.ArgumentParser(prog="latexwalker", add_help=False)
|
||||
|
||||
parser.add_argument(
|
||||
"--output-format",
|
||||
metavar="FORMAT",
|
||||
dest="output_format",
|
||||
choices=["human", "json"],
|
||||
default="human",
|
||||
help='Requested output format for the node tree ("human" or "json")',
|
||||
)
|
||||
parser.add_argument(
|
||||
"--json-indent",
|
||||
metavar="NUMSPACES",
|
||||
dest="json_indent",
|
||||
type=int,
|
||||
default=2,
|
||||
help="Indentation in JSON output (specify number of spaces "
|
||||
"per indentation level)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--json-compact",
|
||||
dest="json_indent",
|
||||
action="store_const",
|
||||
const=None,
|
||||
help="Output compact JSON",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--keep-inline-math",
|
||||
action="store_const",
|
||||
const=True,
|
||||
dest="keep_inline_math",
|
||||
default=True,
|
||||
help=argparse.SUPPRESS,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-keep-inline-math",
|
||||
action="store_const",
|
||||
const=False,
|
||||
dest="keep_inline_math",
|
||||
help=argparse.SUPPRESS,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--tolerant-parsing",
|
||||
action="store_const",
|
||||
const=True,
|
||||
dest="tolerant_parsing",
|
||||
default=True,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-tolerant-parsing",
|
||||
action="store_const",
|
||||
const=False,
|
||||
dest="tolerant_parsing",
|
||||
help="Tolerate syntax errors when parsing, and attempt "
|
||||
"to continue (default yes)",
|
||||
)
|
||||
|
||||
# I'm not sure this flag is useful and if it should be exposed at all.
|
||||
# Accept it, but make it hidden.
|
||||
parser.add_argument(
|
||||
"--strict-braces",
|
||||
action="store_const",
|
||||
const=True,
|
||||
dest="strict_braces",
|
||||
default=False,
|
||||
help=argparse.SUPPRESS,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-strict-braces",
|
||||
action="store_const",
|
||||
const=False,
|
||||
dest="strict_braces",
|
||||
# help="Report errors for mismatching LaTeX braces (default no)"
|
||||
help=argparse.SUPPRESS,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"-q",
|
||||
"--quiet",
|
||||
dest="logging_level",
|
||||
action="store_const",
|
||||
const=logging.ERROR,
|
||||
default=logging.INFO,
|
||||
help="Suppress warning messages",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-v",
|
||||
"--verbose",
|
||||
dest="logging_level",
|
||||
action="store_const",
|
||||
const=logging.DEBUG,
|
||||
help="Verbose output",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--version",
|
||||
action="version",
|
||||
version="pylatexenc {}".format(version_str),
|
||||
help="Show version information and exit",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--help", action="help", help="Show this help information and exit"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--code",
|
||||
"-c",
|
||||
action="store",
|
||||
default=None,
|
||||
metavar="LATEX_CODE",
|
||||
help="Convert the given LATEX_CODE to unicode text instead of reading "
|
||||
"from FILE or standard input. You cannot specify FILEs if you use this "
|
||||
"option, and any standard input is ignored.",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"files",
|
||||
metavar="FILE",
|
||||
nargs="*",
|
||||
help="Input files (if none specified, read from stdandard input)",
|
||||
)
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
logging.basicConfig()
|
||||
logging.getLogger().setLevel(args.logging_level)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
latex = ""
|
||||
if args.code:
|
||||
if args.files:
|
||||
logger.error(
|
||||
"Cannot specify both FILEs and --code option. "
|
||||
"Use --help option for more information."
|
||||
)
|
||||
sys.exit(1)
|
||||
latex = args.code
|
||||
else:
|
||||
for line in fileinput.input(files=args.files):
|
||||
latex += line
|
||||
|
||||
latexwalker = LatexWalker(
|
||||
latex, tolerant_parsing=args.tolerant_parsing, strict_braces=args.strict_braces
|
||||
)
|
||||
|
||||
(nodelist, pos, len_) = latexwalker.get_latex_nodes()
|
||||
|
||||
if args.output_format == "human":
|
||||
print("\n--- NODES ---\n")
|
||||
for n in nodelist:
|
||||
disp_node(n)
|
||||
print("\n-------------\n")
|
||||
return
|
||||
|
||||
if args.output_format == "json":
|
||||
json.dump(
|
||||
{
|
||||
"nodelist": nodelist,
|
||||
},
|
||||
sys.stdout,
|
||||
cls=make_json_encoder(latexwalker),
|
||||
indent=args.json_indent,
|
||||
)
|
||||
sys.stdout.write("\n")
|
||||
return
|
||||
|
||||
raise ValueError("Invalid output format: " + args.output_format)
|
||||
|
||||
|
||||
def run_main():
|
||||
|
||||
try:
|
||||
|
||||
main()
|
||||
|
||||
except SystemExit:
|
||||
raise
|
||||
except: # lgtm [py/catch-base-exception]
|
||||
import pdb
|
||||
import traceback
|
||||
|
||||
traceback.print_exc()
|
||||
pdb.post_mortem()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
# run_main() # debug
|
||||
main()
|
||||
|
|
@ -33,7 +33,7 @@
|
|||
#
|
||||
# - tests pass: https://travis-ci.org/github/phfaist/pylatexenc
|
||||
#
|
||||
# - LGTM looks great: https://lgtm.com/projects/g/phfaist/pylatexenc/
|
||||
# - LGTM looks good: https://lgtm.com/projects/g/phfaist/pylatexenc/
|
||||
#
|
||||
# - python package creation works: (python setup.py sdist, pip install
|
||||
# dist/pylatexenc-xxx.tar.gz)
|
||||
|
|
|
|||
|
|
@ -1,12 +1,15 @@
|
|||
from typing import List
|
||||
|
||||
from segtok.segmenter import split_multi
|
||||
|
||||
from .data import sentence_ending_punctuations
|
||||
from .nlp import nlp
|
||||
|
||||
|
||||
def get_sentences(text: str, ignore_partial: bool = False) -> List[str]:
|
||||
doc = nlp(text)
|
||||
possible_sentences = [s.text for s in doc.sents]
|
||||
if text.strip() == "":
|
||||
return []
|
||||
|
||||
possible_sentences = [s for s in split_multi(text) if s]
|
||||
|
||||
if ignore_partial:
|
||||
possible_sentences = [
|
||||
|
|
|
|||
3
great_ai/src/great_ai/utilities/matcher/__init__.py
Normal file
3
great_ai/src/great_ai/utilities/matcher/__init__.py
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
from .fast_tokenize import fast_tokenize
|
||||
from .filter_sentences import filter_sentences
|
||||
from .normalize import normalize
|
||||
30
great_ai/src/great_ai/utilities/matcher/fast_tokenize.py
Normal file
30
great_ai/src/great_ai/utilities/matcher/fast_tokenize.py
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
import re
|
||||
from typing import List, Union
|
||||
|
||||
from segtok.tokenizer import word_tokenizer
|
||||
|
||||
from ..get_sentences import get_sentences
|
||||
from .normalize import normalize
|
||||
|
||||
|
||||
def fast_tokenize(
|
||||
text: Union[List[str], str], ignore_partial: bool = False
|
||||
) -> List[List[str]]:
|
||||
if isinstance(text, str):
|
||||
text = normalize(text)
|
||||
text = get_sentences(text, ignore_partial=ignore_partial)
|
||||
|
||||
results: List[List[str]] = []
|
||||
|
||||
for sentence in text:
|
||||
sentence = re.sub(r"\bare\b", "is", sentence)
|
||||
sentence = re.sub(r"\ban\b", "a", sentence)
|
||||
sentence = re.sub(r"\bthese\b", "this", sentence)
|
||||
results.append(
|
||||
[
|
||||
token.lower() if token not in {"CITATION", "NUMBER"} else token
|
||||
for token in word_tokenizer(sentence)
|
||||
]
|
||||
)
|
||||
|
||||
return results
|
||||
66
great_ai/src/great_ai/utilities/matcher/filter_sentences.py
Normal file
66
great_ai/src/great_ai/utilities/matcher/filter_sentences.py
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
from pathlib import Path
|
||||
from typing import Dict, List, Union
|
||||
|
||||
import yaml
|
||||
from spacy.matcher import Matcher
|
||||
|
||||
from ..get_sentences import get_sentences
|
||||
from ..nlp import nlp
|
||||
from .fast_tokenize import fast_tokenize
|
||||
from .normalize import normalize
|
||||
|
||||
rules_cache: Dict[str, Matcher] = {}
|
||||
|
||||
|
||||
def filter_sentences(
|
||||
sentences: str,
|
||||
rules_file: Path,
|
||||
inverse: bool = False,
|
||||
ignore_partial: bool = False,
|
||||
) -> List[str]:
|
||||
if str(rules_file) not in rules_cache:
|
||||
with open(rules_file, encoding="utf-8") as f:
|
||||
rule_patterns = yaml.safe_load(f).keys()
|
||||
|
||||
matcher = Matcher(nlp.vocab)
|
||||
rules = [_pattern_to_rule(p) for p in rule_patterns]
|
||||
matcher.add("", rules)
|
||||
rules_cache[str(rules_file)] = matcher
|
||||
|
||||
matcher = rules_cache[str(rules_file)]
|
||||
|
||||
original_sentences = get_sentences(sentences, ignore_partial=ignore_partial)
|
||||
|
||||
tokenized = fast_tokenize(original_sentences, ignore_partial=ignore_partial)
|
||||
|
||||
results: List[str] = []
|
||||
for original_sentence, sentence in zip(original_sentences, tokenized):
|
||||
doc = nlp(normalize(" ".join(sentence)))
|
||||
matches = matcher(doc)
|
||||
if matches:
|
||||
# _, start, end = max(
|
||||
# matches,
|
||||
# key=lambda v: v[2] - v[1],
|
||||
# )
|
||||
# print(str(doc[start:end]))
|
||||
|
||||
if not inverse:
|
||||
results.append(original_sentence)
|
||||
elif inverse:
|
||||
results.append(original_sentence)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def _pattern_to_rule(pattern: str) -> List[Dict[str, Union[bool, str]]]:
|
||||
result: List[Dict[str, Union[bool, str]]] = []
|
||||
for t in pattern.split():
|
||||
if t == "*":
|
||||
result.extend([{"OP": "?"}, {"OP": "?"}])
|
||||
elif t == "CITATION":
|
||||
result.append({"ORTH": "CITATION"})
|
||||
elif t == "NUMBER":
|
||||
result.append({"ORTH": "NUMBER"})
|
||||
else:
|
||||
result.append({"LOWER": t})
|
||||
return result
|
||||
22
great_ai/src/great_ai/utilities/matcher/normalize.py
Normal file
22
great_ai/src/great_ai/utilities/matcher/normalize.py
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
import re
|
||||
|
||||
from ..clean import clean
|
||||
|
||||
|
||||
def normalize(text: str) -> str:
|
||||
text = re.sub(
|
||||
r"""
|
||||
([A-Z]\w+\W+(et\ al.)) # inline reference: Bank et al.
|
||||
| (\[[0-9-, ]+\]) # IEEE style: [1], [2-4], [3, 5]
|
||||
| (\(.*?,?\W+?\d+\)) # APA style: (Bank, 2020)
|
||||
| ([A-Z]\w+ \(?\d+\)) # APA style: Bank (2020)
|
||||
""",
|
||||
" CITATION ",
|
||||
text,
|
||||
flags=re.VERBOSE,
|
||||
)
|
||||
text = re.sub(r"\d[\d,. -]*(st|nd|rd|th)?", " NUMBER ", text)
|
||||
text = clean(text, convert_to_ascii=True)
|
||||
text = re.sub(r"[^a-zA-Z.?!,:;'\" -]", "", text)
|
||||
|
||||
return text
|
||||
|
|
@ -1,21 +1,21 @@
|
|||
try:
|
||||
import en_core_web_lg
|
||||
import en_core_web_sm
|
||||
except ImportError:
|
||||
import subprocess
|
||||
|
||||
print("Spacy model en_core_web_lg not found locally, downloading...")
|
||||
print("Spacy model en_core_web_sm not found locally, downloading...")
|
||||
|
||||
subprocess.call(
|
||||
[
|
||||
"pip",
|
||||
"install",
|
||||
"https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.2.0/en_core_web_lg-3.2.0-py3-none-any.whl",
|
||||
"https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl",
|
||||
]
|
||||
)
|
||||
import en_core_web_lg
|
||||
import en_core_web_sm
|
||||
|
||||
from .external.negspacy import negation # noqa: F401 it's important to import this
|
||||
|
||||
nlp = en_core_web_lg.load()
|
||||
nlp = en_core_web_sm.load()
|
||||
|
||||
nlp.add_pipe("negex")
|
||||
|
|
|
|||
|
|
@ -9,16 +9,13 @@ from .logger import create_logger
|
|||
|
||||
logger = create_logger("parallel_map")
|
||||
|
||||
|
||||
def parallel_map(
|
||||
function: Callable[[Any], Any],
|
||||
values: Iterable[Any],
|
||||
chunk_size: Optional[int] = None,
|
||||
concurrency: Optional[int] = None,
|
||||
concurrency: int = psutil.cpu_count(),
|
||||
disable_progress: bool = False,
|
||||
) -> List[Any]:
|
||||
if concurrency is None:
|
||||
concurrency = psutil.cpu_count()
|
||||
assert concurrency > 0
|
||||
assert chunk_size is None or chunk_size > 0
|
||||
|
||||
|
|
@ -30,7 +27,7 @@ def parallel_map(
|
|||
logger.info(
|
||||
f"Starting parallel map, concurrency: {concurrency}, chunk size: {chunk_size}"
|
||||
)
|
||||
|
||||
|
||||
if concurrency == 1 or len(values) <= chunk_size:
|
||||
logger.warning(f"Running in series, there is no reason for parallelism")
|
||||
iterable = values if disable_progress else tqdm(values)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
from .affiliation import Affiliation
|
||||
from .author import Author
|
||||
from .bookmark import Bookmark
|
||||
from .bookmark_title import BookmarkTitle
|
||||
from .element import Element, Meta, MetaType, Paragraph, Title
|
||||
from .publication_metadata import PublicationMetadata
|
||||
from .text import Text
|
||||
|
|
|
|||
|
|
@ -0,0 +1,10 @@
|
|||
from pydantic import BaseModel
|
||||
|
||||
from .bookmark_title import BookmarkTitle
|
||||
|
||||
|
||||
class Bookmark(BaseModel):
|
||||
title: BookmarkTitle
|
||||
original_title: str
|
||||
document_order: int
|
||||
coordinates: str
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
from typing import Literal
|
||||
|
||||
BookmarkTitle = Literal[
|
||||
"Abstract",
|
||||
"Author contribution",
|
||||
"Introduction",
|
||||
"Background",
|
||||
"Methods",
|
||||
"Results",
|
||||
"Discussion",
|
||||
"Outlook",
|
||||
"Conclusion",
|
||||
"Conflict of interest",
|
||||
"Acknowledgement",
|
||||
"Annex",
|
||||
]
|
||||
|
|
@ -1,7 +1,9 @@
|
|||
from typing import Optional
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class Text(BaseModel):
|
||||
content: str
|
||||
document_order: int
|
||||
coordinates: str
|
||||
coordinates: Optional[str]
|
||||
|
|
|
|||
|
|
@ -1,22 +1,43 @@
|
|||
from functools import cached_property
|
||||
from typing import Any, List, Optional, Union
|
||||
import os
|
||||
import re
|
||||
from functools import cached_property, lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Any, List, Optional, Pattern, Tuple, Union
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
from bs4.element import NavigableString, Tag
|
||||
|
||||
from great_ai.utilities.publication_tei.models.element import Paragraph
|
||||
from sus.publication_tei.models.element import Paragraph
|
||||
|
||||
from ..clean import clean
|
||||
from .models import Affiliation, Author, Element, Meta, PublicationMetadata, Text, Title
|
||||
from ..lemmatize_text import lemmatize_text
|
||||
from ..matcher import filter_sentences
|
||||
from ..unique import unique
|
||||
from .models import (
|
||||
Affiliation,
|
||||
Author,
|
||||
Bookmark,
|
||||
BookmarkTitle,
|
||||
Element,
|
||||
Meta,
|
||||
MetaType,
|
||||
PublicationMetadata,
|
||||
Text,
|
||||
Title,
|
||||
)
|
||||
from .titles_of_interest import titles_of_interest
|
||||
|
||||
THIS_FOLDER = Path(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
|
||||
class PublicationTEI:
|
||||
# remove template sentenaces, such as copyright notices
|
||||
agressive_cleaning_enabled = True
|
||||
|
||||
def __init__(self, tei: str):
|
||||
self._document_order_counter = 0
|
||||
|
||||
if tei:
|
||||
cleaned_xml = clean(tei, ignore_xml=True)
|
||||
self.soup = BeautifulSoup(cleaned_xml, "xml")
|
||||
self.soup = BeautifulSoup(tei, "xml")
|
||||
else:
|
||||
self.soup = BeautifulSoup()
|
||||
|
||||
|
|
@ -79,6 +100,132 @@ class PublicationTEI:
|
|||
for sentence in element.sentences
|
||||
]
|
||||
|
||||
@cached_property
|
||||
def bookmarks(self) -> List[Bookmark]:
|
||||
candidates: List[Bookmark] = [
|
||||
*self._find_matching_meta("Abstract", "abstract_start")[0],
|
||||
*self._find_matching_title("Abstract"),
|
||||
*self._find_matching_title("Author contribution"),
|
||||
*self._find_matching_meta("Acknowledgement", "acknowledgements_start")[0],
|
||||
*self._find_matching_title("Acknowledgement"),
|
||||
*self._find_matching_title("Conflict of interest"),
|
||||
]
|
||||
|
||||
_, start = self._find_matching_meta(None, "abstract_end")
|
||||
|
||||
candidates += [
|
||||
*self._find_matching_title("Background", start),
|
||||
*self._find_matching_title("Methods", start),
|
||||
*self._find_matching_title("Results", start),
|
||||
*self._find_matching_title("Discussion", start),
|
||||
*self._find_matching_title("Introduction", start),
|
||||
*self._find_matching_title("Conclusion", start),
|
||||
*self._find_matching_title("Outlook", start),
|
||||
*self._find_matching_meta("Annex", "annex_start", start)[0],
|
||||
]
|
||||
|
||||
candidates = sorted(candidates, key=lambda c: c.document_order)
|
||||
candidates = unique(candidates, key=lambda c: c.document_order)
|
||||
candidates = unique(candidates, key=lambda c: c.title)
|
||||
|
||||
return candidates
|
||||
|
||||
@cached_property
|
||||
def abstract_sentences(self) -> List[Text]:
|
||||
try:
|
||||
abstract_start = next(
|
||||
i
|
||||
for i, m in enumerate(self.content)
|
||||
if isinstance(m, Meta) and m.meta_type == "abstract_start"
|
||||
)
|
||||
abstract_end = next(
|
||||
i
|
||||
for i, m in enumerate(self.content)
|
||||
if isinstance(m, Meta) and m.meta_type == "abstract_end"
|
||||
)
|
||||
return [
|
||||
s
|
||||
for p in self.content[abstract_start:abstract_end]
|
||||
if isinstance(p, Paragraph)
|
||||
for s in p.sentences
|
||||
]
|
||||
except StopIteration:
|
||||
pass # let's try another way
|
||||
|
||||
try:
|
||||
abstract_start = next(
|
||||
m.document_order for m in self.bookmarks if m.title == "Abstract"
|
||||
)
|
||||
abstract_sentences: List[Text] = []
|
||||
for c in self.content:
|
||||
if isinstance(c, Paragraph):
|
||||
abstract_sentences.extend(
|
||||
s for s in c.sentences if s.document_order > abstract_start
|
||||
)
|
||||
elif len(abstract_sentences) >= 5:
|
||||
break
|
||||
return abstract_sentences
|
||||
except StopIteration:
|
||||
pass # let's try another way
|
||||
|
||||
return self.sentences[:10]
|
||||
|
||||
@cached_property
|
||||
def introduction_sentences(self) -> List[Text]:
|
||||
"""Includes abstract"""
|
||||
introduction_end = 4
|
||||
|
||||
try:
|
||||
introduction_start = [
|
||||
m.document_order for m in self.bookmarks if m.title == "Introduction"
|
||||
][-1]
|
||||
|
||||
for m in self.bookmarks:
|
||||
if m.title != "Introduction" and m.document_order > introduction_start:
|
||||
introduction_end = m.document_order
|
||||
break
|
||||
except IndexError:
|
||||
pass
|
||||
|
||||
try:
|
||||
introduction_end = max(
|
||||
next(
|
||||
i
|
||||
for i, m in enumerate(self.content)
|
||||
if isinstance(m, Meta) and m.meta_type == "abstract_end"
|
||||
),
|
||||
introduction_end,
|
||||
)
|
||||
except StopIteration:
|
||||
pass
|
||||
|
||||
introduction_sentences: List[Text] = []
|
||||
for c in self.content:
|
||||
if isinstance(c, Paragraph):
|
||||
introduction_sentences.extend(
|
||||
s for s in c.sentences if s.document_order < introduction_end
|
||||
)
|
||||
|
||||
return introduction_sentences
|
||||
|
||||
@cached_property
|
||||
def conclusion_sentences(self) -> List[Text]:
|
||||
try:
|
||||
conclusion_start = next(
|
||||
m.document_order for m in self.bookmarks if m.title == "Conclusion"
|
||||
)
|
||||
conclusion_sentences: List[Text] = []
|
||||
for c in self.content:
|
||||
if isinstance(c, Paragraph):
|
||||
conclusion_sentences.extend(
|
||||
s for s in c.sentences if s.document_order > conclusion_start
|
||||
)
|
||||
elif len(conclusion_sentences) >= 8:
|
||||
break
|
||||
return conclusion_sentences
|
||||
except StopIteration:
|
||||
return self.sentences[-10:]
|
||||
|
||||
def _parse_author(self, raw: Tag) -> Author:
|
||||
return Author(
|
||||
name=(
|
||||
|
|
@ -133,7 +280,9 @@ class PublicationTEI:
|
|||
results.append(Meta(meta_type="annex_start"))
|
||||
results.extend(self._get_primitives(r))
|
||||
results.append(Meta(meta_type="annex_end"))
|
||||
elif not r.find_parents(["abstract", "div"]):
|
||||
elif not r.find_parents(
|
||||
["abstract", "div", "figure"]
|
||||
): # figures are omitted as well
|
||||
results.extend(self._get_primitives(r))
|
||||
|
||||
return results
|
||||
|
|
@ -143,14 +292,20 @@ class PublicationTEI:
|
|||
|
||||
for r in raw.find_all(["head", "p"]):
|
||||
if r.name == "head" and r.get("coords") and r.get_text():
|
||||
results.append(Title(text=self._parse_text(r)))
|
||||
text = self._parse_text(r)
|
||||
if text:
|
||||
results.append(Title(text=text))
|
||||
elif r.name == "p" and r.find_all("s"):
|
||||
results.append(
|
||||
Paragraph(
|
||||
sentences=[
|
||||
self._parse_text(sentence)
|
||||
for sentence in r.find_all("s")
|
||||
if sentence.get_text()
|
||||
t
|
||||
for t in [
|
||||
self._parse_text(sentence, ignore_partial=True)
|
||||
for sentence in r.find_all("s")
|
||||
if sentence.get_text()
|
||||
]
|
||||
if t
|
||||
]
|
||||
)
|
||||
)
|
||||
|
|
@ -164,9 +319,25 @@ class PublicationTEI:
|
|||
clean(element.get_text(separator=" ", strip=True)) if element else default
|
||||
)
|
||||
|
||||
def _parse_text(self, raw: Tag) -> Text:
|
||||
def _parse_text(self, raw: Tag, ignore_partial: bool = False) -> Optional[Text]:
|
||||
text = raw.get_text()
|
||||
if text is None:
|
||||
return None
|
||||
|
||||
if self.agressive_cleaning_enabled:
|
||||
filtered = filter_sentences(
|
||||
text,
|
||||
THIS_FOLDER / "templates.yaml",
|
||||
inverse=True,
|
||||
ignore_partial=ignore_partial,
|
||||
)
|
||||
text = " ".join(filtered)
|
||||
|
||||
if not text.strip():
|
||||
return None
|
||||
|
||||
return Text(
|
||||
content=clean(raw.get_text()),
|
||||
content=text,
|
||||
document_order=self._generate_document_order_id(),
|
||||
coordinates=raw.get("coords"),
|
||||
)
|
||||
|
|
@ -175,3 +346,78 @@ class PublicationTEI:
|
|||
value = self._document_order_counter
|
||||
self._document_order_counter += 1
|
||||
return value
|
||||
|
||||
def _find_matching_meta(
|
||||
self,
|
||||
bookmark_title: Optional[BookmarkTitle],
|
||||
meta_type: MetaType,
|
||||
start: int = 0,
|
||||
) -> Tuple[List[Bookmark], int]:
|
||||
for i, e in enumerate(self.content[start:], start=start):
|
||||
if not isinstance(e, Meta):
|
||||
continue
|
||||
|
||||
if e.meta_type == meta_type:
|
||||
for e_next in self.content[i + 1 :]:
|
||||
if isinstance(e_next, Title):
|
||||
return [
|
||||
Bookmark(
|
||||
title=bookmark_title,
|
||||
original_title=e_next.text.content,
|
||||
document_order=e_next.text.document_order,
|
||||
coordinates=e_next.text.coordinates,
|
||||
)
|
||||
] if bookmark_title else [], i
|
||||
if isinstance(e_next, Paragraph) and e_next.sentences:
|
||||
return [
|
||||
Bookmark(
|
||||
title=bookmark_title,
|
||||
original_title="",
|
||||
document_order=e_next.sentences[0].document_order,
|
||||
coordinates=e_next.sentences[0].coordinates,
|
||||
)
|
||||
] if bookmark_title else [], i
|
||||
|
||||
return [], 0
|
||||
|
||||
def _find_matching_title(
|
||||
self,
|
||||
bookmark_title: BookmarkTitle,
|
||||
start: int = 0,
|
||||
) -> List[Bookmark]:
|
||||
return [
|
||||
Bookmark(
|
||||
title=bookmark_title,
|
||||
original_title=e.text.content,
|
||||
document_order=e.text.document_order,
|
||||
coordinates=e.text.coordinates,
|
||||
)
|
||||
for e in self.content[start:]
|
||||
if isinstance(e, Title)
|
||||
and self._match_title(e.text.content, titles_of_interest[bookmark_title])
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _match_title(title: str, keywords: Tuple[Union[Pattern, str], ...]) -> bool:
|
||||
title = PublicationTEI._process_section_title(title)
|
||||
|
||||
if any(
|
||||
PublicationTEI._process_section_title(k) in title
|
||||
for k in keywords
|
||||
if isinstance(k, str)
|
||||
):
|
||||
return True
|
||||
|
||||
return any(k.match(title) for k in keywords if isinstance(k, Pattern))
|
||||
|
||||
@staticmethod
|
||||
@lru_cache(maxsize=2000)
|
||||
def _process_section_title(title: str) -> str:
|
||||
title = re.sub(r"^\s*[ivx]+[.,)]? ", "", title) # Remove leading Roman-numerals
|
||||
title = clean(title, convert_to_ascii=True)
|
||||
title = re.sub(
|
||||
r"[^a-zA-Z ]", "", title
|
||||
) # Remove everything but letters and spaces (hypens are also removed)
|
||||
title_tokens = lemmatize_text(title)
|
||||
title = " ".join(t for t in title_tokens if t.strip())
|
||||
return title
|
||||
|
|
|
|||
837
great_ai/src/great_ai/utilities/publication_tei/templates.yaml
Normal file
837
great_ai/src/great_ai/utilities/publication_tei/templates.yaml
Normal file
|
|
@ -0,0 +1,837 @@
|
|||
included in the article's creative commons: 3827
|
||||
is a open access article distributed: 3822
|
||||
a open access article distributed under: 3761
|
||||
open access article distributed under the: 3761
|
||||
access article distributed under the terms: 3761
|
||||
the terms and conditions of the: 3688
|
||||
this article is a open access: 3678
|
||||
article is a open access article: 3678
|
||||
article distributed under the terms and: 3678
|
||||
distributed under the terms and conditions: 3678
|
||||
under the terms and conditions of: 3678
|
||||
terms and conditions of the creative: 3678
|
||||
and conditions of the creative commons: 3678
|
||||
conditions of the creative commons attribution: 3678
|
||||
in the article's creative commons licence: 3503
|
||||
authors declare that they have no: 3315
|
||||
https :// doi.org / NUMBER /: 3189
|
||||
the authors declare that they have: 3015
|
||||
remains neutral with regard to jurisdictional: 2464
|
||||
neutral with regard to jurisdictional claims: 2464
|
||||
with regard to jurisdictional claims in: 2464
|
||||
regard to jurisdictional claims in published: 2462
|
||||
to jurisdictional claims in published maps: 2462
|
||||
the original author ( s ): 2461
|
||||
jurisdictional claims in published maps and: 2461
|
||||
claims in published maps and institutional: 2461
|
||||
author ( s ) and the: 2460
|
||||
original author ( s ) and: 2456
|
||||
in published maps and institutional affiliations: 2415
|
||||
nature remains neutral with regard to: 2399
|
||||
springer nature remains neutral with regard: 2392
|
||||
( s ) and the source: 2353
|
||||
of the creative commons attribution CITATION: 2302
|
||||
", provide a link to the": 2299
|
||||
the source , provide a link: 2293
|
||||
source , provide a link to: 2293
|
||||
you give appropriate credit to the: 2292
|
||||
give appropriate credit to the original: 2292
|
||||
appropriate credit to the original author: 2292
|
||||
and indicate if changes were made: 2289
|
||||
credit to the original author (: 2281
|
||||
to the original author ( s: 2281
|
||||
provide a link to the creative: 2280
|
||||
a link to the creative commons: 2280
|
||||
s ) and the source ,: 2279
|
||||
) and the source , provide: 2277
|
||||
and the source , provide a: 2277
|
||||
", and indicate if changes were": 2271
|
||||
at https :// doi.org / NUMBER: 2195
|
||||
creative commons attribution NUMBER international license: 2158
|
||||
", distribution and reproduction in any": 2158
|
||||
distribution and reproduction in any medium: 2157
|
||||
can be found online at https: 2106
|
||||
be found online at https ://: 2106
|
||||
article can be found online at: 2069
|
||||
license , which permits use ,: 2063
|
||||
this article can be found online: 2010
|
||||
online version contains supplementary material available: 1980
|
||||
supplementary material available at https ://: 1979
|
||||
version contains supplementary material available at: 1978
|
||||
contains supplementary material available at https: 1973
|
||||
the online version contains supplementary material: 1957
|
||||
note springer nature remains neutral with: 1956
|
||||
other third party material in this: 1945
|
||||
to view a copy of this: 1945
|
||||
or other third party material in: 1944
|
||||
need to obtain permission directly from: 1944
|
||||
to obtain permission directly from the: 1944
|
||||
images or other third party material: 1943
|
||||
regulation or exceeds the permitted use: 1943
|
||||
or exceeds the permitted use ,: 1943
|
||||
obtain permission directly from the copyright: 1943
|
||||
the permitted use , you will: 1942
|
||||
by statutory regulation or exceeds the: 1941
|
||||
statutory regulation or exceeds the permitted: 1941
|
||||
exceeds the permitted use , you: 1941
|
||||
use , you will need to: 1941
|
||||
", you will need to obtain": 1941
|
||||
you will need to obtain permission: 1941
|
||||
permitted use , you will need: 1940
|
||||
will need to obtain permission directly: 1940
|
||||
permitted by statutory regulation or exceeds: 1939
|
||||
publisher's note springer nature remains neutral: 1939
|
||||
and your intended use is not: 1938
|
||||
your intended use is not permitted: 1938
|
||||
not permitted by statutory regulation or: 1938
|
||||
intended use is not permitted by: 1937
|
||||
is not permitted by statutory regulation: 1937
|
||||
permission directly from the copyright holder: 1937
|
||||
use is not permitted by statutory: 1936
|
||||
material is not included in the: 1933
|
||||
the images or other third party: 1932
|
||||
if material is not included in: 1926
|
||||
in a credit line to the: 1924
|
||||
otherwise in a credit line to: 1923
|
||||
this article is included in the: 1922
|
||||
third party material in this article: 1921
|
||||
party material in this article is: 1921
|
||||
material in this article is included: 1921
|
||||
in this article is included in: 1921
|
||||
", unless indicated otherwise in a": 1920
|
||||
indicated otherwise in a credit line: 1920
|
||||
is included in the article's creative: 1919
|
||||
unless indicated otherwise in a credit: 1919
|
||||
article is included in the article's: 1918
|
||||
a credit line to the material: 1914
|
||||
not included in the article's creative: 1907
|
||||
is not included in the article's: 1906
|
||||
is licensed under a creative commons: 1898
|
||||
authors declare no conflict of interest: 1883
|
||||
and reproduction in any medium or: 1870
|
||||
reproduction in any medium or format: 1870
|
||||
in any medium or format ,: 1870
|
||||
any medium or format , as: 1870
|
||||
medium or format , as long: 1870
|
||||
or format , as long as: 1870
|
||||
format , as long as you: 1869
|
||||
", as long as you give": 1869
|
||||
as long as you give appropriate: 1869
|
||||
long as you give appropriate credit: 1869
|
||||
as you give appropriate credit to: 1869
|
||||
", adaptation , distribution and reproduction": 1866
|
||||
adaptation , distribution and reproduction in: 1866
|
||||
use , sharing , adaptation ,: 1861
|
||||
", sharing , adaptation , distribution": 1861
|
||||
sharing , adaptation , distribution and: 1861
|
||||
NUMBER international license , which permits: 1846
|
||||
article is licensed under a creative: 1839
|
||||
this article is licensed under a: 1838
|
||||
under a creative commons attribution NUMBER: 1817
|
||||
licensed under a creative commons attribution: 1811
|
||||
a creative commons attribution NUMBER international: 1807
|
||||
view a copy of this licence: 1799
|
||||
a copy of this licence ,: 1799
|
||||
copy of this licence , visit: 1799
|
||||
of this licence , visit http: 1796
|
||||
this licence , visit http ://: 1796
|
||||
access this article is licensed under: 1794
|
||||
the authors would like to thank: 1793
|
||||
which permits use , sharing ,: 1791
|
||||
permits use , sharing , adaptation: 1790
|
||||
", which permits use , sharing": 1789
|
||||
open access this article is licensed: 1782
|
||||
international license , which permits use: 1776
|
||||
attribution NUMBER international license , which: 1773
|
||||
commons attribution NUMBER international license ,: 1772
|
||||
creative commons licence and your intended: 1769
|
||||
commons licence and your intended use: 1768
|
||||
licence and your intended use is: 1768
|
||||
the article's creative commons licence and: 1762
|
||||
article's creative commons licence and your: 1762
|
||||
creative commons licence , unless indicated: 1750
|
||||
commons licence , unless indicated otherwise: 1749
|
||||
licence , unless indicated otherwise in: 1749
|
||||
article's creative commons licence , unless: 1743
|
||||
the article's creative commons licence ,: 1742
|
||||
the authors declare no conflict of: 1742
|
||||
to this article can be found: 1714
|
||||
licence , and indicate if changes: 1698
|
||||
link to the creative commons licence: 1696
|
||||
commons licence , and indicate if: 1695
|
||||
to the creative commons licence ,: 1694
|
||||
the creative commons licence , and: 1694
|
||||
creative commons licence , and indicate: 1694
|
||||
the authors declare no competing interests: 1564
|
||||
and / or publication of this: 1530
|
||||
/ or publication of this article: 1523
|
||||
the research , authorship , and: 1410
|
||||
data to this article can be: 1395
|
||||
supplementary data to this article can: 1392
|
||||
competing financial interests or personal relationships: 1388
|
||||
research , authorship , and /: 1385
|
||||
", authorship , and / or": 1385
|
||||
authorship , and / or publication: 1385
|
||||
is available from the corresponding author: 1385
|
||||
", and / or publication of": 1384
|
||||
no known competing financial interests or: 1382
|
||||
known competing financial interests or personal: 1382
|
||||
declare that they have no known: 1377
|
||||
that they have no known competing: 1376
|
||||
of the creative commons attribution (: 1372
|
||||
have no known competing financial interests: 1371
|
||||
financial interests or personal relationships that: 1371
|
||||
the creative commons attribution ( cc: 1370
|
||||
creative commons attribution ( cc by: 1370
|
||||
they have no known competing financial: 1370
|
||||
to influence the work reported in: 1369
|
||||
influence the work reported in this: 1369
|
||||
interests or personal relationships that could: 1368
|
||||
appeared to influence the work reported: 1368
|
||||
or personal relationships that could have: 1367
|
||||
have appeared to influence the work: 1367
|
||||
could have appeared to influence the: 1364
|
||||
relationships that could have appeared to: 1363
|
||||
that could have appeared to influence: 1363
|
||||
personal relationships that could have appeared: 1362
|
||||
is distributed under the terms of: 1265
|
||||
relationships that could be construed as: 1259
|
||||
/ licenses / by / NUMBER: 1243
|
||||
in the absence of any commercial: 1242
|
||||
the absence of any commercial or: 1241
|
||||
commercial or financial relationships that could: 1241
|
||||
or financial relationships that could be: 1240
|
||||
absence of any commercial or financial: 1239
|
||||
any commercial or financial relationships that: 1239
|
||||
as a potential conflict of interest: 1239
|
||||
was conducted in the absence of: 1238
|
||||
of any commercial or financial relationships: 1238
|
||||
conducted in the absence of any: 1237
|
||||
financial relationships that could be construed: 1236
|
||||
be construed as a potential conflict: 1236
|
||||
the research was conducted in the: 1235
|
||||
construed as a potential conflict of: 1235
|
||||
that the research was conducted in: 1233
|
||||
research was conducted in the absence: 1233
|
||||
:// doi.org / NUMBER / s: 1220
|
||||
doi.org / NUMBER / s NUMBER: 1220
|
||||
declare that the research was conducted: 1212
|
||||
authors declare that the research was: 1204
|
||||
read and approved the final manuscript: 1196
|
||||
study is available from the corresponding: 1163
|
||||
creativecommons.org / licenses / by /: 1148
|
||||
:// creativecommons.org / licenses / by: 1147
|
||||
material available at https :// doi: 1135
|
||||
authors read and approved the final: 1074
|
||||
http :// creativecommons.org / licenses /: 1063
|
||||
that they have no conflict of: 1061
|
||||
all authors read and approved the: 1055
|
||||
requests for materials should be addressed: 1054
|
||||
for materials should be addressed to: 1054
|
||||
correspondence and requests for materials should: 1053
|
||||
and requests for materials should be: 1052
|
||||
findings of this study is available: 1046
|
||||
they have no conflict of interest: 1044
|
||||
declare that they have no conflict: 1020
|
||||
", visit http :// creat iveco": 1016
|
||||
licence , visit http :// creat: 1015
|
||||
the authors declare that the research: 993
|
||||
", on the other hand ,": 989
|
||||
available at https :// doi.org /: 925
|
||||
informed consent was obtained from all: 904
|
||||
this article is protected by copyright: 882
|
||||
found online at https :// doi: 871
|
||||
additional supporting information may be found: 866
|
||||
and moral rights for the publications: 865
|
||||
moral rights for the publications made: 865
|
||||
rights for the publications made accessible: 865
|
||||
for the publications made accessible in: 865
|
||||
is retained by the authors and: 865
|
||||
retained by the authors and /: 865
|
||||
by the authors and / or: 865
|
||||
the authors and / or other: 865
|
||||
authors and / or other copyright: 865
|
||||
and / or other copyright owners: 865
|
||||
/ or other copyright owners and: 865
|
||||
or other copyright owners and it: 865
|
||||
other copyright owners and it is: 865
|
||||
copyright owners and it is a: 865
|
||||
owners and it is a condition: 865
|
||||
and it is a condition of: 865
|
||||
it is a condition of accessing: 865
|
||||
is a condition of accessing publications: 865
|
||||
a condition of accessing publications that: 865
|
||||
condition of accessing publications that users: 865
|
||||
of accessing publications that users recognise: 865
|
||||
accessing publications that users recognise and: 865
|
||||
publications that users recognise and abide: 865
|
||||
that users recognise and abide by: 865
|
||||
users recognise and abide by the: 865
|
||||
recognise and abide by the legal: 865
|
||||
and abide by the legal requirements: 865
|
||||
abide by the legal requirements associated: 865
|
||||
by the legal requirements associated with: 865
|
||||
the legal requirements associated with this: 865
|
||||
copyright and moral rights for the: 864
|
||||
the publications made accessible in the: 864
|
||||
publications made accessible in the public: 864
|
||||
made accessible in the public portal: 864
|
||||
accessible in the public portal is: 864
|
||||
in the public portal is retained: 864
|
||||
the public portal is retained by: 864
|
||||
public portal is retained by the: 864
|
||||
portal is retained by the authors: 864
|
||||
legal requirements associated with this rights: 863
|
||||
the terms of the creative commons: 839
|
||||
and reproduction in any medium ,: 834
|
||||
reproduction in any medium , provided: 833
|
||||
declare that they have no competing: 823
|
||||
conflicts of interest with respect to: 811
|
||||
this work was supported by the: 804
|
||||
that they have no competing interests: 783
|
||||
respect to the research , authorship: 781
|
||||
with respect to the research ,: 780
|
||||
potential conflicts of interest with respect: 779
|
||||
interest with respect to the research: 776
|
||||
of article NUMBER fa of the: 775
|
||||
article NUMBER fa of the dutch: 775
|
||||
NUMBER fa of the dutch copyright: 775
|
||||
fa of the dutch copyright act: 775
|
||||
potential conflict of interest was reported: 775
|
||||
conflict of interest was reported by: 775
|
||||
no potential conflict of interest was: 773
|
||||
of interest was reported by the: 770
|
||||
in the present study , we: 767
|
||||
under the terms of article NUMBER: 766
|
||||
the terms of article NUMBER fa: 766
|
||||
terms of article NUMBER fa of: 766
|
||||
this study is available from the: 765
|
||||
support for the research , authorship: 762
|
||||
financial support for the research ,: 760
|
||||
of the dutch copyright act ,: 758
|
||||
distributed under the terms of the: 743
|
||||
publication is distributed under the terms: 742
|
||||
distributed under the terms of article: 742
|
||||
the original work is properly cited: 738
|
||||
if the publication is distributed under: 734
|
||||
the publication is distributed under the: 734
|
||||
the dutch copyright act , indicated: 734
|
||||
dutch copyright act , indicated by: 734
|
||||
copyright act , indicated by the: 734
|
||||
act , indicated by the taverne: 734
|
||||
", indicated by the taverne license": 734
|
||||
indicated by the taverne license above: 734
|
||||
by the taverne license above ,: 734
|
||||
the taverne license above , please: 734
|
||||
taverne license above , please follow: 734
|
||||
license above , please follow below: 734
|
||||
above , please follow below link: 734
|
||||
", please follow below link for": 734
|
||||
please follow below link for the: 734
|
||||
follow below link for the end: 734
|
||||
online at https :// doi.org /: 733
|
||||
of this study is available from: 731
|
||||
to the research , authorship ,: 728
|
||||
from NUMBER % to NUMBER %: 728
|
||||
CITATION , NUMBER CITATION , NUMBER: 723
|
||||
reprints and permission information is available: 722
|
||||
information is available at http ://: 722
|
||||
available at http :// www.nature.com /: 722
|
||||
is available at http :// www.nature.com: 721
|
||||
at http :// www.nature.com / reprints: 721
|
||||
terms of the creative commons attribution: 717
|
||||
and permission information is available at: 710
|
||||
permission information is available at http: 710
|
||||
provided the original work is properly: 707
|
||||
", provided the original work is": 706
|
||||
http :// www.nature.com / reprints publisher's: 706
|
||||
:// www.nature.com / reprints publisher's note: 705
|
||||
www.nature.com / reprints publisher's note springer: 705
|
||||
/ reprints publisher's note springer nature: 705
|
||||
reprints publisher's note springer nature remains: 705
|
||||
associated with this article can be: 700
|
||||
with this article can be found: 695
|
||||
available from the corresponding author upon: 691
|
||||
no potential conflicts of interest with: 685
|
||||
for the research , authorship ,: 685
|
||||
", visit http :// creativecommons.org /": 683
|
||||
visit http :// creativecommons.org / licenses: 683
|
||||
conflict of interest the authors declare: 678
|
||||
declared no potential conflicts of interest: 666
|
||||
visit http :// creat iveco mmons: 665
|
||||
for this article can be found: 658
|
||||
had no role in the design: 651
|
||||
material for this article can be: 645
|
||||
this is a open access article: 644
|
||||
the corresponding author upon reasonable request: 637
|
||||
supplementary material for this article can: 633
|
||||
from the corresponding author upon reasonable: 633
|
||||
the final version of the manuscript: 624
|
||||
the author ( s ) declared: 618
|
||||
the supplementary material for this article: 617
|
||||
in the public , commercial ,: 613
|
||||
the public , commercial , or: 612
|
||||
by the author ( s ): 599
|
||||
of interest the authors declare that: 594
|
||||
available from the corresponding author on: 589
|
||||
this article can be found ,: 587
|
||||
article can be found , in: 587
|
||||
link to the creative commons license: 585
|
||||
be found in the online version: 584
|
||||
", the aim of this study": 583
|
||||
be found online in the supporting: 583
|
||||
found online in the supporting information: 583
|
||||
", it has been shown that": 582
|
||||
we would like to thank the: 580
|
||||
supporting information may be found online: 579
|
||||
information may be found online in: 577
|
||||
may be found online in the: 577
|
||||
be found , in the online: 577
|
||||
found , in the online version: 577
|
||||
license , and indicate if changes: 576
|
||||
", in the online version ,": 574
|
||||
in the online version , at: 574
|
||||
for their contribution to the peer: 574
|
||||
their contribution to the peer review: 574
|
||||
this research did not receive any: 572
|
||||
reprints and permissions information is available: 572
|
||||
to the creative commons license ,: 571
|
||||
the creative commons license , and: 571
|
||||
creative commons license , and indicate: 571
|
||||
commons license , and indicate if: 571
|
||||
the funders had no role in: 570
|
||||
and permissions information is available at: 570
|
||||
did not receive any specific grant: 566
|
||||
agencies in the public , commercial: 566
|
||||
from the corresponding author on reasonable: 564
|
||||
in any medium , provided the: 563
|
||||
any medium , provided the original: 562
|
||||
to the peer review of this: 562
|
||||
the peer review of this work: 562
|
||||
not receive any specific grant from: 561
|
||||
from funding agencies in the public: 561
|
||||
the corresponding author on reasonable request: 561
|
||||
is available at www.nature.com / reprints: 559
|
||||
funding agencies in the public ,: 558
|
||||
permissions information is available at www.nature.com: 558
|
||||
information is available at www.nature.com /: 558
|
||||
research did not receive any specific: 552
|
||||
/ NUMBER / s NUMBER correspondence: 552
|
||||
NUMBER correspondence and requests for materials: 552
|
||||
below link for the end user: 551
|
||||
NUMBER / s NUMBER correspondence and: 551
|
||||
/ s NUMBER correspondence and requests: 551
|
||||
s NUMBER correspondence and requests for: 551
|
||||
online in the supporting information section: 550
|
||||
public , commercial , or not-for-profit: 549
|
||||
found online at https :// www.frontiersin.org: 549
|
||||
online at https :// www.frontiersin.org /: 549
|
||||
at https :// www.frontiersin.org / articles: 548
|
||||
https :// www.frontiersin.org / articles /: 548
|
||||
", distribution , and reproduction in": 546
|
||||
distribution , and reproduction in any: 546
|
||||
", and reproduction in any medium": 546
|
||||
:// www.frontiersin.org / articles / NUMBER: 545
|
||||
www.frontiersin.org / articles / NUMBER /: 544
|
||||
contribution to the peer review of: 542
|
||||
grant from funding agencies in the: 541
|
||||
use , distribution , and reproduction: 537
|
||||
", commercial , or not-for-profit sectors": 535
|
||||
received no financial support for the: 535
|
||||
no financial support for the research: 534
|
||||
receive any specific grant from funding: 529
|
||||
specific grant from funding agencies in: 529
|
||||
approved the final version of the: 529
|
||||
any specific grant from funding agencies: 528
|
||||
author ( s ) declared no: 526
|
||||
is available online at https ://: 525
|
||||
( s ) declared no potential: 525
|
||||
s ) declared no potential conflicts: 525
|
||||
) declared no potential conflicts of: 525
|
||||
article is distributed under the terms: 520
|
||||
this article is distributed under the: 518
|
||||
"% and NUMBER % , respectively": 507
|
||||
access this article is distributed under: 501
|
||||
open access this article is distributed: 498
|
||||
authors declare that there is no: 487
|
||||
the following is available online at: 485
|
||||
NUMBER % , NUMBER % ,: 484
|
||||
authors would like to thank the: 479
|
||||
is a open access article under: 477
|
||||
a open access article under the: 477
|
||||
medium , provided the original work: 473
|
||||
/ NUMBER /) , which permits: 471
|
||||
the online version , at doi: 466
|
||||
interest was reported by the author: 464
|
||||
supplementary material associated with this article: 452
|
||||
material associated with this article can: 452
|
||||
we would like to thank all: 449
|
||||
http :// creat iveco mmons.org /: 448
|
||||
:// creat iveco mmons.org / licen: 446
|
||||
creat iveco mmons.org / licen ses: 446
|
||||
iveco mmons.org / licen ses /: 446
|
||||
following is available online at https: 446
|
||||
available online at https :// www.mdpi.com: 445
|
||||
online at https :// www.mdpi.com /: 445
|
||||
union's horizon NUMBER research and innovation: 443
|
||||
during the current study is available: 443
|
||||
european union's horizon NUMBER research and: 441
|
||||
the netherlands organization for scientific research: 441
|
||||
the authors declare that there is: 441
|
||||
NUMBER international license ( http ://: 440
|
||||
of the authors and do not: 439
|
||||
www.mdpi.com / article / NUMBER /: 438
|
||||
NUMBER CITATION , NUMBER CITATION ,: 436
|
||||
access article under the terms of: 435
|
||||
article under the terms of the: 435
|
||||
/ by / NUMBER /) ,: 435
|
||||
by / NUMBER /) , which: 435
|
||||
/ licen ses / by /: 434
|
||||
licen ses / by / NUMBER: 434
|
||||
open access article under the terms: 434
|
||||
current study is available from the: 434
|
||||
the current study is available from: 433
|
||||
", which permits unrestricted use ,": 431
|
||||
mmons.org / licen ses / by: 429
|
||||
have no conflicts of interest to: 427
|
||||
which permits unrestricted use , distribution: 427
|
||||
", there is a need for": 425
|
||||
permits unrestricted use , distribution ,: 425
|
||||
unrestricted use , distribution , and: 425
|
||||
have read and agreed to the: 423
|
||||
authors have read and agreed to: 422
|
||||
any medium , provided you give: 422
|
||||
medium , provided you give appropriate: 422
|
||||
", provided you give appropriate credit": 422
|
||||
provided you give appropriate credit to: 422
|
||||
was reported by the author (: 422
|
||||
reported by the author ( s: 422
|
||||
NUMBER /) , which permits unrestricted: 421
|
||||
in any medium , provided you: 421
|
||||
peer review information nature communications thanks: 421
|
||||
the european union's horizon NUMBER research: 420
|
||||
and agreed to the published version: 419
|
||||
agreed to the published version of: 419
|
||||
the published version of the manuscript: 419
|
||||
to the published version of the: 418
|
||||
the authors and do not necessarily: 417
|
||||
/) , which permits unrestricted use: 417
|
||||
read and agreed to the published: 416
|
||||
those of the authors and do: 414
|
||||
all authors have read and agreed: 413
|
||||
that there is no conflict of: 411
|
||||
attribution NUMBER international license ( http: 410
|
||||
author ( s ) received no: 405
|
||||
consent was obtained from all subjects: 404
|
||||
authors have no conflicts of interest: 403
|
||||
( s ) received no financial: 403
|
||||
s ) received no financial support: 403
|
||||
) received no financial support for: 403
|
||||
NUMBER % and NUMBER % of: 400
|
||||
contributed to the article and approved: 400
|
||||
to the article and approved the: 400
|
||||
authors contributed to the article and: 397
|
||||
the author ( s ) received: 396
|
||||
the article and approved the submitted: 396
|
||||
article and approved the submitted version: 395
|
||||
informed consent statement informed consent was: 393
|
||||
accountable for all aspects of the: 389
|
||||
https :// www.mdpi.com / article /: 389
|
||||
:// www.mdpi.com / article / NUMBER: 389
|
||||
consent statement informed consent was obtained: 389
|
||||
at https :// www.mdpi.com / article: 388
|
||||
statement informed consent was obtained from: 388
|
||||
no role in the design of: 385
|
||||
) for their contribution to the: 383
|
||||
", we would like to thank": 381
|
||||
is available on request from the: 378
|
||||
had no role in study design: 377
|
||||
responsibility for the integrity of the: 375
|
||||
of the creative commons attribution NUMBER: 375
|
||||
in the supporting information section at: 374
|
||||
the supporting information section at the: 374
|
||||
supporting information section at the end: 374
|
||||
reviewer ( s ) for their: 374
|
||||
this study is available on request: 373
|
||||
all authors contributed to the article: 373
|
||||
( s ) for their contribution: 372
|
||||
s ) for their contribution to: 372
|
||||
", analysis and interpretation of data": 371
|
||||
available on request from the corresponding: 370
|
||||
in accordance with the ethical standards: 370
|
||||
the manuscript for important intellectual content: 368
|
||||
for all aspects of the work: 367
|
||||
role in the design of the: 367
|
||||
authors declare no conflicts of interest: 366
|
||||
", analysis , and interpretation of": 365
|
||||
on request from the corresponding author: 364
|
||||
be accountable for all aspects of: 362
|
||||
obtained from all subjects involved in: 361
|
||||
funders had no role in the: 360
|
||||
for scientific research ( nwo ): 359
|
||||
the creative commons attribution NUMBER international: 359
|
||||
was obtained from all subjects involved: 359
|
||||
authors and do not necessarily represent: 358
|
||||
to be accountable for all aspects: 356
|
||||
original work is properly cited ,: 355
|
||||
data in the writing of the: 354
|
||||
no role in study design ,: 353
|
||||
from all subjects involved in the: 353
|
||||
all subjects involved in the study: 351
|
||||
provenance and peer review not commissioned: 346
|
||||
study is available on request from: 346
|
||||
there is no conflict of interest: 346
|
||||
wrote the first draft of the: 345
|
||||
NUMBER / NUMBER / NUMBER /: 345
|
||||
written informed consent to participate in: 341
|
||||
commons attribution NUMBER international license (: 341
|
||||
interpretation of data in the writing: 339
|
||||
of data in the writing of: 339
|
||||
ses / by / NUMBER /: 337
|
||||
we would also like to thank: 336
|
||||
the integrity of the data and: 335
|
||||
and peer review not commissioned externally: 334
|
||||
of this study was to assess: 334
|
||||
peer review not commissioned externally peer: 332
|
||||
review not commissioned externally peer reviewed: 332
|
||||
declare that there is no conflict: 331
|
||||
role in study design , data: 328
|
||||
written informed consent was obtained from: 327
|
||||
no conflicts of interest to declare: 326
|
||||
consent to participate in this study: 326
|
||||
in the article's creative commons license: 326
|
||||
do not necessarily represent those of: 325
|
||||
( http :// creativecommons.org / licenses: 324
|
||||
the authors have no conflicts of: 322
|
||||
informed consent to participate in this: 322
|
||||
with the ethical standards of the: 322
|
||||
license ( http :// creativecommons.org /: 322
|
||||
solely those of the authors and: 322
|
||||
and do not necessarily represent those: 322
|
||||
in this article is solely those: 320
|
||||
this article is solely those of: 320
|
||||
article is solely those of the: 320
|
||||
integrity of the data and the: 319
|
||||
of the data and the accuracy: 319
|
||||
the data and the accuracy of: 319
|
||||
accordance with the ethical standards of: 319
|
||||
data and the accuracy of the: 318
|
||||
during the conduct of the study: 317
|
||||
the creative commons attribution CITATION ,: 317
|
||||
expressed in this article is solely: 316
|
||||
and approved the final version of: 315
|
||||
publisher's note all claims expressed in: 315
|
||||
note all claims expressed in this: 315
|
||||
all claims expressed in this article: 315
|
||||
claims expressed in this article is: 315
|
||||
not necessarily represent those of their: 315
|
||||
necessarily represent those of their affiliated: 315
|
||||
represent those of their affiliated organizations: 315
|
||||
or those of the publisher ,: 315
|
||||
those of the publisher , the: 314
|
||||
of the publisher , the editors: 314
|
||||
in the writing of the manuscript: 313
|
||||
affiliated organizations , or those of: 312
|
||||
organizations , or those of the: 312
|
||||
or claim that may be made: 312
|
||||
claim that may be made by: 312
|
||||
that may be made by its: 312
|
||||
is not guaranteed or endorsed by: 312
|
||||
not guaranteed or endorsed by the: 312
|
||||
guaranteed or endorsed by the publisher: 312
|
||||
authors would like to thank all: 311
|
||||
related to this article can be: 311
|
||||
those of their affiliated organizations ,: 311
|
||||
of their affiliated organizations , or: 311
|
||||
their affiliated organizations , or those: 311
|
||||
", or those of the publisher": 311
|
||||
any product that may be evaluated: 311
|
||||
this article , or claim that: 311
|
||||
article , or claim that may: 311
|
||||
", or claim that may be": 311
|
||||
product that may be evaluated in: 310
|
||||
may be made by its manufacturer: 310
|
||||
the accuracy of the data analysis: 309
|
||||
", there is a need to": 309
|
||||
of the creative commons attribution license: 309
|
||||
that may be evaluated in this: 309
|
||||
may be evaluated in this article: 309
|
||||
be made by its manufacturer ,: 309
|
||||
made by its manufacturer , is: 309
|
||||
by its manufacturer , is not: 309
|
||||
its manufacturer , is not guaranteed: 309
|
||||
manufacturer , is not guaranteed or: 309
|
||||
", is not guaranteed or endorsed": 309
|
||||
", there is a lack of": 308
|
||||
the authors declare no conflicts of: 308
|
||||
be evaluated in this article ,: 308
|
||||
evaluated in this article , or: 308
|
||||
in this article , or claim: 308
|
||||
the data in the study and: 306
|
||||
in this paper , we present: 306
|
||||
information may be found in the: 305
|
||||
this study was to investigate the: 305
|
||||
", on the one hand ,": 305
|
||||
may be found in the online: 304
|
||||
is not publicly available due to: 304
|
||||
authors report no conflicts of interest: 304
|
||||
online version , at doi NUMBER: 303
|
||||
version , at doi NUMBER /: 303
|
||||
supporting information may be found in: 302
|
||||
in the public , commercial or: 301
|
||||
from any funding agency in the: 300
|
||||
any funding agency in the public: 300
|
||||
were in accordance with the ethical: 300
|
||||
the publisher , the editors and: 300
|
||||
publisher , the editors and the: 300
|
||||
", the editors and the reviewers": 300
|
||||
funding agency in the public ,: 299
|
||||
agency in the public , commercial: 299
|
||||
found in the online version of: 297
|
||||
there is no conflicts of interest: 295
|
||||
can be found in the online: 295
|
||||
article can be found in the: 291
|
||||
use , distribution and reproduction in: 289
|
||||
the online version of this article: 287
|
||||
before it is published in its: 287
|
||||
it is published in its final: 287
|
||||
to submit the manuscript for publication: 286
|
||||
international license ( http :// creativecommons.org: 285
|
||||
is published in its final form: 285
|
||||
decision to submit the manuscript for: 284
|
||||
", which permits use , distribution": 284
|
||||
which permits use , distribution and: 284
|
||||
permits use , distribution and reproduction: 284
|
||||
", as shown in fig. NUMBER": 284
|
||||
errors may be discovered which could: 284
|
||||
may be discovered which could affect: 284
|
||||
be discovered which could affect the: 284
|
||||
discovered which could affect the content: 284
|
||||
which could affect the content ,: 284
|
||||
could affect the content , and: 284
|
||||
affect the content , and all: 284
|
||||
the content , and all legal: 284
|
||||
content , and all legal disclaimers: 284
|
||||
all legal disclaimers that apply to: 283
|
||||
legal disclaimers that apply to the: 283
|
||||
disclaimers that apply to the journal: 283
|
||||
design , data collection and analysis: 282
|
||||
", and all legal disclaimers that": 282
|
||||
and all legal disclaimers that apply: 282
|
||||
from the european union's horizon NUMBER: 280
|
||||
and the other , anonymous ,: 280
|
||||
is a pdf file of a: 278
|
||||
this article can be found in: 277
|
||||
helsinki declaration and its later amendments: 276
|
||||
NUMBER helsinki declaration and its later: 275
|
||||
the NUMBER helsinki declaration and its: 274
|
||||
during the production process , errors: 274
|
||||
the production process , errors may: 274
|
||||
production process , errors may be: 274
|
||||
process , errors may be discovered: 274
|
||||
this is a pdf file of: 273
|
||||
review before it is published in: 273
|
||||
", errors may be discovered which": 273
|
||||
will undergo additional copyediting , typesetting: 271
|
||||
and review before it is published: 271
|
||||
note that , during the production: 271
|
||||
that , during the production process: 271
|
||||
", during the production process ,": 271
|
||||
with the NUMBER helsinki declaration and: 270
|
||||
this version will undergo additional copyediting: 270
|
||||
version will undergo additional copyediting ,: 270
|
||||
undergo additional copyediting , typesetting and: 270
|
||||
additional copyediting , typesetting and review: 270
|
||||
copyediting , typesetting and review before: 270
|
||||
", typesetting and review before it": 270
|
||||
typesetting and review before it is: 270
|
||||
published in its final form ,: 270
|
||||
in its final form , but: 270
|
||||
its final form , but we: 270
|
||||
final form , but we is: 270
|
||||
form , but we is providing: 270
|
||||
", but we is providing this": 270
|
||||
but we is providing this version: 270
|
||||
we is providing this version to: 270
|
||||
is providing this version to give: 270
|
||||
providing this version to give early: 270
|
||||
this version to give early visibility: 270
|
||||
version to give early visibility of: 270
|
||||
to give early visibility of the: 270
|
||||
give early visibility of the article: 270
|
||||
please note that , during the: 270
|
||||
netherlands organization for scientific research (: 269
|
||||
acute respiratory syndrome coronavirus NUMBER CITATION: 268
|
||||
a pdf file of a article: 268
|
||||
pdf file of a article that: 268
|
||||
file of a article that has: 268
|
||||
of a article that has undergone: 268
|
||||
a article that has undergone enhancements: 268
|
||||
article that has undergone enhancements after: 268
|
||||
that has undergone enhancements after acceptance: 268
|
||||
has undergone enhancements after acceptance ,: 268
|
||||
undergone enhancements after acceptance , such: 268
|
||||
enhancements after acceptance , such as: 268
|
||||
after acceptance , such as the: 268
|
||||
acceptance , such as the addition: 268
|
||||
as the addition of a cover: 268
|
||||
the addition of a cover page: 268
|
||||
addition of a cover page and: 268
|
||||
of a cover page and metadata: 268
|
||||
a cover page and metadata ,: 268
|
||||
cover page and metadata , and: 268
|
||||
page and metadata , and formatting: 268
|
||||
and metadata , and formatting for: 268
|
||||
metadata , and formatting for readability: 268
|
||||
", and formatting for readability ,": 268
|
||||
and formatting for readability , but: 268
|
||||
formatting for readability , but it: 268
|
||||
for readability , but it is: 268
|
||||
readability , but it is not: 268
|
||||
but it is not yet the: 268
|
||||
it is not yet the definitive: 268
|
||||
is not yet the definitive version: 268
|
||||
not yet the definitive version of: 268
|
||||
yet the definitive version of record: 268
|
||||
and its later amendments or comparable: 267
|
||||
its later amendments or comparable ethical: 267
|
||||
the netherlands organisation for scientific research: 265
|
||||
analysis and interpretation of data ,: 265
|
||||
NUMBER https :// doi.org / NUMBER: 265
|
||||
later amendments or comparable ethical standards: 265
|
||||
the public , commercial or not-for-profit: 265
|
||||
the aim of this paper is: 263
|
||||
"% , NUMBER % and NUMBER": 262
|
||||
the other , anonymous , reviewer: 261
|
||||
and with the NUMBER helsinki declaration: 260
|
||||
human participants were reviewed and approved: 259
|
||||
participants were reviewed and approved by: 259
|
||||
involving human participants were reviewed and: 258
|
||||
studies involving human participants were reviewed: 257
|
||||
and analysis , decision to publish: 257
|
||||
it should be noted that the: 257
|
||||
in the decision to publish the: 256
|
||||
study design , data collection ,: 256
|
||||
found in the online version at: 256
|
||||
the authors report no conflicts of: 255
|
||||
under the terms of the creat: 255
|
||||
the terms of the creat ive: 255
|
||||
terms of the creat ive commo: 255
|
||||
of the creat ive commo ns: 255
|
||||
the creat ive commo ns attri: 255
|
||||
they have no conflicts of interest: 255
|
||||
data presented in this study is: 255
|
||||
participants provided their written informed consent: 254
|
||||
their written informed consent to participate: 254
|
||||
the decision to publish the results: 254
|
||||
the data presented in this study: 254
|
||||
provided their written informed consent to: 253
|
||||
CITATION license , which permits others: 252
|
||||
license , which permits others to: 252
|
||||
all authors approved the final version: 251
|
||||
that apply to the journal pertain: 251
|
||||
anonymous , reviewer ( s ): 251
|
||||
study is included in the article: 250
|
||||
other , anonymous , reviewer (: 250
|
||||
", anonymous , reviewer ( s": 250
|
||||
", reviewer ( s ) for": 250
|
||||
|
|
@ -0,0 +1,145 @@
|
|||
import re
|
||||
from typing import Dict, Pattern, Tuple, Union
|
||||
|
||||
from .models import BookmarkTitle
|
||||
|
||||
# partly inspired by https://github.com/KMCS-NII/AASC/blob/master/section_classify/section_classify.pl
|
||||
titles_of_interest: Dict[BookmarkTitle, Tuple[Union[str, Pattern], ...]] = {
|
||||
"Abstract": ("abstract",),
|
||||
"Author contribution": (
|
||||
"author contribution",
|
||||
"credit authorship contribution statement",
|
||||
"credit author statement",
|
||||
"corresponding author",
|
||||
),
|
||||
"Introduction": (
|
||||
"introduction",
|
||||
"preliminaries",
|
||||
"research question",
|
||||
"objective",
|
||||
"purpose",
|
||||
re.compile(r"^aim$"),
|
||||
"proposition",
|
||||
"what this study adds",
|
||||
"what does this study add",
|
||||
),
|
||||
"Background": (
|
||||
"background",
|
||||
"previous",
|
||||
"prior",
|
||||
"relevant work",
|
||||
"state of the art",
|
||||
"related work",
|
||||
"research in context",
|
||||
"earlier work",
|
||||
"related work",
|
||||
"problem",
|
||||
"challenge",
|
||||
"goal",
|
||||
"literature review",
|
||||
re.compile(r"^review$"),
|
||||
re.compile(r"^existing"),
|
||||
"comparison with existing literature",
|
||||
"literature search",
|
||||
"comparison with other studies",
|
||||
"study area",
|
||||
),
|
||||
"Methods": (
|
||||
"method",
|
||||
"approach",
|
||||
re.compile(r"^our"),
|
||||
"proposed",
|
||||
"study design",
|
||||
"procedure",
|
||||
"formulation",
|
||||
"methodology",
|
||||
"research design",
|
||||
"study protocol",
|
||||
"design",
|
||||
re.compile(r"experimental (?!result)"),
|
||||
"overview",
|
||||
"approach",
|
||||
"system",
|
||||
"definition",
|
||||
"algorithm",
|
||||
"model",
|
||||
"setup",
|
||||
"implementation",
|
||||
"evaluation metric",
|
||||
"scheme",
|
||||
),
|
||||
"Results": (
|
||||
"result",
|
||||
"data",
|
||||
"outcome",
|
||||
"statistic",
|
||||
"statistics",
|
||||
"statistical analysis",
|
||||
"analysis",
|
||||
"measure",
|
||||
"experiment",
|
||||
"finding",
|
||||
"hypothesis testing",
|
||||
"proof of proposition",
|
||||
"proof of concept",
|
||||
"report",
|
||||
"implication of all the available evidence",
|
||||
"contribution",
|
||||
"evaluation",
|
||||
"performance",
|
||||
"demonstration",
|
||||
"demonstrating",
|
||||
"example",
|
||||
"study",
|
||||
"studies",
|
||||
"qualitative comparison",
|
||||
"quantitative comparison",
|
||||
"validation",
|
||||
),
|
||||
"Discussion": (
|
||||
"discussion",
|
||||
"recommendation",
|
||||
"implication",
|
||||
"findings",
|
||||
"impact",
|
||||
"limitation",
|
||||
"assessment",
|
||||
"quality assessment",
|
||||
"interpretation",
|
||||
"contribution",
|
||||
"explanation",
|
||||
),
|
||||
"Outlook": (
|
||||
"outlook",
|
||||
"future research",
|
||||
"future",
|
||||
"follow up",
|
||||
"remaining",
|
||||
"prospect",
|
||||
"perspective",
|
||||
),
|
||||
"Conclusion": (
|
||||
"conclusion",
|
||||
"conclude",
|
||||
"concluding",
|
||||
"concluding",
|
||||
"final remark",
|
||||
"remark",
|
||||
"conclusie",
|
||||
),
|
||||
"Conflict of interest": (
|
||||
"compete interest",
|
||||
"funding support and author disclosure",
|
||||
"conflict of interest",
|
||||
"disclosure statement",
|
||||
"role of the funding source",
|
||||
"disclaimer",
|
||||
"conflicting interest",
|
||||
"declaration of interest",
|
||||
"risk of bias",
|
||||
"funding",
|
||||
"disclosure",
|
||||
"declaration",
|
||||
),
|
||||
"Acknowledgement": ("acknowledgement", "acknowledgment"),
|
||||
}
|
||||
|
|
@ -4,7 +4,7 @@ from typing import Any, Callable, Iterable, List
|
|||
def unique(
|
||||
values: Iterable[Any], *, key: Callable[[Any], Any] = lambda v: v
|
||||
) -> List[Any]:
|
||||
"""Only keep first occurrences and maintian order"""
|
||||
"""Only keep first occurrences and maintain order"""
|
||||
|
||||
key_values = {}
|
||||
for v in values:
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -3,6 +3,7 @@ import unittest
|
|||
from src.great_ai.utilities.clean import clean
|
||||
|
||||
|
||||
|
||||
class TestClean(unittest.TestCase):
|
||||
def test_xml_handling(self) -> None:
|
||||
xml = '<strong>Hi, </strong> my name<br/>is <span style="color: hotpink;"> András</span>! <3 <> < ></><> <> <|'
|
||||
|
|
@ -59,6 +60,13 @@ class TestClean(unittest.TestCase):
|
|||
|
||||
self.assertEqual(clean(text), cleaned)
|
||||
|
||||
def test_T_I_T_L_E_case(self) -> None:
|
||||
text = "an a r t i c l e is to F I N D the purpose of a tree"
|
||||
|
||||
cleaned = "an article is to FIND the purpose of a tree"
|
||||
|
||||
self.assertEqual(clean(text), cleaned)
|
||||
|
||||
def test_bracket_removal(self) -> None:
|
||||
text = "something [0], and [frefe, ferf]"
|
||||
cleaned = "something, and"
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import unittest
|
|||
from src.great_ai.utilities.get_sentences import get_sentences
|
||||
|
||||
|
||||
|
||||
class TestGetSentences(unittest.TestCase):
|
||||
def test_default(self) -> None:
|
||||
text = "This is a complete sentence. So is this. However this is n" # ot.
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ class TestLemmatizeText(unittest.TestCase):
|
|||
"to_PART",
|
||||
"create_VERB",
|
||||
"a_DET",
|
||||
"less_ADJ",
|
||||
"less_ADV",
|
||||
"resource_NOUN",
|
||||
"-_PUNCT",
|
||||
"intensive_ADJ",
|
||||
|
|
@ -117,7 +117,7 @@ class TestLemmatizeText(unittest.TestCase):
|
|||
"to_PART",
|
||||
"create_VERB",
|
||||
"a_DET",
|
||||
"less_ADJ",
|
||||
"less_ADV",
|
||||
"resource_NOUN",
|
||||
"-_PUNCT",
|
||||
"intensive_ADJ",
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import unittest
|
|||
from src.great_ai.utilities.match_names.match_names import match_names
|
||||
|
||||
|
||||
|
||||
class TestMatchNames(unittest.TestCase):
|
||||
def test_grid(self) -> None:
|
||||
names = [
|
||||
|
|
|
|||
|
|
@ -3,7 +3,17 @@ from pathlib import Path
|
|||
|
||||
from src.great_ai.utilities.publication_tei import PublicationTEI
|
||||
|
||||
from .data.parsed import authors, content, metadata, sentences
|
||||
|
||||
from .data.parsed import (
|
||||
abstract,
|
||||
authors,
|
||||
bookmarks,
|
||||
conclusion,
|
||||
content,
|
||||
introduction,
|
||||
metadata,
|
||||
sentences,
|
||||
)
|
||||
|
||||
DATA_PATH = Path(__file__).parent.resolve() / "data"
|
||||
|
||||
|
|
@ -30,6 +40,20 @@ class TestPublicationTEI(unittest.TestCase):
|
|||
def test_sentences(self) -> None:
|
||||
self.assertEqual(PublicationTEI(self.test_xml).sentences, sentences)
|
||||
|
||||
def test_bookmarks(self) -> None:
|
||||
self.assertEqual(PublicationTEI(self.test_xml).bookmarks, bookmarks)
|
||||
|
||||
def test_abstract(self) -> None:
|
||||
self.assertEqual(PublicationTEI(self.test_xml).abstract_sentences, abstract)
|
||||
|
||||
def test_introduction(self) -> None:
|
||||
self.assertEqual(
|
||||
PublicationTEI(self.test_xml).introduction_sentences, introduction
|
||||
)
|
||||
|
||||
def test_conclusion(self) -> None:
|
||||
self.assertEqual(PublicationTEI(self.test_xml).conclusion_sentences, conclusion)
|
||||
|
||||
def test_empty1(self) -> None:
|
||||
tei = PublicationTEI("<TEI/>")
|
||||
tei.publication_metadata, tei.publication_metadata, tei.authors, tei.content, tei.sentences
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import unittest
|
|||
|
||||
from src.great_ai.utilities.unique import unique
|
||||
|
||||
|
||||
original = [
|
||||
("a", 1),
|
||||
("b", 5),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue