Add sus updates

great_ai/README.md

@@ -9,6 +9,7 @@
 - [parallel_map](src/sus/parallel_map.py)
 - [match_names](src/sus/match_names/match_names.py)
 - [evaluate_ranking](src/sus/evaluate_ranking/evaluate_ranking.py)
+- [get_sentences](src/sus/get_sentences.py)
 
 ### Requires loading spacy model
 
@@ -17,14 +18,14 @@
 > Add this to the Dockerfile for caching the spaCy model:
 >
 > ```docker
-> RUN pip install --no-cache-dir en-core-web-lg@https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.2.0/en_core_web_lg-3.2.0-py3-none-any.whl
+> RUN pip install --no-cache-dir en-core-web-sm@https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl
 > ```
 
-- [spacy model (nlp)](src/sus/nlp.py)
+- [publication TEI](src/sus/publication_tei/publication_tei.py)
-- [get_sentences](src/sus/get_sentences.py)
 - [lemmatize_text](src/sus/lemmatize_text.py)
 - [lemmatize_token](src/sus/lemmatize_token.py)
-- [publication TEI](src/sus/publication_tei/publication_tei.py)
+- [spacy model (nlp)](src/sus/nlp.py)
+- [filter_sentences](src/sus/matcher/filter_sentences.py)
 
 ## Development
 

great_ai/setup.cfg

@@ -1,14 +1,14 @@
 [metadata]
-name = sus
+name = great-ai
-version = 0.0.3
+version = 0.0.1
-author = ScoutinScience B.V.
+author = András Schmelczer
 author_email = andras@scoutinscience.com
-description = [S]coutinScience [u]tilitie[s]: reusable utilities for text processing
+description = 
 long_description = file: README.md
 long_description_content_type = text/markdown
-url = https://github.com/ScoutinScience/platform
+url = https://github.com/ScoutinScience/great-ai
 project_urls =
-    Bug Tracker = https://github.com/ScoutinScience/platform/issues
+    Bug Tracker = https://github.com/ScoutinScience/great-ai/issues
 classifiers =
     Programming Language :: Python :: 3
     Operating System :: OS Independent
@@ -27,13 +27,17 @@ install_requires =
     psutil >= 5.9.0
     beautifulsoup4 >= 4.10.0
     lxml >= 4.6.0
-    spacy >= 3.2.0
+    spacy >= 3.3.0
     pydantic >= 1.8.0
     scikit-learn >= 1.0.0
     matplotlib >= 3.5.0
     numpy >= 1.22.0
     langcodes[data] >= 3.3.0
+    segtok >= 1.5.11
     langdetect >= 1.0.9
 
+[options.package_data]
+* = *.json, *.yaml, *.yml
+
 [options.packages.find]
 where = src

great_ai/src/great_ai/utilities/clean.py

@@ -9,7 +9,6 @@ from .external.pylatexenc.latex2text import LatexNodes2Text
 from .logger.create_logger import create_logger
 
 logger = create_logger("clean")
-
 latex = LatexNodes2Text()
 
 
@@ -46,6 +45,10 @@ def clean(
             text = "".join([c for c in text if not unicodedata.combining(c)])
             text = unidecode.unidecode(text)
 
+    text = re.sub(
+        r"\b[a-zA-Z](?:[\t ]+[a-zA-Z]\b)+", lambda m: re.sub(r"[\t ]", "", m[0]), text
+    )  # A R T I C L E => ARTICLE
+
     if remove_brackets:
         text = re.sub(r"\[[^\]]*\]", " ", text)
 

great_ai/src/great_ai/utilities/external/negspacy/termsets.py

@@ -40,7 +40,7 @@ preceding = [
     "doubt",
     "negative for",
     "no",
-    "vergreat_ai.utilities",
+    "versus",
     "without",
     "doesn't",
     "doesnt",
@@ -104,7 +104,7 @@ pseudo_clinical = pseudo + [
     "not ruled out",
     "not been ruled out",
     "not drain",
-    "no great_ai.utilitiespicious change",
+    "no suspicious change",
     "no interval change",
     "no significant interval change",
 ]

great_ai/src/great_ai/utilities/external/pylatexenc/latex2text/__init__.py

@@ -287,7 +287,7 @@ def placeholder_node_formatter(placeholdertext, block=True):
     r"""
     This function returns a callable that can be used in
     :py:class:`MacroTextSpec`, :py:class:`EnvironmentTextSpec`, or
-    :py:class:`SpecialsTextSpec` for latex nodes that do not have a great textual
+    :py:class:`SpecialsTextSpec` for latex nodes that do not have a good textual
     representation, providing as text replacement the simple placeholder text
     ``'< P L A C E H O L D E R   T E X T >'``.
 
@@ -316,7 +316,7 @@ def fmt_placeholder_node(node, l2tobj):
     r"""
     This function can be used as callable in :py:class:`MacroTextSpec`,
     :py:class:`EnvironmentTextSpec`, or :py:class:`SpecialsTextSpec` for latex
-    nodes that do not have a great textual representation.  The text replacement
+    nodes that do not have a good textual representation.  The text replacement
     is the placeholder text
     ``'< N A M E   O F   T H E   M A C R O   O R   E N V I R O N M E N T >'``.
 

great_ai/src/great_ai/utilities/external/pylatexenc/latex2text/_defaultspecs.py

@@ -266,7 +266,7 @@ _latex_specs_approximations = {
             ("fcolorbox", "%(5)s"),
             ("hspace", ""),
             ("vspace", "\n"),
-            # \\ is treated as an "approximation" because a great text renderer would
+            # \\ is treated as an "approximation" because a good text renderer would
             # have to actually note that this is a end-of-line marker which is not
             # to be confused with other newlines in the paragraph (which can be
             # reflowed)

great_ai/src/great_ai/utilities/external/pylatexenc/latexencode/_unicode_to_latex_encoder.py

@@ -342,7 +342,7 @@ class UnicodeToLatexEncoder(object):
 
        Possible protection schemes are:
 
-         - 'braces' (the default):  Any great_ai.utilitiespicious replacement text (that
+         - 'braces' (the default):  Any suspicious replacement text (that
            might look fragile) is placed in curly braces ``{...}``.
 
          - 'braces-all':  All replacement latex escapes are surrounded in

great_ai/src/great_ai/utilities/external/pylatexenc/latexwalker/__main__.py

@@ -0,0 +1,230 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2018 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+import argparse
+import fileinput
+import json
+import logging
+import sys
+
+from ..latexwalker import LatexWalker, disp_node, make_json_encoder
+from ..version import version_str
+
+
+def main(argv=None):
+
+    if argv is None:
+        argv = sys.argv[1:]
+
+    parser = argparse.ArgumentParser(prog="latexwalker", add_help=False)
+
+    parser.add_argument(
+        "--output-format",
+        metavar="FORMAT",
+        dest="output_format",
+        choices=["human", "json"],
+        default="human",
+        help='Requested output format for the node tree ("human" or "json")',
+    )
+    parser.add_argument(
+        "--json-indent",
+        metavar="NUMSPACES",
+        dest="json_indent",
+        type=int,
+        default=2,
+        help="Indentation in JSON output (specify number of spaces "
+        "per indentation level)",
+    )
+    parser.add_argument(
+        "--json-compact",
+        dest="json_indent",
+        action="store_const",
+        const=None,
+        help="Output compact JSON",
+    )
+
+    parser.add_argument(
+        "--keep-inline-math",
+        action="store_const",
+        const=True,
+        dest="keep_inline_math",
+        default=True,
+        help=argparse.SUPPRESS,
+    )
+    parser.add_argument(
+        "--no-keep-inline-math",
+        action="store_const",
+        const=False,
+        dest="keep_inline_math",
+        help=argparse.SUPPRESS,
+    )
+
+    parser.add_argument(
+        "--tolerant-parsing",
+        action="store_const",
+        const=True,
+        dest="tolerant_parsing",
+        default=True,
+    )
+    parser.add_argument(
+        "--no-tolerant-parsing",
+        action="store_const",
+        const=False,
+        dest="tolerant_parsing",
+        help="Tolerate syntax errors when parsing, and attempt "
+        "to continue (default yes)",
+    )
+
+    # I'm not sure this flag is useful and if it should be exposed at all.
+    # Accept it, but make it hidden.
+    parser.add_argument(
+        "--strict-braces",
+        action="store_const",
+        const=True,
+        dest="strict_braces",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
+    parser.add_argument(
+        "--no-strict-braces",
+        action="store_const",
+        const=False,
+        dest="strict_braces",
+        # help="Report errors for mismatching LaTeX braces (default no)"
+        help=argparse.SUPPRESS,
+    )
+
+    parser.add_argument(
+        "-q",
+        "--quiet",
+        dest="logging_level",
+        action="store_const",
+        const=logging.ERROR,
+        default=logging.INFO,
+        help="Suppress warning messages",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        dest="logging_level",
+        action="store_const",
+        const=logging.DEBUG,
+        help="Verbose output",
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version="pylatexenc {}".format(version_str),
+        help="Show version information and exit",
+    )
+    parser.add_argument(
+        "--help", action="help", help="Show this help information and exit"
+    )
+
+    parser.add_argument(
+        "--code",
+        "-c",
+        action="store",
+        default=None,
+        metavar="LATEX_CODE",
+        help="Convert the given LATEX_CODE to unicode text instead of reading "
+        "from FILE or standard input.  You cannot specify FILEs if you use this "
+        "option, and any standard input is ignored.",
+    )
+
+    parser.add_argument(
+        "files",
+        metavar="FILE",
+        nargs="*",
+        help="Input files (if none specified, read from stdandard input)",
+    )
+
+    args = parser.parse_args(argv)
+
+    logging.basicConfig()
+    logging.getLogger().setLevel(args.logging_level)
+    logger = logging.getLogger(__name__)
+
+    latex = ""
+    if args.code:
+        if args.files:
+            logger.error(
+                "Cannot specify both FILEs and --code option. "
+                "Use --help option for more information."
+            )
+            sys.exit(1)
+        latex = args.code
+    else:
+        for line in fileinput.input(files=args.files):
+            latex += line
+
+    latexwalker = LatexWalker(
+        latex, tolerant_parsing=args.tolerant_parsing, strict_braces=args.strict_braces
+    )
+
+    (nodelist, pos, len_) = latexwalker.get_latex_nodes()
+
+    if args.output_format == "human":
+        print("\n--- NODES ---\n")
+        for n in nodelist:
+            disp_node(n)
+        print("\n-------------\n")
+        return
+
+    if args.output_format == "json":
+        json.dump(
+            {
+                "nodelist": nodelist,
+            },
+            sys.stdout,
+            cls=make_json_encoder(latexwalker),
+            indent=args.json_indent,
+        )
+        sys.stdout.write("\n")
+        return
+
+    raise ValueError("Invalid output format: " + args.output_format)
+
+
+def run_main():
+
+    try:
+
+        main()
+
+    except SystemExit:
+        raise
+    except:  # lgtm [py/catch-base-exception]
+        import pdb
+        import traceback
+
+        traceback.print_exc()
+        pdb.post_mortem()
+
+
+if __name__ == "__main__":
+
+    # run_main() # debug
+    main()

great_ai/src/great_ai/utilities/external/pylatexenc/version.py

@@ -33,7 +33,7 @@
 #
 #       - tests pass: https://travis-ci.org/github/phfaist/pylatexenc
 #
-#       - LGTM looks great: https://lgtm.com/projects/g/phfaist/pylatexenc/
+#       - LGTM looks good: https://lgtm.com/projects/g/phfaist/pylatexenc/
 #
 #       - python package creation works: (python setup.py sdist, pip install
 #         dist/pylatexenc-xxx.tar.gz)

great_ai/src/great_ai/utilities/get_sentences.py

@@ -1,12 +1,15 @@
 from typing import List
 
+from segtok.segmenter import split_multi
+
 from .data import sentence_ending_punctuations
-from .nlp import nlp
 
 
 def get_sentences(text: str, ignore_partial: bool = False) -> List[str]:
-    doc = nlp(text)
+    if text.strip() == "":
-    possible_sentences = [s.text for s in doc.sents]
+        return []
+
+    possible_sentences = [s for s in split_multi(text) if s]
 
     if ignore_partial:
         possible_sentences = [

great_ai/src/great_ai/utilities/matcher/__init__.py

@@ -0,0 +1,3 @@
+from .fast_tokenize import fast_tokenize
+from .filter_sentences import filter_sentences
+from .normalize import normalize

great_ai/src/great_ai/utilities/matcher/fast_tokenize.py

@@ -0,0 +1,30 @@
+import re
+from typing import List, Union
+
+from segtok.tokenizer import word_tokenizer
+
+from ..get_sentences import get_sentences
+from .normalize import normalize
+
+
+def fast_tokenize(
+    text: Union[List[str], str], ignore_partial: bool = False
+) -> List[List[str]]:
+    if isinstance(text, str):
+        text = normalize(text)
+        text = get_sentences(text, ignore_partial=ignore_partial)
+
+    results: List[List[str]] = []
+
+    for sentence in text:
+        sentence = re.sub(r"\bare\b", "is", sentence)
+        sentence = re.sub(r"\ban\b", "a", sentence)
+        sentence = re.sub(r"\bthese\b", "this", sentence)
+        results.append(
+            [
+                token.lower() if token not in {"CITATION", "NUMBER"} else token
+                for token in word_tokenizer(sentence)
+            ]
+        )
+
+    return results

great_ai/src/great_ai/utilities/matcher/filter_sentences.py

@@ -0,0 +1,66 @@
+from pathlib import Path
+from typing import Dict, List, Union
+
+import yaml
+from spacy.matcher import Matcher
+
+from ..get_sentences import get_sentences
+from ..nlp import nlp
+from .fast_tokenize import fast_tokenize
+from .normalize import normalize
+
+rules_cache: Dict[str, Matcher] = {}
+
+
+def filter_sentences(
+    sentences: str,
+    rules_file: Path,
+    inverse: bool = False,
+    ignore_partial: bool = False,
+) -> List[str]:
+    if str(rules_file) not in rules_cache:
+        with open(rules_file, encoding="utf-8") as f:
+            rule_patterns = yaml.safe_load(f).keys()
+
+        matcher = Matcher(nlp.vocab)
+        rules = [_pattern_to_rule(p) for p in rule_patterns]
+        matcher.add("", rules)
+        rules_cache[str(rules_file)] = matcher
+
+    matcher = rules_cache[str(rules_file)]
+
+    original_sentences = get_sentences(sentences, ignore_partial=ignore_partial)
+
+    tokenized = fast_tokenize(original_sentences, ignore_partial=ignore_partial)
+
+    results: List[str] = []
+    for original_sentence, sentence in zip(original_sentences, tokenized):
+        doc = nlp(normalize(" ".join(sentence)))
+        matches = matcher(doc)
+        if matches:
+            # _, start, end = max(
+            #     matches,
+            #     key=lambda v: v[2] - v[1],
+            # )
+            # print(str(doc[start:end]))
+
+            if not inverse:
+                results.append(original_sentence)
+        elif inverse:
+            results.append(original_sentence)
+
+    return results
+
+
+def _pattern_to_rule(pattern: str) -> List[Dict[str, Union[bool, str]]]:
+    result: List[Dict[str, Union[bool, str]]] = []
+    for t in pattern.split():
+        if t == "*":
+            result.extend([{"OP": "?"}, {"OP": "?"}])
+        elif t == "CITATION":
+            result.append({"ORTH": "CITATION"})
+        elif t == "NUMBER":
+            result.append({"ORTH": "NUMBER"})
+        else:
+            result.append({"LOWER": t})
+    return result

great_ai/src/great_ai/utilities/matcher/normalize.py

@@ -0,0 +1,22 @@
+import re
+
+from ..clean import clean
+
+
+def normalize(text: str) -> str:
+    text = re.sub(
+        r"""
+            ([A-Z]\w+\W+(et\ al.))     # inline reference: Bank et al.
+            | (\[[0-9-, ]+\])          # IEEE style: [1], [2-4], [3, 5]
+            | (\(.*?,?\W+?\d+\))       # APA style: (Bank, 2020)
+            | ([A-Z]\w+ \(?\d+\))      # APA style: Bank (2020)
+        """,
+        " CITATION ",
+        text,
+        flags=re.VERBOSE,
+    )
+    text = re.sub(r"\d[\d,. -]*(st|nd|rd|th)?", " NUMBER ", text)
+    text = clean(text, convert_to_ascii=True)
+    text = re.sub(r"[^a-zA-Z.?!,:;'\" -]", "", text)
+
+    return text

great_ai/src/great_ai/utilities/nlp.py

@@ -1,21 +1,21 @@
 try:
-    import en_core_web_lg
+    import en_core_web_sm
 except ImportError:
     import subprocess
 
-    print("Spacy model en_core_web_lg not found locally, downloading...")
+    print("Spacy model en_core_web_sm not found locally, downloading...")
 
     subprocess.call(
         [
             "pip",
             "install",
-            "https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.2.0/en_core_web_lg-3.2.0-py3-none-any.whl",
+            "https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl",
         ]
     )
-    import en_core_web_lg
+    import en_core_web_sm
 
 from .external.negspacy import negation  # noqa: F401 it's important to import this
 
-nlp = en_core_web_lg.load()
+nlp = en_core_web_sm.load()
 
 nlp.add_pipe("negex")

great_ai/src/great_ai/utilities/parallel_map.py

@@ -9,16 +9,13 @@ from .logger import create_logger
 
 logger = create_logger("parallel_map")
 
-
 def parallel_map(
     function: Callable[[Any], Any],
     values: Iterable[Any],
     chunk_size: Optional[int] = None,
-    concurrency: Optional[int] = None,
+    concurrency: int = psutil.cpu_count(),
     disable_progress: bool = False,
 ) -> List[Any]:
-    if concurrency is None:
-        concurrency = psutil.cpu_count()
     assert concurrency > 0
     assert chunk_size is None or chunk_size > 0
 

great_ai/src/great_ai/utilities/publication_tei/models/__init__.py

@@ -1,5 +1,7 @@
 from .affiliation import Affiliation
 from .author import Author
+from .bookmark import Bookmark
+from .bookmark_title import BookmarkTitle
 from .element import Element, Meta, MetaType, Paragraph, Title
 from .publication_metadata import PublicationMetadata
 from .text import Text

great_ai/src/great_ai/utilities/publication_tei/models/bookmark.py

@@ -0,0 +1,10 @@
+from pydantic import BaseModel
+
+from .bookmark_title import BookmarkTitle
+
+
+class Bookmark(BaseModel):
+    title: BookmarkTitle
+    original_title: str
+    document_order: int
+    coordinates: str

great_ai/src/great_ai/utilities/publication_tei/models/bookmark_title.py

@@ -0,0 +1,16 @@
+from typing import Literal
+
+BookmarkTitle = Literal[
+    "Abstract",
+    "Author contribution",
+    "Introduction",
+    "Background",
+    "Methods",
+    "Results",
+    "Discussion",
+    "Outlook",
+    "Conclusion",
+    "Conflict of interest",
+    "Acknowledgement",
+    "Annex",
+]

great_ai/src/great_ai/utilities/publication_tei/models/text.py

@@ -1,7 +1,9 @@
+from typing import Optional
+
 from pydantic import BaseModel
 
 
 class Text(BaseModel):
     content: str
     document_order: int
-    coordinates: str
+    coordinates: Optional[str]

great_ai/src/great_ai/utilities/publication_tei/publication_tei.py

@@ -1,22 +1,43 @@
-from functools import cached_property
+import os
-from typing import Any, List, Optional, Union
+import re
+from functools import cached_property, lru_cache
+from pathlib import Path
+from typing import Any, List, Optional, Pattern, Tuple, Union
 
 from bs4 import BeautifulSoup
 from bs4.element import NavigableString, Tag
-
+from sus.publication_tei.models.element import Paragraph
-from great_ai.utilities.publication_tei.models.element import Paragraph
 
 from ..clean import clean
-from .models import Affiliation, Author, Element, Meta, PublicationMetadata, Text, Title
+from ..lemmatize_text import lemmatize_text
+from ..matcher import filter_sentences
+from ..unique import unique
+from .models import (
+    Affiliation,
+    Author,
+    Bookmark,
+    BookmarkTitle,
+    Element,
+    Meta,
+    MetaType,
+    PublicationMetadata,
+    Text,
+    Title,
+)
+from .titles_of_interest import titles_of_interest
+
+THIS_FOLDER = Path(os.path.dirname(os.path.abspath(__file__)))
 
 
 class PublicationTEI:
+    # remove template sentenaces, such as copyright notices
+    agressive_cleaning_enabled = True
+
     def __init__(self, tei: str):
         self._document_order_counter = 0
 
         if tei:
-            cleaned_xml = clean(tei, ignore_xml=True)
+            self.soup = BeautifulSoup(tei, "xml")
-            self.soup = BeautifulSoup(cleaned_xml, "xml")
         else:
             self.soup = BeautifulSoup()
 
@@ -79,6 +100,132 @@ class PublicationTEI:
             for sentence in element.sentences
         ]
 
+    @cached_property
+    def bookmarks(self) -> List[Bookmark]:
+        candidates: List[Bookmark] = [
+            *self._find_matching_meta("Abstract", "abstract_start")[0],
+            *self._find_matching_title("Abstract"),
+            *self._find_matching_title("Author contribution"),
+            *self._find_matching_meta("Acknowledgement", "acknowledgements_start")[0],
+            *self._find_matching_title("Acknowledgement"),
+            *self._find_matching_title("Conflict of interest"),
+        ]
+
+        _, start = self._find_matching_meta(None, "abstract_end")
+
+        candidates += [
+            *self._find_matching_title("Background", start),
+            *self._find_matching_title("Methods", start),
+            *self._find_matching_title("Results", start),
+            *self._find_matching_title("Discussion", start),
+            *self._find_matching_title("Introduction", start),
+            *self._find_matching_title("Conclusion", start),
+            *self._find_matching_title("Outlook", start),
+            *self._find_matching_meta("Annex", "annex_start", start)[0],
+        ]
+
+        candidates = sorted(candidates, key=lambda c: c.document_order)
+        candidates = unique(candidates, key=lambda c: c.document_order)
+        candidates = unique(candidates, key=lambda c: c.title)
+
+        return candidates
+
+    @cached_property
+    def abstract_sentences(self) -> List[Text]:
+        try:
+            abstract_start = next(
+                i
+                for i, m in enumerate(self.content)
+                if isinstance(m, Meta) and m.meta_type == "abstract_start"
+            )
+            abstract_end = next(
+                i
+                for i, m in enumerate(self.content)
+                if isinstance(m, Meta) and m.meta_type == "abstract_end"
+            )
+            return [
+                s
+                for p in self.content[abstract_start:abstract_end]
+                if isinstance(p, Paragraph)
+                for s in p.sentences
+            ]
+        except StopIteration:
+            pass  # let's try another way
+
+        try:
+            abstract_start = next(
+                m.document_order for m in self.bookmarks if m.title == "Abstract"
+            )
+            abstract_sentences: List[Text] = []
+            for c in self.content:
+                if isinstance(c, Paragraph):
+                    abstract_sentences.extend(
+                        s for s in c.sentences if s.document_order > abstract_start
+                    )
+                elif len(abstract_sentences) >= 5:
+                    break
+            return abstract_sentences
+        except StopIteration:
+            pass  # let's try another way
+
+        return self.sentences[:10]
+
+    @cached_property
+    def introduction_sentences(self) -> List[Text]:
+        """Includes abstract"""
+        introduction_end = 4
+
+        try:
+            introduction_start = [
+                m.document_order for m in self.bookmarks if m.title == "Introduction"
+            ][-1]
+
+            for m in self.bookmarks:
+                if m.title != "Introduction" and m.document_order > introduction_start:
+                    introduction_end = m.document_order
+                    break
+        except IndexError:
+            pass
+
+        try:
+            introduction_end = max(
+                next(
+                    i
+                    for i, m in enumerate(self.content)
+                    if isinstance(m, Meta) and m.meta_type == "abstract_end"
+                ),
+                introduction_end,
+            )
+        except StopIteration:
+            pass
+
+        introduction_sentences: List[Text] = []
+        for c in self.content:
+            if isinstance(c, Paragraph):
+                introduction_sentences.extend(
+                    s for s in c.sentences if s.document_order < introduction_end
+                )
+
+        return introduction_sentences
+
+    @cached_property
+    def conclusion_sentences(self) -> List[Text]:
+        try:
+            conclusion_start = next(
+                m.document_order for m in self.bookmarks if m.title == "Conclusion"
+            )
+            conclusion_sentences: List[Text] = []
+            for c in self.content:
+                if isinstance(c, Paragraph):
+                    conclusion_sentences.extend(
+                        s for s in c.sentences if s.document_order > conclusion_start
+                    )
+                elif len(conclusion_sentences) >= 8:
+                    break
+            return conclusion_sentences
+        except StopIteration:
+            return self.sentences[-10:]
+
     def _parse_author(self, raw: Tag) -> Author:
         return Author(
             name=(
@@ -133,7 +280,9 @@ class PublicationTEI:
                 results.append(Meta(meta_type="annex_start"))
                 results.extend(self._get_primitives(r))
                 results.append(Meta(meta_type="annex_end"))
-            elif not r.find_parents(["abstract", "div"]):
+            elif not r.find_parents(
+                ["abstract", "div", "figure"]
+            ):  # figures are omitted as well
                 results.extend(self._get_primitives(r))
 
         return results
@@ -143,14 +292,20 @@ class PublicationTEI:
 
         for r in raw.find_all(["head", "p"]):
             if r.name == "head" and r.get("coords") and r.get_text():
-                results.append(Title(text=self._parse_text(r)))
+                text = self._parse_text(r)
+                if text:
+                    results.append(Title(text=text))
             elif r.name == "p" and r.find_all("s"):
                 results.append(
                     Paragraph(
                         sentences=[
-                            self._parse_text(sentence)
+                            t
-                            for sentence in r.find_all("s")
+                            for t in [
-                            if sentence.get_text()
+                                self._parse_text(sentence, ignore_partial=True)
+                                for sentence in r.find_all("s")
+                                if sentence.get_text()
+                            ]
+                            if t
                         ]
                     )
                 )
@@ -164,9 +319,25 @@ class PublicationTEI:
             clean(element.get_text(separator=" ", strip=True)) if element else default
         )
 
-    def _parse_text(self, raw: Tag) -> Text:
+    def _parse_text(self, raw: Tag, ignore_partial: bool = False) -> Optional[Text]:
+        text = raw.get_text()
+        if text is None:
+            return None
+
+        if self.agressive_cleaning_enabled:
+            filtered = filter_sentences(
+                text,
+                THIS_FOLDER / "templates.yaml",
+                inverse=True,
+                ignore_partial=ignore_partial,
+            )
+            text = " ".join(filtered)
+
+        if not text.strip():
+            return None
+
         return Text(
-            content=clean(raw.get_text()),
+            content=text,
             document_order=self._generate_document_order_id(),
             coordinates=raw.get("coords"),
         )
@@ -175,3 +346,78 @@ class PublicationTEI:
         value = self._document_order_counter
         self._document_order_counter += 1
         return value
+
+    def _find_matching_meta(
+        self,
+        bookmark_title: Optional[BookmarkTitle],
+        meta_type: MetaType,
+        start: int = 0,
+    ) -> Tuple[List[Bookmark], int]:
+        for i, e in enumerate(self.content[start:], start=start):
+            if not isinstance(e, Meta):
+                continue
+
+            if e.meta_type == meta_type:
+                for e_next in self.content[i + 1 :]:
+                    if isinstance(e_next, Title):
+                        return [
+                            Bookmark(
+                                title=bookmark_title,
+                                original_title=e_next.text.content,
+                                document_order=e_next.text.document_order,
+                                coordinates=e_next.text.coordinates,
+                            )
+                        ] if bookmark_title else [], i
+                    if isinstance(e_next, Paragraph) and e_next.sentences:
+                        return [
+                            Bookmark(
+                                title=bookmark_title,
+                                original_title="",
+                                document_order=e_next.sentences[0].document_order,
+                                coordinates=e_next.sentences[0].coordinates,
+                            )
+                        ] if bookmark_title else [], i
+
+        return [], 0
+
+    def _find_matching_title(
+        self,
+        bookmark_title: BookmarkTitle,
+        start: int = 0,
+    ) -> List[Bookmark]:
+        return [
+            Bookmark(
+                title=bookmark_title,
+                original_title=e.text.content,
+                document_order=e.text.document_order,
+                coordinates=e.text.coordinates,
+            )
+            for e in self.content[start:]
+            if isinstance(e, Title)
+            and self._match_title(e.text.content, titles_of_interest[bookmark_title])
+        ]
+
+    @staticmethod
+    def _match_title(title: str, keywords: Tuple[Union[Pattern, str], ...]) -> bool:
+        title = PublicationTEI._process_section_title(title)
+
+        if any(
+            PublicationTEI._process_section_title(k) in title
+            for k in keywords
+            if isinstance(k, str)
+        ):
+            return True
+
+        return any(k.match(title) for k in keywords if isinstance(k, Pattern))
+
+    @staticmethod
+    @lru_cache(maxsize=2000)
+    def _process_section_title(title: str) -> str:
+        title = re.sub(r"^\s*[ivx]+[.,)]? ", "", title)  # Remove leading Roman-numerals
+        title = clean(title, convert_to_ascii=True)
+        title = re.sub(
+            r"[^a-zA-Z ]", "", title
+        )  # Remove everything but letters and spaces (hypens are also removed)
+        title_tokens = lemmatize_text(title)
+        title = " ".join(t for t in title_tokens if t.strip())
+        return title

great_ai/src/great_ai/utilities/publication_tei/templates.yaml

@@ -0,0 +1,837 @@
+included in the article's creative commons: 3827
+is a open access article distributed: 3822
+a open access article distributed under: 3761
+open access article distributed under the: 3761
+access article distributed under the terms: 3761
+the terms and conditions of the: 3688
+this article is a open access: 3678
+article is a open access article: 3678
+article distributed under the terms and: 3678
+distributed under the terms and conditions: 3678
+under the terms and conditions of: 3678
+terms and conditions of the creative: 3678
+and conditions of the creative commons: 3678
+conditions of the creative commons attribution: 3678
+in the article's creative commons licence: 3503
+authors declare that they have no: 3315
+https :// doi.org / NUMBER /: 3189
+the authors declare that they have: 3015
+remains neutral with regard to jurisdictional: 2464
+neutral with regard to jurisdictional claims: 2464
+with regard to jurisdictional claims in: 2464
+regard to jurisdictional claims in published: 2462
+to jurisdictional claims in published maps: 2462
+the original author ( s ): 2461
+jurisdictional claims in published maps and: 2461
+claims in published maps and institutional: 2461
+author ( s ) and the: 2460
+original author ( s ) and: 2456
+in published maps and institutional affiliations: 2415
+nature remains neutral with regard to: 2399
+springer nature remains neutral with regard: 2392
+( s ) and the source: 2353
+of the creative commons attribution CITATION: 2302
+", provide a link to the": 2299
+the source , provide a link: 2293
+source , provide a link to: 2293
+you give appropriate credit to the: 2292
+give appropriate credit to the original: 2292
+appropriate credit to the original author: 2292
+and indicate if changes were made: 2289
+credit to the original author (: 2281
+to the original author ( s: 2281
+provide a link to the creative: 2280
+a link to the creative commons: 2280
+s ) and the source ,: 2279
+) and the source , provide: 2277
+and the source , provide a: 2277
+", and indicate if changes were": 2271
+at https :// doi.org / NUMBER: 2195
+creative commons attribution NUMBER international license: 2158
+", distribution and reproduction in any": 2158
+distribution and reproduction in any medium: 2157
+can be found online at https: 2106
+be found online at https ://: 2106
+article can be found online at: 2069
+license , which permits use ,: 2063
+this article can be found online: 2010
+online version contains supplementary material available: 1980
+supplementary material available at https ://: 1979
+version contains supplementary material available at: 1978
+contains supplementary material available at https: 1973
+the online version contains supplementary material: 1957
+note springer nature remains neutral with: 1956
+other third party material in this: 1945
+to view a copy of this: 1945
+or other third party material in: 1944
+need to obtain permission directly from: 1944
+to obtain permission directly from the: 1944
+images or other third party material: 1943
+regulation or exceeds the permitted use: 1943
+or exceeds the permitted use ,: 1943
+obtain permission directly from the copyright: 1943
+the permitted use , you will: 1942
+by statutory regulation or exceeds the: 1941
+statutory regulation or exceeds the permitted: 1941
+exceeds the permitted use , you: 1941
+use , you will need to: 1941
+", you will need to obtain": 1941
+you will need to obtain permission: 1941
+permitted use , you will need: 1940
+will need to obtain permission directly: 1940
+permitted by statutory regulation or exceeds: 1939
+publisher's note springer nature remains neutral: 1939
+and your intended use is not: 1938
+your intended use is not permitted: 1938
+not permitted by statutory regulation or: 1938
+intended use is not permitted by: 1937
+is not permitted by statutory regulation: 1937
+permission directly from the copyright holder: 1937
+use is not permitted by statutory: 1936
+material is not included in the: 1933
+the images or other third party: 1932
+if material is not included in: 1926
+in a credit line to the: 1924
+otherwise in a credit line to: 1923
+this article is included in the: 1922
+third party material in this article: 1921
+party material in this article is: 1921
+material in this article is included: 1921
+in this article is included in: 1921
+", unless indicated otherwise in a": 1920
+indicated otherwise in a credit line: 1920
+is included in the article's creative: 1919
+unless indicated otherwise in a credit: 1919
+article is included in the article's: 1918
+a credit line to the material: 1914
+not included in the article's creative: 1907
+is not included in the article's: 1906
+is licensed under a creative commons: 1898
+authors declare no conflict of interest: 1883
+and reproduction in any medium or: 1870
+reproduction in any medium or format: 1870
+in any medium or format ,: 1870
+any medium or format , as: 1870
+medium or format , as long: 1870
+or format , as long as: 1870
+format , as long as you: 1869
+", as long as you give": 1869
+as long as you give appropriate: 1869
+long as you give appropriate credit: 1869
+as you give appropriate credit to: 1869
+", adaptation , distribution and reproduction": 1866
+adaptation , distribution and reproduction in: 1866
+use , sharing , adaptation ,: 1861
+", sharing , adaptation , distribution": 1861
+sharing , adaptation , distribution and: 1861
+NUMBER international license , which permits: 1846
+article is licensed under a creative: 1839
+this article is licensed under a: 1838
+under a creative commons attribution NUMBER: 1817
+licensed under a creative commons attribution: 1811
+a creative commons attribution NUMBER international: 1807
+view a copy of this licence: 1799
+a copy of this licence ,: 1799
+copy of this licence , visit: 1799
+of this licence , visit http: 1796
+this licence , visit http ://: 1796
+access this article is licensed under: 1794
+the authors would like to thank: 1793
+which permits use , sharing ,: 1791
+permits use , sharing , adaptation: 1790
+", which permits use , sharing": 1789
+open access this article is licensed: 1782
+international license , which permits use: 1776
+attribution NUMBER international license , which: 1773
+commons attribution NUMBER international license ,: 1772
+creative commons licence and your intended: 1769
+commons licence and your intended use: 1768
+licence and your intended use is: 1768
+the article's creative commons licence and: 1762
+article's creative commons licence and your: 1762
+creative commons licence , unless indicated: 1750
+commons licence , unless indicated otherwise: 1749
+licence , unless indicated otherwise in: 1749
+article's creative commons licence , unless: 1743
+the article's creative commons licence ,: 1742
+the authors declare no conflict of: 1742
+to this article can be found: 1714
+licence , and indicate if changes: 1698
+link to the creative commons licence: 1696
+commons licence , and indicate if: 1695
+to the creative commons licence ,: 1694
+the creative commons licence , and: 1694
+creative commons licence , and indicate: 1694
+the authors declare no competing interests: 1564
+and / or publication of this: 1530
+/ or publication of this article: 1523
+the research , authorship , and: 1410
+data to this article can be: 1395
+supplementary data to this article can: 1392
+competing financial interests or personal relationships: 1388
+research , authorship , and /: 1385
+", authorship , and / or": 1385
+authorship , and / or publication: 1385
+is available from the corresponding author: 1385
+", and / or publication of": 1384
+no known competing financial interests or: 1382
+known competing financial interests or personal: 1382
+declare that they have no known: 1377
+that they have no known competing: 1376
+of the creative commons attribution (: 1372
+have no known competing financial interests: 1371
+financial interests or personal relationships that: 1371
+the creative commons attribution ( cc: 1370
+creative commons attribution ( cc by: 1370
+they have no known competing financial: 1370
+to influence the work reported in: 1369
+influence the work reported in this: 1369
+interests or personal relationships that could: 1368
+appeared to influence the work reported: 1368
+or personal relationships that could have: 1367
+have appeared to influence the work: 1367
+could have appeared to influence the: 1364
+relationships that could have appeared to: 1363
+that could have appeared to influence: 1363
+personal relationships that could have appeared: 1362
+is distributed under the terms of: 1265
+relationships that could be construed as: 1259
+/ licenses / by / NUMBER: 1243
+in the absence of any commercial: 1242
+the absence of any commercial or: 1241
+commercial or financial relationships that could: 1241
+or financial relationships that could be: 1240
+absence of any commercial or financial: 1239
+any commercial or financial relationships that: 1239
+as a potential conflict of interest: 1239
+was conducted in the absence of: 1238
+of any commercial or financial relationships: 1238
+conducted in the absence of any: 1237
+financial relationships that could be construed: 1236
+be construed as a potential conflict: 1236
+the research was conducted in the: 1235
+construed as a potential conflict of: 1235
+that the research was conducted in: 1233
+research was conducted in the absence: 1233
+:// doi.org / NUMBER / s: 1220
+doi.org / NUMBER / s NUMBER: 1220
+declare that the research was conducted: 1212
+authors declare that the research was: 1204
+read and approved the final manuscript: 1196
+study is available from the corresponding: 1163
+creativecommons.org / licenses / by /: 1148
+:// creativecommons.org / licenses / by: 1147
+material available at https :// doi: 1135
+authors read and approved the final: 1074
+http :// creativecommons.org / licenses /: 1063
+that they have no conflict of: 1061
+all authors read and approved the: 1055
+requests for materials should be addressed: 1054
+for materials should be addressed to: 1054
+correspondence and requests for materials should: 1053
+and requests for materials should be: 1052
+findings of this study is available: 1046
+they have no conflict of interest: 1044
+declare that they have no conflict: 1020
+", visit http :// creat iveco": 1016
+licence , visit http :// creat: 1015
+the authors declare that the research: 993
+", on the other hand ,": 989
+available at https :// doi.org /: 925
+informed consent was obtained from all: 904
+this article is protected by copyright: 882
+found online at https :// doi: 871
+additional supporting information may be found: 866
+and moral rights for the publications: 865
+moral rights for the publications made: 865
+rights for the publications made accessible: 865
+for the publications made accessible in: 865
+is retained by the authors and: 865
+retained by the authors and /: 865
+by the authors and / or: 865
+the authors and / or other: 865
+authors and / or other copyright: 865
+and / or other copyright owners: 865
+/ or other copyright owners and: 865
+or other copyright owners and it: 865
+other copyright owners and it is: 865
+copyright owners and it is a: 865
+owners and it is a condition: 865
+and it is a condition of: 865
+it is a condition of accessing: 865
+is a condition of accessing publications: 865
+a condition of accessing publications that: 865
+condition of accessing publications that users: 865
+of accessing publications that users recognise: 865
+accessing publications that users recognise and: 865
+publications that users recognise and abide: 865
+that users recognise and abide by: 865
+users recognise and abide by the: 865
+recognise and abide by the legal: 865
+and abide by the legal requirements: 865
+abide by the legal requirements associated: 865
+by the legal requirements associated with: 865
+the legal requirements associated with this: 865
+copyright and moral rights for the: 864
+the publications made accessible in the: 864
+publications made accessible in the public: 864
+made accessible in the public portal: 864
+accessible in the public portal is: 864
+in the public portal is retained: 864
+the public portal is retained by: 864
+public portal is retained by the: 864
+portal is retained by the authors: 864
+legal requirements associated with this rights: 863
+the terms of the creative commons: 839
+and reproduction in any medium ,: 834
+reproduction in any medium , provided: 833
+declare that they have no competing: 823
+conflicts of interest with respect to: 811
+this work was supported by the: 804
+that they have no competing interests: 783
+respect to the research , authorship: 781
+with respect to the research ,: 780
+potential conflicts of interest with respect: 779
+interest with respect to the research: 776
+of article NUMBER fa of the: 775
+article NUMBER fa of the dutch: 775
+NUMBER fa of the dutch copyright: 775
+fa of the dutch copyright act: 775
+potential conflict of interest was reported: 775
+conflict of interest was reported by: 775
+no potential conflict of interest was: 773
+of interest was reported by the: 770
+in the present study , we: 767
+under the terms of article NUMBER: 766
+the terms of article NUMBER fa: 766
+terms of article NUMBER fa of: 766
+this study is available from the: 765
+support for the research , authorship: 762
+financial support for the research ,: 760
+of the dutch copyright act ,: 758
+distributed under the terms of the: 743
+publication is distributed under the terms: 742
+distributed under the terms of article: 742
+the original work is properly cited: 738
+if the publication is distributed under: 734
+the publication is distributed under the: 734
+the dutch copyright act , indicated: 734
+dutch copyright act , indicated by: 734
+copyright act , indicated by the: 734
+act , indicated by the taverne: 734
+", indicated by the taverne license": 734
+indicated by the taverne license above: 734
+by the taverne license above ,: 734
+the taverne license above , please: 734
+taverne license above , please follow: 734
+license above , please follow below: 734
+above , please follow below link: 734
+", please follow below link for": 734
+please follow below link for the: 734
+follow below link for the end: 734
+online at https :// doi.org /: 733
+of this study is available from: 731
+to the research , authorship ,: 728
+from NUMBER % to NUMBER %: 728
+CITATION , NUMBER CITATION , NUMBER: 723
+reprints and permission information is available: 722
+information is available at http ://: 722
+available at http :// www.nature.com /: 722
+is available at http :// www.nature.com: 721
+at http :// www.nature.com / reprints: 721
+terms of the creative commons attribution: 717
+and permission information is available at: 710
+permission information is available at http: 710
+provided the original work is properly: 707
+", provided the original work is": 706
+http :// www.nature.com / reprints publisher's: 706
+:// www.nature.com / reprints publisher's note: 705
+www.nature.com / reprints publisher's note springer: 705
+/ reprints publisher's note springer nature: 705
+reprints publisher's note springer nature remains: 705
+associated with this article can be: 700
+with this article can be found: 695
+available from the corresponding author upon: 691
+no potential conflicts of interest with: 685
+for the research , authorship ,: 685
+", visit http :// creativecommons.org /": 683
+visit http :// creativecommons.org / licenses: 683
+conflict of interest the authors declare: 678
+declared no potential conflicts of interest: 666
+visit http :// creat iveco mmons: 665
+for this article can be found: 658
+had no role in the design: 651
+material for this article can be: 645
+this is a open access article: 644
+the corresponding author upon reasonable request: 637
+supplementary material for this article can: 633
+from the corresponding author upon reasonable: 633
+the final version of the manuscript: 624
+the author ( s ) declared: 618
+the supplementary material for this article: 617
+in the public , commercial ,: 613
+the public , commercial , or: 612
+by the author ( s ): 599
+of interest the authors declare that: 594
+available from the corresponding author on: 589
+this article can be found ,: 587
+article can be found , in: 587
+link to the creative commons license: 585
+be found in the online version: 584
+", the aim of this study": 583
+be found online in the supporting: 583
+found online in the supporting information: 583
+", it has been shown that": 582
+we would like to thank the: 580
+supporting information may be found online: 579
+information may be found online in: 577
+may be found online in the: 577
+be found , in the online: 577
+found , in the online version: 577
+license , and indicate if changes: 576
+", in the online version ,": 574
+in the online version , at: 574
+for their contribution to the peer: 574
+their contribution to the peer review: 574
+this research did not receive any: 572
+reprints and permissions information is available: 572
+to the creative commons license ,: 571
+the creative commons license , and: 571
+creative commons license , and indicate: 571
+commons license , and indicate if: 571
+the funders had no role in: 570
+and permissions information is available at: 570
+did not receive any specific grant: 566
+agencies in the public , commercial: 566
+from the corresponding author on reasonable: 564
+in any medium , provided the: 563
+any medium , provided the original: 562
+to the peer review of this: 562
+the peer review of this work: 562
+not receive any specific grant from: 561
+from funding agencies in the public: 561
+the corresponding author on reasonable request: 561
+is available at www.nature.com / reprints: 559
+funding agencies in the public ,: 558
+permissions information is available at www.nature.com: 558
+information is available at www.nature.com /: 558
+research did not receive any specific: 552
+/ NUMBER / s NUMBER correspondence: 552
+NUMBER correspondence and requests for materials: 552
+below link for the end user: 551
+NUMBER / s NUMBER correspondence and: 551
+/ s NUMBER correspondence and requests: 551
+s NUMBER correspondence and requests for: 551
+online in the supporting information section: 550
+public , commercial , or not-for-profit: 549
+found online at https :// www.frontiersin.org: 549
+online at https :// www.frontiersin.org /: 549
+at https :// www.frontiersin.org / articles: 548
+https :// www.frontiersin.org / articles /: 548
+", distribution , and reproduction in": 546
+distribution , and reproduction in any: 546
+", and reproduction in any medium": 546
+:// www.frontiersin.org / articles / NUMBER: 545
+www.frontiersin.org / articles / NUMBER /: 544
+contribution to the peer review of: 542
+grant from funding agencies in the: 541
+use , distribution , and reproduction: 537
+", commercial , or not-for-profit sectors": 535
+received no financial support for the: 535
+no financial support for the research: 534
+receive any specific grant from funding: 529
+specific grant from funding agencies in: 529
+approved the final version of the: 529
+any specific grant from funding agencies: 528
+author ( s ) declared no: 526
+is available online at https ://: 525
+( s ) declared no potential: 525
+s ) declared no potential conflicts: 525
+) declared no potential conflicts of: 525
+article is distributed under the terms: 520
+this article is distributed under the: 518
+"% and NUMBER % , respectively": 507
+access this article is distributed under: 501
+open access this article is distributed: 498
+authors declare that there is no: 487
+the following is available online at: 485
+NUMBER % , NUMBER % ,: 484
+authors would like to thank the: 479
+is a open access article under: 477
+a open access article under the: 477
+medium , provided the original work: 473
+/ NUMBER /) , which permits: 471
+the online version , at doi: 466
+interest was reported by the author: 464
+supplementary material associated with this article: 452
+material associated with this article can: 452
+we would like to thank all: 449
+http :// creat iveco mmons.org /: 448
+:// creat iveco mmons.org / licen: 446
+creat iveco mmons.org / licen ses: 446
+iveco mmons.org / licen ses /: 446
+following is available online at https: 446
+available online at https :// www.mdpi.com: 445
+online at https :// www.mdpi.com /: 445
+union's horizon NUMBER research and innovation: 443
+during the current study is available: 443
+european union's horizon NUMBER research and: 441
+the netherlands organization for scientific research: 441
+the authors declare that there is: 441
+NUMBER international license ( http ://: 440
+of the authors and do not: 439
+www.mdpi.com / article / NUMBER /: 438
+NUMBER CITATION , NUMBER CITATION ,: 436
+access article under the terms of: 435
+article under the terms of the: 435
+/ by / NUMBER /) ,: 435
+by / NUMBER /) , which: 435
+/ licen ses / by /: 434
+licen ses / by / NUMBER: 434
+open access article under the terms: 434
+current study is available from the: 434
+the current study is available from: 433
+", which permits unrestricted use ,": 431
+mmons.org / licen ses / by: 429
+have no conflicts of interest to: 427
+which permits unrestricted use , distribution: 427
+", there is a need for": 425
+permits unrestricted use , distribution ,: 425
+unrestricted use , distribution , and: 425
+have read and agreed to the: 423
+authors have read and agreed to: 422
+any medium , provided you give: 422
+medium , provided you give appropriate: 422
+", provided you give appropriate credit": 422
+provided you give appropriate credit to: 422
+was reported by the author (: 422
+reported by the author ( s: 422
+NUMBER /) , which permits unrestricted: 421
+in any medium , provided you: 421
+peer review information nature communications thanks: 421
+the european union's horizon NUMBER research: 420
+and agreed to the published version: 419
+agreed to the published version of: 419
+the published version of the manuscript: 419
+to the published version of the: 418
+the authors and do not necessarily: 417
+/) , which permits unrestricted use: 417
+read and agreed to the published: 416
+those of the authors and do: 414
+all authors have read and agreed: 413
+that there is no conflict of: 411
+attribution NUMBER international license ( http: 410
+author ( s ) received no: 405
+consent was obtained from all subjects: 404
+authors have no conflicts of interest: 403
+( s ) received no financial: 403
+s ) received no financial support: 403
+) received no financial support for: 403
+NUMBER % and NUMBER % of: 400
+contributed to the article and approved: 400
+to the article and approved the: 400
+authors contributed to the article and: 397
+the author ( s ) received: 396
+the article and approved the submitted: 396
+article and approved the submitted version: 395
+informed consent statement informed consent was: 393
+accountable for all aspects of the: 389
+https :// www.mdpi.com / article /: 389
+:// www.mdpi.com / article / NUMBER: 389
+consent statement informed consent was obtained: 389
+at https :// www.mdpi.com / article: 388
+statement informed consent was obtained from: 388
+no role in the design of: 385
+) for their contribution to the: 383
+", we would like to thank": 381
+is available on request from the: 378
+had no role in study design: 377
+responsibility for the integrity of the: 375
+of the creative commons attribution NUMBER: 375
+in the supporting information section at: 374
+the supporting information section at the: 374
+supporting information section at the end: 374
+reviewer ( s ) for their: 374
+this study is available on request: 373
+all authors contributed to the article: 373
+( s ) for their contribution: 372
+s ) for their contribution to: 372
+", analysis and interpretation of data": 371
+available on request from the corresponding: 370
+in accordance with the ethical standards: 370
+the manuscript for important intellectual content: 368
+for all aspects of the work: 367
+role in the design of the: 367
+authors declare no conflicts of interest: 366
+", analysis , and interpretation of": 365
+on request from the corresponding author: 364
+be accountable for all aspects of: 362
+obtained from all subjects involved in: 361
+funders had no role in the: 360
+for scientific research ( nwo ): 359
+the creative commons attribution NUMBER international: 359
+was obtained from all subjects involved: 359
+authors and do not necessarily represent: 358
+to be accountable for all aspects: 356
+original work is properly cited ,: 355
+data in the writing of the: 354
+no role in study design ,: 353
+from all subjects involved in the: 353
+all subjects involved in the study: 351
+provenance and peer review not commissioned: 346
+study is available on request from: 346
+there is no conflict of interest: 346
+wrote the first draft of the: 345
+NUMBER / NUMBER / NUMBER /: 345
+written informed consent to participate in: 341
+commons attribution NUMBER international license (: 341
+interpretation of data in the writing: 339
+of data in the writing of: 339
+ses / by / NUMBER /: 337
+we would also like to thank: 336
+the integrity of the data and: 335
+and peer review not commissioned externally: 334
+of this study was to assess: 334
+peer review not commissioned externally peer: 332
+review not commissioned externally peer reviewed: 332
+declare that there is no conflict: 331
+role in study design , data: 328
+written informed consent was obtained from: 327
+no conflicts of interest to declare: 326
+consent to participate in this study: 326
+in the article's creative commons license: 326
+do not necessarily represent those of: 325
+( http :// creativecommons.org / licenses: 324
+the authors have no conflicts of: 322
+informed consent to participate in this: 322
+with the ethical standards of the: 322
+license ( http :// creativecommons.org /: 322
+solely those of the authors and: 322
+and do not necessarily represent those: 322
+in this article is solely those: 320
+this article is solely those of: 320
+article is solely those of the: 320
+integrity of the data and the: 319
+of the data and the accuracy: 319
+the data and the accuracy of: 319
+accordance with the ethical standards of: 319
+data and the accuracy of the: 318
+during the conduct of the study: 317
+the creative commons attribution CITATION ,: 317
+expressed in this article is solely: 316
+and approved the final version of: 315
+publisher's note all claims expressed in: 315
+note all claims expressed in this: 315
+all claims expressed in this article: 315
+claims expressed in this article is: 315
+not necessarily represent those of their: 315
+necessarily represent those of their affiliated: 315
+represent those of their affiliated organizations: 315
+or those of the publisher ,: 315
+those of the publisher , the: 314
+of the publisher , the editors: 314
+in the writing of the manuscript: 313
+affiliated organizations , or those of: 312
+organizations , or those of the: 312
+or claim that may be made: 312
+claim that may be made by: 312
+that may be made by its: 312
+is not guaranteed or endorsed by: 312
+not guaranteed or endorsed by the: 312
+guaranteed or endorsed by the publisher: 312
+authors would like to thank all: 311
+related to this article can be: 311
+those of their affiliated organizations ,: 311
+of their affiliated organizations , or: 311
+their affiliated organizations , or those: 311
+", or those of the publisher": 311
+any product that may be evaluated: 311
+this article , or claim that: 311
+article , or claim that may: 311
+", or claim that may be": 311
+product that may be evaluated in: 310
+may be made by its manufacturer: 310
+the accuracy of the data analysis: 309
+", there is a need to": 309
+of the creative commons attribution license: 309
+that may be evaluated in this: 309
+may be evaluated in this article: 309
+be made by its manufacturer ,: 309
+made by its manufacturer , is: 309
+by its manufacturer , is not: 309
+its manufacturer , is not guaranteed: 309
+manufacturer , is not guaranteed or: 309
+", is not guaranteed or endorsed": 309
+", there is a lack of": 308
+the authors declare no conflicts of: 308
+be evaluated in this article ,: 308
+evaluated in this article , or: 308
+in this article , or claim: 308
+the data in the study and: 306
+in this paper , we present: 306
+information may be found in the: 305
+this study was to investigate the: 305
+", on the one hand ,": 305
+may be found in the online: 304
+is not publicly available due to: 304
+authors report no conflicts of interest: 304
+online version , at doi NUMBER: 303
+version , at doi NUMBER /: 303
+supporting information may be found in: 302
+in the public , commercial or: 301
+from any funding agency in the: 300
+any funding agency in the public: 300
+were in accordance with the ethical: 300
+the publisher , the editors and: 300
+publisher , the editors and the: 300
+", the editors and the reviewers": 300
+funding agency in the public ,: 299
+agency in the public , commercial: 299
+found in the online version of: 297
+there is no conflicts of interest: 295
+can be found in the online: 295
+article can be found in the: 291
+use , distribution and reproduction in: 289
+the online version of this article: 287
+before it is published in its: 287
+it is published in its final: 287
+to submit the manuscript for publication: 286
+international license ( http :// creativecommons.org: 285
+is published in its final form: 285
+decision to submit the manuscript for: 284
+", which permits use , distribution": 284
+which permits use , distribution and: 284
+permits use , distribution and reproduction: 284
+", as shown in fig. NUMBER": 284
+errors may be discovered which could: 284
+may be discovered which could affect: 284
+be discovered which could affect the: 284
+discovered which could affect the content: 284
+which could affect the content ,: 284
+could affect the content , and: 284
+affect the content , and all: 284
+the content , and all legal: 284
+content , and all legal disclaimers: 284
+all legal disclaimers that apply to: 283
+legal disclaimers that apply to the: 283
+disclaimers that apply to the journal: 283
+design , data collection and analysis: 282
+", and all legal disclaimers that": 282
+and all legal disclaimers that apply: 282
+from the european union's horizon NUMBER: 280
+and the other , anonymous ,: 280
+is a pdf file of a: 278
+this article can be found in: 277
+helsinki declaration and its later amendments: 276
+NUMBER helsinki declaration and its later: 275
+the NUMBER helsinki declaration and its: 274
+during the production process , errors: 274
+the production process , errors may: 274
+production process , errors may be: 274
+process , errors may be discovered: 274
+this is a pdf file of: 273
+review before it is published in: 273
+", errors may be discovered which": 273
+will undergo additional copyediting , typesetting: 271
+and review before it is published: 271
+note that , during the production: 271
+that , during the production process: 271
+", during the production process ,": 271
+with the NUMBER helsinki declaration and: 270
+this version will undergo additional copyediting: 270
+version will undergo additional copyediting ,: 270
+undergo additional copyediting , typesetting and: 270
+additional copyediting , typesetting and review: 270
+copyediting , typesetting and review before: 270
+", typesetting and review before it": 270
+typesetting and review before it is: 270
+published in its final form ,: 270
+in its final form , but: 270
+its final form , but we: 270
+final form , but we is: 270
+form , but we is providing: 270
+", but we is providing this": 270
+but we is providing this version: 270
+we is providing this version to: 270
+is providing this version to give: 270
+providing this version to give early: 270
+this version to give early visibility: 270
+version to give early visibility of: 270
+to give early visibility of the: 270
+give early visibility of the article: 270
+please note that , during the: 270
+netherlands organization for scientific research (: 269
+acute respiratory syndrome coronavirus NUMBER CITATION: 268
+a pdf file of a article: 268
+pdf file of a article that: 268
+file of a article that has: 268
+of a article that has undergone: 268
+a article that has undergone enhancements: 268
+article that has undergone enhancements after: 268
+that has undergone enhancements after acceptance: 268
+has undergone enhancements after acceptance ,: 268
+undergone enhancements after acceptance , such: 268
+enhancements after acceptance , such as: 268
+after acceptance , such as the: 268
+acceptance , such as the addition: 268
+as the addition of a cover: 268
+the addition of a cover page: 268
+addition of a cover page and: 268
+of a cover page and metadata: 268
+a cover page and metadata ,: 268
+cover page and metadata , and: 268
+page and metadata , and formatting: 268
+and metadata , and formatting for: 268
+metadata , and formatting for readability: 268
+", and formatting for readability ,": 268
+and formatting for readability , but: 268
+formatting for readability , but it: 268
+for readability , but it is: 268
+readability , but it is not: 268
+but it is not yet the: 268
+it is not yet the definitive: 268
+is not yet the definitive version: 268
+not yet the definitive version of: 268
+yet the definitive version of record: 268
+and its later amendments or comparable: 267
+its later amendments or comparable ethical: 267
+the netherlands organisation for scientific research: 265
+analysis and interpretation of data ,: 265
+NUMBER https :// doi.org / NUMBER: 265
+later amendments or comparable ethical standards: 265
+the public , commercial or not-for-profit: 265
+the aim of this paper is: 263
+"% , NUMBER % and NUMBER": 262
+the other , anonymous , reviewer: 261
+and with the NUMBER helsinki declaration: 260
+human participants were reviewed and approved: 259
+participants were reviewed and approved by: 259
+involving human participants were reviewed and: 258
+studies involving human participants were reviewed: 257
+and analysis , decision to publish: 257
+it should be noted that the: 257
+in the decision to publish the: 256
+study design , data collection ,: 256
+found in the online version at: 256
+the authors report no conflicts of: 255
+under the terms of the creat: 255
+the terms of the creat ive: 255
+terms of the creat ive commo: 255
+of the creat ive commo ns: 255
+the creat ive commo ns attri: 255
+they have no conflicts of interest: 255
+data presented in this study is: 255
+participants provided their written informed consent: 254
+their written informed consent to participate: 254
+the decision to publish the results: 254
+the data presented in this study: 254
+provided their written informed consent to: 253
+CITATION license , which permits others: 252
+license , which permits others to: 252
+all authors approved the final version: 251
+that apply to the journal pertain: 251
+anonymous , reviewer ( s ): 251
+study is included in the article: 250
+other , anonymous , reviewer (: 250
+", anonymous , reviewer ( s": 250
+", reviewer ( s ) for": 250

great_ai/src/great_ai/utilities/publication_tei/titles_of_interest.py

@@ -0,0 +1,145 @@
+import re
+from typing import Dict, Pattern, Tuple, Union
+
+from .models import BookmarkTitle
+
+# partly inspired by https://github.com/KMCS-NII/AASC/blob/master/section_classify/section_classify.pl
+titles_of_interest: Dict[BookmarkTitle, Tuple[Union[str, Pattern], ...]] = {
+    "Abstract": ("abstract",),
+    "Author contribution": (
+        "author contribution",
+        "credit authorship contribution statement",
+        "credit author statement",
+        "corresponding author",
+    ),
+    "Introduction": (
+        "introduction",
+        "preliminaries",
+        "research question",
+        "objective",
+        "purpose",
+        re.compile(r"^aim$"),
+        "proposition",
+        "what this study adds",
+        "what does this study add",
+    ),
+    "Background": (
+        "background",
+        "previous",
+        "prior",
+        "relevant work",
+        "state of the art",
+        "related work",
+        "research in context",
+        "earlier work",
+        "related work",
+        "problem",
+        "challenge",
+        "goal",
+        "literature review",
+        re.compile(r"^review$"),
+        re.compile(r"^existing"),
+        "comparison with existing literature",
+        "literature search",
+        "comparison with other studies",
+        "study area",
+    ),
+    "Methods": (
+        "method",
+        "approach",
+        re.compile(r"^our"),
+        "proposed",
+        "study design",
+        "procedure",
+        "formulation",
+        "methodology",
+        "research design",
+        "study protocol",
+        "design",
+        re.compile(r"experimental (?!result)"),
+        "overview",
+        "approach",
+        "system",
+        "definition",
+        "algorithm",
+        "model",
+        "setup",
+        "implementation",
+        "evaluation metric",
+        "scheme",
+    ),
+    "Results": (
+        "result",
+        "data",
+        "outcome",
+        "statistic",
+        "statistics",
+        "statistical analysis",
+        "analysis",
+        "measure",
+        "experiment",
+        "finding",
+        "hypothesis testing",
+        "proof of proposition",
+        "proof of concept",
+        "report",
+        "implication of all the available evidence",
+        "contribution",
+        "evaluation",
+        "performance",
+        "demonstration",
+        "demonstrating",
+        "example",
+        "study",
+        "studies",
+        "qualitative comparison",
+        "quantitative comparison",
+        "validation",
+    ),
+    "Discussion": (
+        "discussion",
+        "recommendation",
+        "implication",
+        "findings",
+        "impact",
+        "limitation",
+        "assessment",
+        "quality assessment",
+        "interpretation",
+        "contribution",
+        "explanation",
+    ),
+    "Outlook": (
+        "outlook",
+        "future research",
+        "future",
+        "follow up",
+        "remaining",
+        "prospect",
+        "perspective",
+    ),
+    "Conclusion": (
+        "conclusion",
+        "conclude",
+        "concluding",
+        "concluding",
+        "final remark",
+        "remark",
+        "conclusie",
+    ),
+    "Conflict of interest": (
+        "compete interest",
+        "funding support and author disclosure",
+        "conflict of interest",
+        "disclosure statement",
+        "role of the funding source",
+        "disclaimer",
+        "conflicting interest",
+        "declaration of interest",
+        "risk of bias",
+        "funding",
+        "disclosure",
+        "declaration",
+    ),
+    "Acknowledgement": ("acknowledgement", "acknowledgment"),
+}

great_ai/src/great_ai/utilities/unique.py

@@ -4,7 +4,7 @@ from typing import Any, Callable, Iterable, List
 def unique(
     values: Iterable[Any], *, key: Callable[[Any], Any] = lambda v: v
 ) -> List[Any]:
-    """Only keep first occurrences and maintian order"""
+    """Only keep first occurrences and maintain order"""
 
     key_values = {}
     for v in values:

great_ai/tests/sus/test_clean.py

@@ -3,6 +3,7 @@ import unittest
 from src.great_ai.utilities.clean import clean
 
 
+
 class TestClean(unittest.TestCase):
     def test_xml_handling(self) -> None:
         xml = '<strong>Hi, </strong> my name<br/>is <span style="color: hotpink;"> András</span>! &lt;&#51; <> < ></><> &lt;&gt; <|'
@@ -59,6 +60,13 @@ class TestClean(unittest.TestCase):
 
         self.assertEqual(clean(text), cleaned)
 
+    def test_T_I_T_L_E_case(self) -> None:
+        text = "an a r t i  c l e is to F I N   D the purpose of a tree"
+
+        cleaned = "an article is to FIND the purpose of a tree"
+
+        self.assertEqual(clean(text), cleaned)
+
     def test_bracket_removal(self) -> None:
         text = "something [0], and [frefe, ferf]"
         cleaned = "something, and"

great_ai/tests/sus/test_get_sentences.py

@@ -3,6 +3,7 @@ import unittest
 from src.great_ai.utilities.get_sentences import get_sentences
 
 
+
 class TestGetSentences(unittest.TestCase):
     def test_default(self) -> None:
         text = "This is a complete sentence. So is this. However this is n"  # ot.

great_ai/tests/sus/test_lemmatize_text.py

@@ -57,7 +57,7 @@ class TestLemmatizeText(unittest.TestCase):
             "to_PART",
             "create_VERB",
             "a_DET",
-            "less_ADJ",
+            "less_ADV",
             "resource_NOUN",
             "-_PUNCT",
             "intensive_ADJ",
@@ -117,7 +117,7 @@ class TestLemmatizeText(unittest.TestCase):
             "to_PART",
             "create_VERB",
             "a_DET",
-            "less_ADJ",
+            "less_ADV",
             "resource_NOUN",
             "-_PUNCT",
             "intensive_ADJ",

great_ai/tests/sus/test_match_names.py

@@ -3,6 +3,7 @@ import unittest
 from src.great_ai.utilities.match_names.match_names import match_names
 
 
+
 class TestMatchNames(unittest.TestCase):
     def test_grid(self) -> None:
         names = [

great_ai/tests/sus/test_publication_tei.py

@@ -3,7 +3,17 @@ from pathlib import Path
 
 from src.great_ai.utilities.publication_tei import PublicationTEI
 
-from .data.parsed import authors, content, metadata, sentences
+
+from .data.parsed import (
+    abstract,
+    authors,
+    bookmarks,
+    conclusion,
+    content,
+    introduction,
+    metadata,
+    sentences,
+)
 
 DATA_PATH = Path(__file__).parent.resolve() / "data"
 
@@ -30,6 +40,20 @@ class TestPublicationTEI(unittest.TestCase):
     def test_sentences(self) -> None:
         self.assertEqual(PublicationTEI(self.test_xml).sentences, sentences)
 
+    def test_bookmarks(self) -> None:
+        self.assertEqual(PublicationTEI(self.test_xml).bookmarks, bookmarks)
+
+    def test_abstract(self) -> None:
+        self.assertEqual(PublicationTEI(self.test_xml).abstract_sentences, abstract)
+
+    def test_introduction(self) -> None:
+        self.assertEqual(
+            PublicationTEI(self.test_xml).introduction_sentences, introduction
+        )
+
+    def test_conclusion(self) -> None:
+        self.assertEqual(PublicationTEI(self.test_xml).conclusion_sentences, conclusion)
+
     def test_empty1(self) -> None:
         tei = PublicationTEI("<TEI/>")
         tei.publication_metadata, tei.publication_metadata, tei.authors, tei.content, tei.sentences

great_ai/tests/sus/test_unique.py

@@ -2,6 +2,7 @@ import unittest
 
 from src.great_ai.utilities.unique import unique
 
+
 original = [
     ("a", 1),
     ("b", 5),