Add files

Signed-off-by: András Schmelczer <andras@schmelczer.dev>

.gitignore

@@ -0,0 +1,4 @@
+.env
+.DS_Store
+__pycache__
+.cache

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+  "cSpell.words": ["pydantic", "pyplot", "sklearn", "Tfidf", "Vectorizer"]
+}

example/.gitignore

@@ -0,0 +1,2 @@
+data
+.ipynb_checkpoints

example/config.py

@@ -0,0 +1 @@
+model_key = 

example/helper/__init__.py

@@ -0,0 +1 @@
+from .preprocess import preprocess

example/helper/preprocess.py

@@ -0,0 +1,12 @@
+from sus.clean import clean
+from sus.lemmatize_text import lemmatize_text
+import re
+
+
+def preprocess(text: str) -> str:
+    cleaned = clean(text, convert_to_ascii=True)
+    lemmas = [
+        re.sub(r'\d+', 'NUM', lemma)
+        for lemma in lemmatize_text(cleaned)
+    ]
+    return " ".join(lemmas)

example/models/__init__.py

@@ -0,0 +1 @@
+from .domain_prediction import DomainPrediction

example/models/domain_prediction.py

@@ -0,0 +1,8 @@
+from pydantic import BaseModel
+from typing import List
+
+
+class DomainPrediction(BaseModel):
+    domain: str
+    probability: float
+    explanation: List[str]

example/predict.py

@@ -0,0 +1,56 @@
+
+from models import DomainPrediction
+from typing import Iterable, List, Dict
+from sklearn.pipeline import Pipeline 
+from helper import preprocess
+import re
+# from sus.use_model import use_model
+from config import model_key
+
+
+# @use_model(model_key, version="latest")
+def predict(text: str, model: Pipeline, cut_off_probability: float=0.2) -> List[DomainPrediction]:
+    assert 0 <= cut_off_probability <= 1
+
+    feature_names = model.named_steps['vectorizer'].get_feature_names_out()
+
+    token_mapping = {
+        preprocess(original): original
+        for original in re.sub(r'[^a-zA-Z0-9]', ' ', text).split(' ')
+    }
+    
+    features = model.named_steps['vectorizer'].transform([text])
+    prediction = model.named_steps['classifier'].predict_proba(features)[0]
+    best_classes = sorted(enumerate(prediction), key=lambda v: v[1], reverse=True)
+
+    results: List[DomainPrediction] = []
+    for class_index, probability in best_classes:
+        weights = model.named_steps['classifier'].feature_log_prob_[class_index]
+       
+        results.append(DomainPrediction(
+            domain=model.named_steps['classifier'].classes_[class_index],
+            probability=round(probability * 100),
+            explanation=_get_explanation(
+                feature_names=feature_names,
+                features=features.A[0],
+                weights=weights,
+                token_mapping=token_mapping
+            )
+        ))
+
+        if sum(r.probability for r in results) >= cut_off_probability:
+            break
+    
+    return results
+
+
+def _get_explanation(feature_names: Iterable[str], features: Iterable[float], weights: Iterable[float], token_mapping: Dict[str, str]) -> List[str]:
+    influential = [
+        (value * weight, name)
+        for name, value, weight in zip(feature_names, features, weights)
+        if value
+    ]
+    
+    most_influential = sorted(influential, reverse=True)[:5]
+
+    return [token_mapping[v[1]] for v in most_influential]

example/secrets-backblaze.ini

@@ -0,0 +1,6 @@
+[DEFAULT]
+aws_region_name = us-west-4
+aws_access_key_id = 00411a2454be9f90000000001
+aws_secret_access_key = K004YOSYHePVUHymVzm/vb+AjGahXl8
+large_files_bucket_name = sa-large-files
+endpoint_url = https://s3.us-west-004.backblazeb2.com

example/secrets.ini

@@ -0,0 +1,5 @@
+[DEFAULT]
+aws_region_name = eu-west-2
+aws_access_key_id = AKIAR7XRQBSIKKSKQA6U
+aws_secret_access_key = WPr6mqbtJjmcmibdz12nK6XwQQHsWiXvep7NUexJ
+large_files_bucket_name = sis-large-files

good_ai/.gitignore

@@ -0,0 +1 @@
+build

good_ai/README.md

@@ -0,0 +1,33 @@
+# **S**coutinScience **U**tilitie**S** for text processing [![Lint and test ScoutinScience utilities](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml/badge.svg)](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml)
+
+> amogus
+
+## Exports
+
+- [clean](src/sus/clean.py)
+- [unique](src/sus/unique.py)
+- [parallel_map](src/sus/parallel_map.py)
+- [match_names](src/sus/match_names/match_names.py)
+- [evaluate_ranking](src/sus/evaluate_ranking/evaluate_ranking.py)
+
+### Requires loading spacy model
+
+> This is automatic but will require some time.
+
+> Add this to the Dockerfile for caching the spaCy model:
+>
+> ```docker
+> RUN pip install --no-cache-dir en-core-web-lg@https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.2.0/en_core_web_lg-3.2.0-py3-none-any.whl
+> ```
+
+- [spacy model (nlp)](src/sus/nlp.py)
+- [get_sentences](src/sus/get_sentences.py)
+- [lemmatize_text](src/sus/lemmatize_text.py)
+- [lemmatize_token](src/sus/lemmatize_token.py)
+- [publication TEI](src/sus/publication_tei/publication_tei.py)
+
+## Development
+
+- Optional booleans must have a default value of `False`.
+- No imports in top-level `__init__.py`, in order to not load anything unnecessary automatically
+- Should only be updated through a PR

good_ai/example_secrets.ini

@@ -0,0 +1,6 @@
+[DEFAULT]
+aws_region_name = your_region_like_eu-west-2
+aws_access_key_id = YOUR_ACCESS_KEY_ID
+aws_secret_access_key = YOUR_VERY_SECRET_ACCESS_KEY
+large_files_bucket_name = create_a_bucket_and_put_its_name_here
+endpoint_url = this is optional, for backblaze, use this: https://s3.us-west-002.backblazeb2.com

good_ai/open_s3.md

@@ -0,0 +1,119 @@
+# [open(S3)](https://pypi.org/project/open-large/)
+
+Storing, versioning, and downloading files from S3 made as easy as using `open()` in Python. Caching included.
+
+## Motivation
+
+Oftentimes, especially when working with data-heavy applications, large files can proliferate in a repository. Version controlling them is an obvious next step, however, GitHub's git LFS implementation [doesn't support the deletion](https://docs.github.com/en/repositories/working-with-files/managing-large-files/removing-files-from-git-large-file-storage#git-lfs-objects-in-your-repository) of large files, making it easy for them to eat-up the LFS quota and explode the size of your repos.
+
+## Solution
+
+```
+pip install open-large
+```
+
+### Simple example
+
+```python
+from open_s3 import LargeFile
+
+LargeFile.configure_credentials({
+    "aws_region_name": "your_region_like_eu-west-2",
+    "aws_access_key_id": "YOUR_ACCESS_KEY_ID",
+    "aws_secret_access_key": "YOUR_VERY_SECRET_ACCESS_KEY",
+    "large_files_bucket_name": "create_a_bucket_and_put_its_name_here",
+})
+
+# Creates a new version and deletes the older version leaving the 3 most recently used intact
+with LargeFile("test.txt", "w", keep_last_n=3) as f:
+    for i in range(100000):
+        f.write('test\n')
+
+# By default the latest version is returned
+# but an optional `version` keyword argument can be provided as well
+with LargeFile("test.txt", "r") as f:
+    print(f.readlines()[0])
+```
+
+> Automatically creates a file, writes to it, uploads it to S3, and then queries the most recent version of it.
+> In this case, the latest version is already in the local cache, no download is required.
+
+### More details
+
+`LargeFile` behaves like an opened file (in the background it is a temp file after all). Binary reading and writing is supported along with the [different keywords](https://docs.python.org/3/library/functions.html#open) `open()` accepts.
+
+The local cache can be configured with these properties:
+
+```python
+LargeFile.cache_path = Path('.cache')
+LargeFile.max_cache_size = "30 GB"
+```
+
+#### I only need a path
+
+In case you only need a path to the "remote" file, this pattern can be applied:
+
+```python
+path_to_model = LargeFile("folder-of-my-bert-model", version=31).get()
+```
+
+> This will first download the file/folder into your local cache folder. Then, it returns a `Path` object to the local version. Which can be turned into a string with `str(path_to_model)`.
+
+The same approach works for uploads:
+
+```python
+LargeFile("folder-of-my-bert-model").push('path_to_local/folder_or_file')
+```
+
+> This way, both regular files and folders can be handled. The uploaded file is called **folder-of-my-bert-model**, the local name is ignored.
+
+Lastly, all version of the remote object can be deleted by calling `LargeFile("my-file").delete()`. It will still reside in your local cache afterwards, its deletion will happen next time your local cache has to be pruned.
+
+### Command-line example
+
+The package can be used as a module from the command-line to give you more flexibility.
+
+#### Setup
+
+Create an .ini file (or use _~/.aws/credentials_). It may look like this:
+
+```ini
+[DEFAULT]
+aws_region_name = your_region_like_eu-west-2
+aws_access_key_id = YOUR_ACCESS_KEY_ID
+aws_secret_access_key = YOUR_VERY_SECRET_ACCESS_KEY
+large_files_bucket_name = my_large_files
+endpoint_url = this is optional, for backblaze, use this: https://s3.us-west-002.backblazeb2.com
+```
+
+> Just like in [example secrets](example_secrets.ini).
+
+#### Print the expected options
+
+```sh
+python3 -m open_s3 --help
+```
+
+#### Upload some files
+
+```sh
+python3 -m open_s3 --secrets secrets.ini --push my_first_file.json folder/my_second_file my_folder
+```
+
+> Only the filename is used as the S3 name, the rest of the path is ignored.
+
+#### Download some files to the local cache
+
+This can be useful when building a Docker image for example. This way, the files can already reside inside the container and need not be downloaded later.
+
+```sh
+python3 -m open_s3 --secrets ~/.aws/credentials --cache my_first_file.json:3 my_second_file my_folder:0
+```
+
+> Versions may be specified by using `:`-s.
+
+#### Delete remote files
+
+```sh
+python3 -m open_s3 --secrets ~/.aws/credentials --delete my_first_file.json
+```

good_ai/pyproject.toml

@@ -0,0 +1,7 @@
+[build-system]
+requires = [
+    "setuptools>=42",
+    "setuptools-git",
+    "wheel"
+]
+build-backend = "setuptools.build_meta"

good_ai/requirements.txt

@@ -0,0 +1,11 @@
+unidecode >= 1.3.0
+multiprocess >= 0.70.0.0
+tqdm >= 4.0.0
+psutil >= 5.9.0
+beautifulsoup4 >= 4.10.0
+lxml >= 4.6.0
+spacy >= 3.2.0
+pydantic >= 1.8.0
+scikit-learn >= 1.0.0
+matplotlib >= 3.5.0
+numpy >= 1.22.0

good_ai/setup.cfg

@@ -0,0 +1,39 @@
+[metadata]
+name = sus
+version = 0.0.3
+author = ScoutinScience B.V.
+author_email = andras@scoutinscience.com
+description = [S]coutinScience [u]tilitie[s]: reusable utilities for text processing
+long_description = file: README.md
+long_description_content_type = text/markdown
+url = https://github.com/ScoutinScience/platform
+project_urls =
+    Bug Tracker = https://github.com/ScoutinScience/platform/issues
+classifiers =
+    Programming Language :: Python :: 3
+    Operating System :: OS Independent
+
+[options]
+package_dir =
+    = src
+packages = find:
+include_package_data = True
+python_requires = >=3.8
+install_requires =
+    click < 8.1.0
+    unidecode >= 1.3.0
+    multiprocess >= 0.70.0.0
+    tqdm >= 4.0.0
+    psutil >= 5.9.0
+    beautifulsoup4 >= 4.10.0
+    lxml >= 4.6.0
+    spacy >= 3.2.0
+    pydantic >= 1.8.0
+    scikit-learn >= 1.0.0
+    matplotlib >= 3.5.0
+    numpy >= 1.22.0
+    langcodes[data] >= 3.3.0
+    langdetect >= 1.0.9
+
+[options.packages.find]
+where = src

good_ai/src/__init__.py

@@ -0,0 +1 @@
+

good_ai/src/open_s3/__init__.py

@@ -0,0 +1 @@
+from .large_file import LargeFile

good_ai/src/open_s3/__main__.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+
+import logging
+from pathlib import Path
+
+from large_file import LargeFile
+from parse_arguments import parse_arguments
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    parser, args = parse_arguments()
+
+    LargeFile.configure_credentials_from_file(args.secrets)
+
+    if not args.cache and not args.push and not args.delete:
+        logging.warn("No action required.")
+        parser.print_help()
+    try:
+        if args.cache:
+            for c in args.cache:
+                split = c.split(":")
+                file_name = split[0]
+                version = None if len(split) == 1 else int(split[1])
+                LargeFile(file_name, "r", version=version).get()
+
+        if args.push:
+            for p in args.push:
+                path = Path(p)
+                LargeFile(path.name, "w").push(path)
+
+        if args.delete:
+            for f in args.delete:
+                LargeFile(f).delete()
+    except Exception as e:
+        logging.error(e)

good_ai/src/open_s3/helper/__init__.py

@@ -0,0 +1,2 @@
+from .progress_bar import DownloadProgressBar, UploadProgressBar
+from .human_readable_to_byte import human_readable_to_byte

good_ai/src/open_s3/helper/human_readable_to_byte.py

@@ -0,0 +1,31 @@
+import re
+
+
+def human_readable_to_byte(size: str) -> int:
+    """Case is ignored, kb, kB, Kb, and KB are all treated as kilobyte."""
+
+    if size.strip() == "0":
+        return 0
+
+    possible_units = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
+    units_re = "|".join(possible_units)
+    regex = re.compile(
+        rf"""
+        \s*                         # trim
+        (?P<scalar>\d+(.\d+)?)      # get scalar, it might be a float
+        \s*                         # ignore optional whitespace
+        (?P<unit>{units_re})        # capture the unit
+    """,
+        flags=re.VERBOSE | re.IGNORECASE,
+    )
+
+    match = regex.match(size)
+    if not match:
+        raise ValueError(f'Could not find values in "{size}"')
+
+    results = match.groupdict()
+
+    scalar = float(results["scalar"])
+    idx = possible_units.index(results["unit"].upper())
+    factor = 1024 ** idx
+    return round(scalar * factor)

good_ai/src/open_s3/helper/progress_bar.py

@@ -0,0 +1,40 @@
+import os
+import threading
+from logging import Logger
+from typing import IO, Any, Optional
+
+from tqdm.auto import tqdm
+from pathlib import Path
+
+import os
+import sys
+import threading
+
+
+class ProgressBar:
+    def __init__(self, file_size: int, logger: Logger, prefix: str):
+        self._file_size = file_size
+        self._logger = logger
+        self._prefix=prefix
+
+        self._seen_so_far = 0
+        self._lock = threading.Lock()
+
+    def __call__(self, bytes_amount: int):
+        with self._lock:
+            self._seen_so_far += bytes_amount
+            percentage = (self._seen_so_far / float(self._file_size)) * 100
+            size_length = len(str(self._file_size))
+            progress = str(self._seen_so_far).rjust(size_length)
+            self._logger.info(f"{self._prefix} {progress}/{self._file_size} bytes ({percentage:.1f}%)")
+
+
+class DownloadProgressBar(ProgressBar):
+    def __init__(self, name: str, size: int, logger: Logger):
+        super().__init__(file_size=size, logger=logger, prefix=f'Downloading {name}')
+
+
+class UploadProgressBar(ProgressBar):
+    def __init__(self, path: Path, logger: Logger):
+        size = os.path.getsize(path)
+        super().__init__(file_size=size, logger=logger, prefix=f'Uploading {path.name}')

good_ai/src/open_s3/large_file.py

@@ -0,0 +1,346 @@
+import configparser
+import logging
+import os
+import shutil
+import tempfile
+from pathlib import Path
+from types import TracebackType
+from typing import IO, Any, Dict, List, Optional, Type, Union
+import boto3
+from botocore.exceptions import ClientError
+
+from helper import DownloadProgressBar, UploadProgressBar, human_readable_to_byte
+
+logger = logging.getLogger("open_s3")
+
+
+class LargeFile:
+    """
+    Store large files in S3. Use local cache for speed up.
+
+    Examples:
+
+    ```
+    with LargeFile("test.txt", "w", keep_last_n=3) as f:
+        for i in range(1000000):
+            f.write('test\n')
+
+    with LargeFile("test.txt", "r") as f:
+        print(f.readlines()[0])
+
+    path_to_cached_text_file = LargeFile("test.txt", version=0).get()
+    ```
+
+    By default, files are stored in the ".cache" folder and the
+    least recently use is deleted after the overall size reaches 30 GBs.
+
+    Change it with the following properties.
+
+    ```
+    LargeFile.cache_path = Path(".cache")
+    LargeFile.max_cache_size = "30GB"
+    ```
+    """
+
+    region_name = None
+    access_key_id = None
+    secret_access_key = None
+    bucket_name = None
+    endpoint_url = None
+
+    cache_path = Path(".cache")
+    max_cache_size: Optional[str] = "30GB"
+
+    def __init__(
+        self,
+        name: str,
+        mode: str = "r",
+        *,
+        buffering: int = -1,
+        encoding: Optional[str] = None,
+        errors: Optional[str] = None,
+        newline: Optional[str] = None,
+        version: Optional[int] = None,
+        keep_last_n: Optional[int] = None,
+        offline_mode: bool = False
+    ):
+        self._name: str = name
+        self._version = version
+        self._mode: str = mode
+        self._keep_last_n = keep_last_n
+        self._offline_mode = offline_mode
+
+        self._buffering = buffering
+        self._encoding = encoding
+        self._errors = errors
+        self._newline = newline
+
+        LargeFile.cache_path.mkdir(parents=True, exist_ok=True)
+
+        self._find_versions()
+        self._check_mode_and_set_version()
+        
+    @classmethod
+    def configure_credentials(
+        cls,
+        *,
+        aws_region_name: str,
+        aws_access_key_id: str,
+        aws_secret_access_key: str,
+        large_files_bucket_name: str,
+        endpoint_url: Optional[str] = None,
+        **_: Dict[str, Any],
+    ) -> None:
+        cls.region_name = aws_region_name
+        cls.access_key_id = aws_access_key_id
+        cls.secret_access_key = aws_secret_access_key
+        cls.bucket_name = large_files_bucket_name
+        cls.endpoint_url = endpoint_url
+
+    @classmethod
+    def configure_credentials_from_file(
+        cls,
+        secrets_path: Union[Path, str],
+    ) -> None:
+
+        if isinstance(secrets_path, str):
+            secrets_path = Path(secrets_path)
+
+        if not secrets_path.exists():
+            raise FileNotFoundError(secrets_path.resolve())
+
+        credentials = configparser.ConfigParser()
+        credentials.read(secrets_path)
+        credentials.default_section
+        cls.configure_credentials(**credentials[credentials.default_section])
+
+    def __enter__(self) -> IO:
+        self._file: IO[Any] = (
+            tempfile.NamedTemporaryFile(
+                mode=self._mode,
+                buffering=self._buffering,
+                encoding=self._encoding,
+                newline=self._newline,
+                errors=self._errors,
+                delete=False,
+                prefix="large-file-",
+            )
+            if "w" in self._mode
+            else open(
+                self.get(),
+                mode=self._mode,
+                buffering=self._buffering,
+                encoding=self._encoding,
+                newline=self._newline,
+                errors=self._errors,
+            )
+        )
+
+        return self._file
+
+    def __exit__(
+        self,
+        type: Optional[Type[BaseException]],
+        exc: Optional[BaseException],
+        traceback: Optional[TracebackType],
+    ) -> bool:
+        self._file.close()
+
+        if type is None:
+            if "w" in self._mode:
+                self.push(Path(self._file.name))
+                os.unlink(self._file.name)
+        else:
+            logger.exception("Could not finish operation.")
+
+        return True
+    
+    @property
+    def version_ids(self) -> List[int]:
+        return [self._get_version_from_key(key) for key in self._versions]
+
+    def get(self, hide_progress: bool = False) -> Path:
+        key = next(
+            key
+            for key in self._versions
+            if self._get_version_from_key(key) == self._version
+        )
+
+        destination = self.cache_path / self._local_name
+        if not destination.exists():
+            logger.info(
+                f"File {self._local_name} does not exist locally, starting download from S3"
+            )
+
+            with tempfile.TemporaryDirectory() as tmp:
+                tmp_file_archive = Path(tmp) / f"{self._local_name}.tar.gz"
+
+                size = self._client.head_object(Bucket=self.bucket_name, Key=key)['ContentLength']
+                self._client.download_file(Bucket=self.bucket_name, Key=key, Filename=str(tmp_file_archive), Callback=None if hide_progress else DownloadProgressBar(size=size, name=key, logger=logger))
+                logger.info(f"Decompressing {self._local_name}")
+                shutil.unpack_archive(str(tmp_file_archive), tmp, "gztar")
+                tmp_file = Path(tmp) / self._local_name
+                shutil.move(str(tmp_file), str(destination))
+        else:
+            logger.info(f"File {self._local_name} found in cache")
+
+        return destination
+
+    def push(self, path: Union[Path, str], hide_progress: bool = False) -> None:
+        if isinstance(path, str):
+            path = Path(path)
+
+        with tempfile.TemporaryDirectory() as tmp:
+            if path.is_file():
+                logger.info(f"Copying file for {self._local_name}")
+                copy = shutil.copy 
+            else:
+                logger.info(f"Copying directory for {self._local_name}")
+                copy = shutil.copytree
+
+            try:
+                # Make local copy in the cache
+                copy(str(path), str(self.cache_path / self._local_name))
+            except shutil.SameFileError:
+                pass  # No worries
+
+            copy(str(path), str(Path(tmp) / self._local_name))
+
+            logger.info(f"Compressing {self._local_name}")
+            shutil.make_archive(
+                str(Path(tmp) / self._local_name),
+                "gztar",
+                tmp,
+            )
+
+            logger.info(f"Uploading {self._local_name} to S3 from {path}")
+
+            file_to_be_uploaded = Path(tmp) / f"{self._local_name}.tar.gz"
+            self._client.upload_file(Filename=str(file_to_be_uploaded), Bucket=self.bucket_name, Key=self._s3_name, Callback=None if hide_progress else UploadProgressBar(file_to_be_uploaded, logger=logger))
+
+        self.clean_up()
+
+    def delete(self) -> None:
+        self._keep_last_n = 0
+        self._delete_old_versions_from_s3()
+
+    def clean_up(self) -> None:
+        self._delete_old_versions_from_s3()
+        self._delete_old_versions_from_disk()
+
+    def _create_client(self) -> None:
+        if (
+            self.region_name is None
+            or self.access_key_id is None
+            or self.secret_access_key is None
+            or self.bucket_name is None
+        ):
+            raise ValueError(
+                "Please configure the S3 access options by calling LargeFile.configure_credentials or set offline_mode=True in the constructor."
+            )
+
+        self._client = boto3.client('s3', aws_access_key_id=self.access_key_id, aws_secret_access_key=self.secret_access_key, region_name=self.region_name, endpoint_url=self.endpoint_url)
+        
+    def _find_versions(self) -> None:
+        if self._offline_mode:
+            self._fetch_versions_from_cache()
+        else:
+            self._create_client()
+            self._fetch_versions_from_s3()
+        
+        if self._versions:
+            logger.info(f"Found versions: {self.version_ids}")
+        else:
+            logger.info("No versions found")
+
+    def _fetch_versions_from_cache(self) -> None:
+        logger.info(f"Fetching offline versions of {self._name}")
+
+        self._versions = [
+            path
+            for path in self.cache_path.glob(f'{self._local_name}-*')
+        ]
+
+    def _fetch_versions_from_s3(self) -> None:
+        logger.info(f"Fetching online versions of {self._name}")
+        found_objects = self._client.list_objects_v2(
+            Bucket=self.bucket_name, Prefix=self._name
+        )
+        self._versions = (
+            sorted(
+                o["Key"]
+                for o in found_objects["Contents"]
+            )
+            if "Contents" in found_objects
+            else []
+        )
+
+    def _check_mode_and_set_version(self) -> None:
+        if "+" in self._mode:
+            raise ValueError("Read-write mode is not allowed.")
+
+        if "w" in self._mode:
+            if self._version is not None:
+                raise ValueError("Providing a version is not allowed in write mode.")
+
+            self._version = self.version_ids[-1] + 1 if self.version_ids else 0
+
+        elif "r" in self._mode:
+            if not self.version_ids:
+                raise FileNotFoundError(
+                    f"File {self._name} not found. No versions are available."
+                )
+
+            if self._version is None:
+                self._version = self.version_ids[-1]
+                logger.info(f"Lastest version of {self._local_name} is {self._version}")
+
+            elif self._version not in self.version_ids:
+                raise FileNotFoundError(
+                    f"File {self._name} not found with version {self._version}. Available versions: {self.version_ids}"
+                )
+        else:
+            raise ValueError("Unsupported file mode.")
+
+    @property
+    def _local_name(self) -> str:
+        return f"{self._name}-{self._version}"
+
+    @property
+    def _s3_name(self) -> str:
+        return f"{self._name}/{self._version}"
+        
+    @staticmethod
+    def _get_version_from_key(key: Union[str, Path]) -> int:
+        if isinstance(key, Path):
+            return int(key.name.split("-")[-1])
+        return int(key.split("/")[-1])
+
+
+    def _delete_old_versions_from_s3(self) -> None:
+        if self._keep_last_n is not None:
+            for key in (self._versions[: -self._keep_last_n] if self._keep_last_n > 0 else self._versions):
+                logger.info(
+                    f"Removing old version (keep_last_n={self._keep_last_n}): {key}"
+                )
+                self._client.delete_object(Bucket=self.bucket_name, Key=key)
+                
+    def _delete_old_versions_from_disk(self) -> None:
+        self.cache_path.mkdir(parents=True, exist_ok=True)
+
+        if self.max_cache_size is None:
+            return
+
+        allowed_size = human_readable_to_byte(self.max_cache_size)
+        assert allowed_size >= 0
+
+        least_recently_read = sorted(
+            [f for f in self.cache_path.glob("*")], key=lambda f: f.stat().st_atime
+        )
+
+        while sum(os.path.getsize(f) for f in least_recently_read) > allowed_size:
+            file = least_recently_read.pop(0)
+            logger.info(
+                f"Deleting file from cache to meet quota (max_cache_size={self.max_cache_size}): {file}"
+            )
+            os.unlink(file)

good_ai/src/open_s3/parse_arguments.py

@@ -0,0 +1,48 @@
+from argparse import ArgumentParser, Namespace
+from typing import Tuple
+
+
+def parse_arguments() -> Tuple[ArgumentParser, Namespace]:
+    parser = ArgumentParser(
+        description="Store and version large files in S3; open them like regular files. Caching included.",
+    )
+
+    parser.add_argument(
+        "-s",
+        "--secrets",
+        type=str,
+        help="path to an .ini configration file with your S3 credentials",
+        required=True,
+    )
+
+    parser.add_argument(
+        "-c",
+        "--cache",
+        nargs="+",
+        type=str,
+        help="download file into local cache, example: file_name:version",
+        required=False,
+    )
+
+    parser.add_argument(
+        "-p",
+        "--push",
+        nargs="+",
+        type=str,
+        help="push a local file into S3 and set it as the most recent version",
+        required=False,
+    )
+
+    parser.add_argument(
+        "-d",
+        "--delete",
+        nargs="+",
+        type=str,
+        help="delete every version of file from S3",
+        required=False,
+    )
+
+    parser.print_usage = parser.print_help  # type: ignore
+    args = parser.parse_args()
+
+    return parser, args

good_ai/src/sus.egg-info/PKG-INFO

@@ -0,0 +1,50 @@
+Metadata-Version: 2.1
+Name: sus
+Version: 0.0.3
+Summary: [S]coutinScience [u]tilitie[s]: reusable utilities for text processing
+Home-page: https://github.com/ScoutinScience/platform
+Author: ScoutinScience B.V.
+Author-email: andras@scoutinscience.com
+License: UNKNOWN
+Project-URL: Bug Tracker, https://github.com/ScoutinScience/platform/issues
+Platform: UNKNOWN
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+
+# **S**coutinScience **U**tilitie**S** for text processing [![Lint and test ScoutinScience utilities](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml/badge.svg)](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml)
+
+> amogus
+
+## Exports
+
+- [clean](src/sus/clean.py)
+- [unique](src/sus/unique.py)
+- [parallel_map](src/sus/parallel_map.py)
+- [match_names](src/sus/match_names/match_names.py)
+- [evaluate_ranking](src/sus/evaluate_ranking/evaluate_ranking.py)
+
+### Requires loading spacy model
+
+> This is automatic but will require some time.
+
+> Add this to the Dockerfile for caching the spaCy model:
+>
+> ```docker
+> RUN pip install --no-cache-dir en-core-web-lg@https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.2.0/en_core_web_lg-3.2.0-py3-none-any.whl
+> ```
+
+- [spacy model (nlp)](src/sus/nlp.py)
+- [get_sentences](src/sus/get_sentences.py)
+- [lemmatize_text](src/sus/lemmatize_text.py)
+- [lemmatize_token](src/sus/lemmatize_token.py)
+- [publication TEI](src/sus/publication_tei/publication_tei.py)
+
+## Development
+
+- Optional booleans must have a default value of `False`.
+- No imports in top-level `__init__.py`, in order to not load anything unnecessary automatically
+- Should only be updated through a PR
+
+

good_ai/src/sus.egg-info/SOURCES.txt

@@ -0,0 +1,78 @@
+.gitignore
+README.md
+pyproject.toml
+requirements.txt
+setup.cfg
+src/__init__.py
+src/sus/__init__.py
+src/sus/clean.py
+src/sus/get_sentences.py
+src/sus/lemmatize_text.py
+src/sus/lemmatize_token.py
+src/sus/nlp.py
+src/sus/parallel_map.py
+src/sus/unique.py
+src/sus.egg-info/PKG-INFO
+src/sus.egg-info/SOURCES.txt
+src/sus.egg-info/dependency_links.txt
+src/sus.egg-info/requires.txt
+src/sus.egg-info/top_level.txt
+src/sus/data/__init__.py
+src/sus/data/american_spellings.py
+src/sus/data/punctuations.py
+src/sus/evaluate_ranking/__init__.py
+src/sus/evaluate_ranking/draw_f1_iso_lines.py
+src/sus/evaluate_ranking/evaluate_ranking.py
+src/sus/external/__init__.py
+src/sus/external/negspacy/README.md
+src/sus/external/negspacy/__init__.py
+src/sus/external/negspacy/negation.py
+src/sus/external/negspacy/termsets.py
+src/sus/external/pylatexenc/README.md
+src/sus/external/pylatexenc/__init__.py
+src/sus/external/pylatexenc/_util.py
+src/sus/external/pylatexenc/version.py
+src/sus/external/pylatexenc/latex2text/__init__.py
+src/sus/external/pylatexenc/latex2text/__main__.py
+src/sus/external/pylatexenc/latex2text/_defaultspecs.py
+src/sus/external/pylatexenc/latexencode/__init__.py
+src/sus/external/pylatexenc/latexencode/__main__.py
+src/sus/external/pylatexenc/latexencode/_partial_latex_encoder.py
+src/sus/external/pylatexenc/latexencode/_uni2latexmap.py
+src/sus/external/pylatexenc/latexencode/_uni2latexmap_xml.py
+src/sus/external/pylatexenc/latexencode/_unicode_to_latex_encoder.py
+src/sus/external/pylatexenc/latexwalker/__init__.py
+src/sus/external/pylatexenc/latexwalker/__main__.py
+src/sus/external/pylatexenc/latexwalker/_defaultspecs.py
+src/sus/external/pylatexenc/macrospec/__init__.py
+src/sus/external/pylatexenc/macrospec/_argparsers.py
+src/sus/language/__init__.py
+src/sus/language/english_name_of_language.py
+src/sus/language/is_english.py
+src/sus/language/predict_language.py
+src/sus/match_names/__init__.py
+src/sus/match_names/config.py
+src/sus/match_names/match_names.py
+src/sus/match_names/name_parts.py
+src/sus/publication_tei/__init__.py
+src/sus/publication_tei/publication_tei.py
+src/sus/publication_tei/models/__init__.py
+src/sus/publication_tei/models/affiliation.py
+src/sus/publication_tei/models/author.py
+src/sus/publication_tei/models/element.py
+src/sus/publication_tei/models/publication_metadata.py
+src/sus/publication_tei/models/text.py
+tests/__init__.py
+tests/test_clean.py
+tests/test_evaluate_ranking.py
+tests/test_get_sentences.py
+tests/test_language.py
+tests/test_lemmatize_text.py
+tests/test_lemmatize_token.py
+tests/test_match_names.py
+tests/test_parallel_map.py
+tests/test_publication_tei.py
+tests/test_unique.py
+tests/data/10.1136_bmjspcare-2021-003026.pdf.tei.xml
+tests/data/bad.tei.xml
+tests/data/parsed.py

good_ai/src/sus.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

good_ai/src/sus.egg-info/requires.txt

@@ -0,0 +1,14 @@
+click<8.1.0
+unidecode>=1.3.0
+multiprocess>=0.70.0.0
+tqdm>=4.0.0
+psutil>=5.9.0
+beautifulsoup4>=4.10.0
+lxml>=4.6.0
+spacy>=3.2.0
+pydantic>=1.8.0
+scikit-learn>=1.0.0
+matplotlib>=3.5.0
+numpy>=1.22.0
+langcodes[data]>=3.3.0
+langdetect>=1.0.9

good_ai/src/sus.egg-info/top_level.txt

@@ -0,0 +1 @@
+sus

good_ai/src/sus/clean.py

@@ -0,0 +1,64 @@
+import html
+import logging
+import re
+import unicodedata
+
+import unidecode
+
+from .data import left_regular_punctuations, right_regular_punctuations
+from .external.pylatexenc.latex2text import LatexNodes2Text
+
+logger = logging.getLogger("clean")
+latex = LatexNodes2Text()
+
+
+joined_left_punctuations = "".join(left_regular_punctuations).replace("]", r"\]")
+joined_right_punctuations = "".join(right_regular_punctuations).replace("[", r"\[")
+
+
+def clean(
+    text: str,
+    ignore_xml: bool = False,
+    ignore_latex: bool = False,
+    remove_brackets: bool = False,
+    convert_to_ascii: bool = False,
+) -> str:
+    if not ignore_xml:
+        text = re.sub(r"<[^>]*>", " ", text)
+        text = html.unescape(text)
+
+    if not ignore_latex:
+        text = text.replace("%", "\\%")  # escape LaTeX comments before parsing as LaTeX
+
+        try:
+            text = latex.latex_to_text(text, tolerant_parsing=True, strict_braces=False)
+            text = text.replace("%s", " ")
+        except:
+            logger.exception("Latex parsing error")
+
+    if convert_to_ascii:
+        text = unicodedata.normalize("NFKD", text)
+
+        try:
+            text.encode("ASCII", errors="strict")
+        except UnicodeEncodeError:
+            text = "".join([c for c in text if not unicodedata.combining(c)])
+            text = unidecode.unidecode(text)
+
+    if remove_brackets:
+        text = re.sub(r"\[[^\]]*\]", " ", text)
+
+    # fix hypens: break- word => break-word
+    text = re.sub(r"(\S)-\s+", r"\1-", text)
+    text = re.sub(r"\s+-(\S)", r"-\1", text)
+
+    # collapse whitespace
+    text = re.sub(r"\s+", " ", text)
+
+    # fix punctuation
+    text = re.sub(rf" ([{joined_left_punctuations}])", r"\1", text)
+    text = re.sub(rf"([{joined_right_punctuations}]) ", r"\1", text)
+
+    text = text.strip()
+
+    return text

good_ai/src/sus/data/__init__.py

@@ -0,0 +1,6 @@
+from .american_spellings import american_spellings
+from .punctuations import (
+    left_regular_punctuations,
+    right_regular_punctuations,
+    sentence_ending_punctuations,
+)

good_ai/src/sus/data/punctuations.py

@@ -0,0 +1,22 @@
+sentence_ending_punctuations = [".", "?", "!", ":", ";"]
+
+# punctuations that usually have no space between them and the character to their left
+left_regular_punctuations = [
+    *sentence_ending_punctuations,
+    ",",
+    "'",
+    "%",
+    ")",
+    "}",
+    "]",
+]
+
+# punctuations that usually have no space between them and the character to their right
+right_regular_punctuations = [
+    "(",
+    "{",
+    "[",
+    "$",
+    "€",
+    "#",
+]

good_ai/src/sus/evaluate_ranking/__init__.py

@@ -0,0 +1,2 @@
+from .draw_f1_iso_lines import draw_f1_iso_lines
+from .evaluate_ranking import evaluate_ranking

good_ai/src/sus/evaluate_ranking/draw_f1_iso_lines.py

@@ -0,0 +1,28 @@
+from typing import Optional
+
+import matplotlib.pyplot as plt
+import numpy as np
+from matplotlib.axes import Axes
+
+
+def draw_f1_iso_lines(
+    resolution: int = 1000,
+    min: float = 0.2,
+    max: float = 0.8,
+    steps: int = 4,
+    axes: Optional[Axes] = None,
+) -> None:
+    if axes is None:
+        axes = plt.axes()
+
+    for f_score in np.linspace(min, max, num=steps):
+        x = np.linspace(f_score / (2 - f_score), 1, num=resolution)
+        y = f_score * x / (2 * x - f_score)
+
+        axes.plot(x[y >= 0], y[y >= 0], color="gray", alpha=0.2)
+
+        axes.annotate(
+            f"f1={f_score:0.1f}",
+            backgroundcolor="w",
+            xy=(0.9, y[int(resolution * 0.9)] + 0.02),
+        )

good_ai/src/sus/evaluate_ranking/evaluate_ranking.py

@@ -0,0 +1,90 @@
+from pathlib import Path
+from typing import Dict, List, Optional, Union
+
+import matplotlib
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.metrics import average_precision_score, precision_recall_curve
+
+from ..unique import unique
+from .draw_f1_iso_lines import draw_f1_iso_lines
+
+
+def evaluate_ranking(
+    expected: List[Union[str, float]],
+    actual_scores: List[float],
+    target_recall: float,
+    title: Optional[str] = "",
+    disable_interpolation: bool = False,
+    axes: Optional[plt.Axes] = None,
+    output_svg: Optional[Path] = None,
+    reverse_order: bool = False,
+    plot: bool = True,
+) -> Dict[Union[str, float], float]:
+    assert 0 <= target_recall <= 1
+
+    if plot and axes is None:
+        fig = plt.figure(figsize=(20, 10))
+        fig.patch.set_facecolor("white")
+        ax = plt.axes()
+    else:
+        ax = axes
+
+    classes = sorted(unique(expected), reverse=reverse_order)
+    str_classes = [str(c) for c in classes]
+
+    with matplotlib.rc_context({"font.size": 20}):
+        if plot:
+            ax.set_xmargin(0)
+
+            draw_f1_iso_lines(axes=ax)
+
+        results: Dict[Union[str, float], float] = {}
+        for i in range(len(classes) - 1):
+            binarized_expected = [
+                (v < classes[i]) if reverse_order else (v > classes[i])
+                for v in expected
+            ]
+            precision, recall, _ = precision_recall_curve(
+                binarized_expected, actual_scores
+            )
+
+            if not disable_interpolation:
+                for j in range(1, len(precision)):
+                    precision[j] = max(precision[j - 1], precision[j])
+
+            closest_recall_index = np.argmin(np.abs(recall - target_recall))
+            precision_at_closest_recall = precision[closest_recall_index]
+            average_precision = average_precision_score(
+                binarized_expected, actual_scores
+            )
+            results[classes[i]] = precision_at_closest_recall
+
+            if plot:
+                ax.plot(
+                    recall,
+                    precision,
+                    label=f"{'|'.join(str_classes[:i + 1])} ↔ {'|'.join(str_classes[i+1:])} (P@{target_recall:.2f}={precision_at_closest_recall:.2f}, AP={average_precision:.2f})",
+                )
+
+        if plot:
+            ax.legend(loc="upper right")
+            ax.axvline(x=target_recall, linestyle="--", color="#55c6bb", linewidth=2.0)
+
+            if title is None:
+                title = "Ranking evaluation"
+
+            ax.set_title(f'{title} ({" < ".join(str_classes)})', pad=20)
+
+            ax.set_xlabel("Recall")
+            ax.set_ylabel("Precision")
+
+            ax.set_xticks([target_recall] + sorted(ax.get_xticks()))
+
+        if plot and output_svg is None:
+            if axes is None:
+                plt.show()
+        elif output_svg:
+            plt.savefig(output_svg, format="svg")
+
+    return results

good_ai/src/sus/external/negspacy/README.md

@@ -0,0 +1 @@
+https://github.com/jenojp/negspacy

good_ai/src/sus/external/negspacy/negation.py

@@ -0,0 +1,222 @@
+import logging
+
+from spacy.language import Language
+from spacy.matcher import PhraseMatcher
+from spacy.tokens import Token
+
+from .termsets import termset
+
+default_ts = termset("en").get_patterns()
+
+
+@Language.factory(
+    "negex",
+    default_config={
+        "neg_termset": default_ts,
+        "extension_name": "negex",
+        "chunk_prefix": list(),
+    },
+)
+class Negex:
+    """
+        A spaCy pipeline component which identifies negated tokens in text.
+
+        Based on: NegEx - A Simple Algorithm for Identifying Negated Findings and Diseasesin Discharge Summaries
+    Chapman, Bridewell, Hanbury, Cooper, Buchanan
+
+    Parameters
+    ----------
+    nlp: object
+        spaCy language object
+    termset_lang: str
+        language code, if using default termsets (e.g. "en" for english)
+    extension_name: str
+        defaults to "negex"; whether entity is negated is then available as ent._.negex
+    pseudo_negations: list
+        list of phrases that cancel out a negation, if empty, defaults are used
+    preceding_negations: list
+        negations that appear before an entity, if empty, defaults are used
+    following_negations: list
+        negations that appear after an entity, if empty, defaults are used
+    termination: list
+        phrases that "terminate" a sentence for processing purposes such as "but". If empty, defaults are used
+
+    """
+
+    def __init__(
+        self,
+        nlp: Language,
+        name: str,
+        neg_termset: dict,
+        extension_name: str,
+        chunk_prefix: list,
+    ):
+        if not Token.has_extension(extension_name):
+            Token.set_extension(extension_name, default=False, force=True)
+
+        ts = neg_termset
+        expected_keys = [
+            "pseudo_negations",
+            "preceding_negations",
+            "following_negations",
+            "termination",
+        ]
+        if not set(ts.keys()) == set(expected_keys):
+            raise KeyError(
+                f"Unexpected or missing keys in 'neg_termset', expected: {expected_keys}, instead got: {list(ts.keys())}"
+            )
+
+        self.pseudo_negations = ts["pseudo_negations"]
+        self.preceding_negations = ts["preceding_negations"]
+        self.following_negations = ts["following_negations"]
+        self.termination = ts["termination"]
+
+        self.nlp = nlp
+        self.extension_name = extension_name
+        self.build_patterns()
+        self.chunk_prefix = list(nlp.tokenizer.pipe(chunk_prefix))
+
+    def build_patterns(self):
+        # efficiently build spaCy matcher patterns
+        self.matcher = PhraseMatcher(self.nlp.vocab, attr="LOWER")
+
+        self.pseudo_patterns = list(self.nlp.tokenizer.pipe(self.pseudo_negations))
+        self.matcher.add("pseudo", None, *self.pseudo_patterns)
+
+        self.preceding_patterns = list(
+            self.nlp.tokenizer.pipe(self.preceding_negations)
+        )
+        self.matcher.add("Preceding", None, *self.preceding_patterns)
+
+        self.following_patterns = list(
+            self.nlp.tokenizer.pipe(self.following_negations)
+        )
+        self.matcher.add("Following", None, *self.following_patterns)
+
+        self.termination_patterns = list(self.nlp.tokenizer.pipe(self.termination))
+        self.matcher.add("Termination", None, *self.termination_patterns)
+
+    def process_negations(self, doc):
+        """
+        Find negations in doc and clean candidate negations to remove pseudo negations
+
+        Parameters
+        ----------
+        doc: object
+            spaCy Doc object
+
+        Returns
+        -------
+        preceding: list
+            list of tuples for preceding negations
+        following: list
+            list of tuples for following negations
+        terminating: list
+            list of tuples of terminating phrases
+
+        """
+        ###
+        # does not work properly in spacy 2.1.8. Will incorporate after 2.2.
+        # Relying on user to use NER in meantime
+        # see https://github.com/jenojp/negspacy/issues/7
+        ###
+        # if not doc.is_nered:
+        #     raise ValueError(
+        #         "Negations are evaluated for Named Entities found in text. "
+        #         "Your SpaCy pipeline does not included Named Entity resolution. "
+        #         "Please ensure it is enabled or choose a different language model that includes it."
+        #     )
+        preceding = list()
+        following = list()
+        terminating = list()
+
+        matches = self.matcher(doc)
+        pseudo = [
+            (match_id, start, end)
+            for match_id, start, end in matches
+            if self.nlp.vocab.strings[match_id] == "pseudo"
+        ]
+
+        for match_id, start, end in matches:
+            if self.nlp.vocab.strings[match_id] == "pseudo":
+                continue
+            pseudo_flag = False
+            for p in pseudo:
+                if start >= p[1] and start <= p[2]:
+                    pseudo_flag = True
+                    continue
+            if not pseudo_flag:
+                if self.nlp.vocab.strings[match_id] == "Preceding":
+                    preceding.append((match_id, start, end))
+                elif self.nlp.vocab.strings[match_id] == "Following":
+                    following.append((match_id, start, end))
+                elif self.nlp.vocab.strings[match_id] == "Termination":
+                    terminating.append((match_id, start, end))
+                else:
+                    logging.warnings(
+                        f"phrase {doc[start:end].text} not in one of the expected matcher types."
+                    )
+        return preceding, following, terminating
+
+    def termination_boundaries(self, doc, terminating):
+        """
+        Create sub sentences based on terminations found in text.
+
+        Parameters
+        ----------
+        doc: object
+            spaCy Doc object
+        terminating: list
+            list of tuples with (match_id, start, end)
+
+        returns
+        -------
+        boundaries: list
+            list of tuples with (start, end) of spans
+
+        """
+        sent_starts = [sent.start for sent in doc.sents]
+        terminating_starts = [t[1] for t in terminating]
+        starts = sent_starts + terminating_starts + [len(doc)]
+        starts.sort()
+        boundaries = list()
+        index = 0
+        for i, start in enumerate(starts):
+            if not i == 0:
+                boundaries.append((index, start))
+            index = start
+        return boundaries
+
+    def negex(self, doc):
+        """
+        Negates entities of interest
+
+        Parameters
+        ----------
+        doc: object
+            spaCy Doc object
+
+        """
+        preceding, following, terminating = self.process_negations(doc)
+        boundaries = self.termination_boundaries(doc, terminating)
+        for b in boundaries:
+            sub_preceding = [i for i in preceding if b[0] <= i[1] < b[1]]
+            sub_following = [i for i in following if b[0] <= i[1] < b[1]]
+
+            for e in doc[b[0] : b[1]]:
+                if any(pre < e.i for pre in [i[1] for i in sub_preceding]):
+                    e._.set(self.extension_name, True)
+                    continue
+                if any(fol > e.i for fol in [i[2] for i in sub_following]):
+                    e._.set(self.extension_name, True)
+                    continue
+                if self.chunk_prefix:
+                    if any(
+                        e.text.lower().startswith(c.text.lower())
+                        for c in self.chunk_prefix
+                    ):
+                        e._.set(self.extension_name, True)
+        return doc
+
+    def __call__(self, doc):
+        return self.negex(doc)

good_ai/src/sus/external/negspacy/termsets.py

@@ -0,0 +1,229 @@
+"""
+Default termsets for various languages
+"""
+
+LANGUAGES = dict()
+
+# english termset dictionary
+en = dict()
+pseudo = [
+    "no further",
+    "not able to be",
+    "not certain if",
+    "not certain whether",
+    "not necessarily",
+    "without any further",
+    "without difficulty",
+    "without further",
+    "might not",
+    "not only",
+    "no increase",
+    "no significant change",
+    "no change",
+    "no definite change",
+    "not extend",
+    "not cause",
+]
+en["pseudo_negations"] = pseudo
+
+preceding = [
+    "absence of",
+    "declined",
+    "denied",
+    "denies",
+    "denying",
+    "no sign of",
+    "no signs of",
+    "not",
+    "not demonstrate",
+    "symptoms atypical",
+    "doubt",
+    "negative for",
+    "no",
+    "versus",
+    "without",
+    "doesn't",
+    "doesnt",
+    "don't",
+    "dont",
+    "didn't",
+    "didnt",
+    "wasn't",
+    "wasnt",
+    "weren't",
+    "werent",
+    "isn't",
+    "isnt",
+    "aren't",
+    "arent",
+    "cannot",
+    "can't",
+    "cant",
+    "couldn't",
+    "couldnt",
+    "never",
+]
+en["preceding_negations"] = preceding
+
+following = [
+    "declined",
+    "unlikely",
+    "was not",
+    "were not",
+    "wasn't",
+    "wasnt",
+    "weren't",
+    "werent",
+]
+en["following_negations"] = following
+
+termination = [
+    "although",
+    "apart from",
+    "as there are",
+    "aside from",
+    "but",
+    "except",
+    "however",
+    "involving",
+    "nevertheless",
+    "still",
+    "though",
+    "which",
+    "yet",
+]
+en["termination"] = termination
+
+LANGUAGES["en"] = en
+
+# en_clinical builds upon en
+en_clinical = dict()
+pseudo_clinical = pseudo + [
+    "gram negative",
+    "not rule out",
+    "not ruled out",
+    "not been ruled out",
+    "not drain",
+    "no suspicious change",
+    "no interval change",
+    "no significant interval change",
+]
+en_clinical["pseudo_negations"] = pseudo_clinical
+
+preceding_clinical = preceding + [
+    "patient was not",
+    "without indication of",
+    "without sign of",
+    "without signs of",
+    "without any reactions or signs of",
+    "no complaints of",
+    "no evidence of",
+    "no cause of",
+    "evaluate for",
+    "fails to reveal",
+    "free of",
+    "never developed",
+    "never had",
+    "did not exhibit",
+    "rules out",
+    "rule out",
+    "rule him out",
+    "rule her out",
+    "rule patient out",
+    "rule the patient out",
+    "ruled out",
+    "ruled him out",
+    "ruled her out",
+    "ruled patient out",
+    "ruled the patient out",
+    "r/o",
+    "ro",
+]
+en_clinical["preceding_negations"] = preceding_clinical
+
+following_clinical = following + ["was ruled out", "were ruled out", "free"]
+en_clinical["following_negations"] = following_clinical
+
+termination_clinical = termination + [
+    "cause for",
+    "cause of",
+    "causes for",
+    "causes of",
+    "etiology for",
+    "etiology of",
+    "origin for",
+    "origin of",
+    "origins for",
+    "origins of",
+    "other possibilities of",
+    "reason for",
+    "reason of",
+    "reasons for",
+    "reasons of",
+    "secondary to",
+    "source for",
+    "source of",
+    "sources for",
+    "sources of",
+    "trigger event for",
+]
+en_clinical["termination"] = termination_clinical
+LANGUAGES["en_clinical"] = en_clinical
+
+en_clinical_sensitive = dict()
+
+preceding_clinical_sensitive = preceding_clinical + [
+    "concern for",
+    "supposed",
+    "which causes",
+    "leads to",
+    "h/o",
+    "history of",
+    "instead of",
+    "if you experience",
+    "if you get",
+    "teaching the patient",
+    "taught the patient",
+    "teach the patient",
+    "educated the patient",
+    "educate the patient",
+    "educating the patient",
+    "monitored for",
+    "monitor for",
+    "test for",
+    "tested for",
+]
+en_clinical_sensitive["pseudo_negations"] = pseudo_clinical
+en_clinical_sensitive["preceding_negations"] = preceding_clinical_sensitive
+en_clinical_sensitive["following_negations"] = following_clinical
+en_clinical_sensitive["termination"] = termination_clinical
+
+LANGUAGES["en_clinical_sensitive"] = en_clinical_sensitive
+
+
+class termset:
+    def __init__(self, termset_lang):
+        self.pattern_types = [
+            "pseudo_negations",
+            "preceding_negations",
+            "following_negations",
+            "termination",
+        ]
+        self.terms = LANGUAGES[termset_lang]
+
+    def get_patterns(self):
+        return self.terms
+
+    def remove_patterns(self, pattern_dict):
+        for key, value in pattern_dict.items():
+            if key in self.pattern_types:
+                self.terms[key] = [i for i in self.terms[key] if i not in value]
+            else:
+                raise ValueError(f"Unexpected key: {key} not in {self.pattern_types}")
+
+    def add_patterns(self, pattern_dict):
+        for key, value in pattern_dict.items():
+            if key in self.pattern_types:
+                self.terms[key] = list(set(self.terms[key] + value))
+            else:
+                raise ValueError(f"Unexpected key: {key} not in {self.pattern_types}")

good_ai/src/sus/external/pylatexenc/README.md

@@ -0,0 +1 @@
+https://github.com/phfaist/pylatexenc

good_ai/src/sus/external/pylatexenc/__init__.py

@@ -0,0 +1,37 @@
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2015 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+
+"""
+Utilities for LaTeX to/from Unicode Text conversion.
+
+Main Site:
+
+    https://github.com/phfaist/pylatexenc/
+
+"""
+
+from .version import version_str as _version_str
+
+__version__ = _version_str

good_ai/src/sus/external/pylatexenc/_util.py

@@ -0,0 +1,161 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2019 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+
+# Internal module. Internal API may move, disappear or otherwise change at any
+# time and without notice.
+
+
+try:
+    # Python >= 3.3
+    from collections.abc import MutableMapping
+except ImportError:
+    from collections import MutableMapping
+
+import bisect
+import warnings
+
+# ------------------------------------------------------------------------------
+
+
+def pylatexenc_deprecated_ver(ver, msg, stacklevel=2):
+    warnings.warn(
+        "Deprecated (pylatexenc {}): {} ".format(ver, msg.strip()),
+        DeprecationWarning,
+        stacklevel=stacklevel + 1,
+    )
+
+
+def pylatexenc_deprecated_2(msg, stacklevel=2):
+    warnings.warn(
+        (
+            "Deprecated (pylatexenc 2.0): {} "
+            "[see https://pylatexenc.readthedocs.io/en/latest/new-in-pylatexenc-2/]"
+        ).format(msg.strip()),
+        DeprecationWarning,
+        stacklevel=stacklevel + 1,
+    )
+
+
+# ------------------------------------------------------------------------------
+
+
+class LazyDict(MutableMapping):
+    r"""
+    A lazy dictionary that loads its data when it is first queried.
+
+    This is used to store the legacy
+    :py:data:`pylatexenc.latexwalker.default_macro_dict` as well as
+    :py:data:`pylatexenc.latex2text.default_macro_dict` etc.  Such that these
+    "dictionaries" are still exposed at the module-level, but the data is loaded
+    only if they are actually queried.
+    """
+
+    def __init__(self, generate_dict_fn):
+        self._full_dict = None
+        self._generate_dict_fn = generate_dict_fn
+
+    def _ensure_instance(self):
+        if self._full_dict is not None:
+            return
+        self._full_dict = self._generate_dict_fn()
+
+    def __getitem__(self, key):
+        self._ensure_instance()
+        return self._full_dict.__getitem__(key)
+
+    def __setitem__(self, key, val):
+        self._ensure_instance()
+        return self._full_dict.__setitem__(key, val)
+
+    def __delitem__(self, key):
+        self._ensure_instance()
+        return self._full_dict.__delitem__(key)
+
+    def __iter__(self):
+        self._ensure_instance()
+        return iter(self._full_dict)
+
+    def __len__(self):
+        self._ensure_instance()
+        return len(self._full_dict)
+
+    def copy(self):
+        self._ensure_instance()
+        return self._full_dict.copy()
+
+    def clear(self):
+        self._ensure_instance()
+        return self._full_dict.clear()
+
+
+# ------------------------------------------------------------------------------
+
+
+class LineNumbersCalculator(object):
+    r"""
+    Utility to calculate line numbers.
+    """
+
+    def __init__(self, s):
+        super(LineNumbersCalculator, self).__init__()
+
+        def find_all_new_lines(x):
+            # first line starts at the beginning of the string
+            yield 0
+            k = 0
+            while k < len(x):
+                k = x.find("\n", k)
+                if k == -1:
+                    return
+                k += 1
+                # s[k] is the character after the newline, i.e., the 0-th column
+                # of the new line
+                yield k
+
+        self._pos_new_lines = list(find_all_new_lines(s))
+
+    def pos_to_lineno_colno(self, pos, as_dict=False):
+        r"""
+        Return the line and column number corresponding to the given `pos`.
+
+        Return a tuple `(lineno, colno)` giving line number and column number.
+        Line numbers start at 1 and column number start at zero, i.e., the
+        beginning of the document (`pos=0`) has line and column number `(1,0)`.
+        If `as_dict=True`, then a dictionary with keys 'lineno', 'colno' is
+        returned instead of a tuple.
+        """
+
+        # find line number in list
+
+        # line_no is the index of the last item in self._pos_new_lines that is <= pos.
+        line_no = bisect.bisect_right(self._pos_new_lines, pos) - 1
+        assert line_no >= 0 and line_no < len(self._pos_new_lines)
+
+        col_no = pos - self._pos_new_lines[line_no]
+        # 1+... so that line and column numbers start at 1
+        if as_dict:
+            return {"lineno": 1 + line_no, "colno": col_no}
+        return (1 + line_no, col_no)

good_ai/src/sus/external/pylatexenc/latex2text/__main__.py

@@ -0,0 +1,325 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2018 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+import argparse
+import fileinput
+import logging
+import sys
+
+from .. import latexwalker
+from ..latex2text import LatexNodes2Text, _strict_latex_spaces_predef
+from ..version import version_str
+
+
+def main(argv=None):
+
+    if argv is None:
+        argv = sys.argv[1:]
+
+    parser = argparse.ArgumentParser(prog="latex2text", add_help=False)
+
+    codegroup = parser.add_argument_group("Input options")
+
+    codegroup.add_argument(
+        "--code",
+        "-c",
+        action="store",
+        default=None,
+        metavar="LATEX_CODE",
+        help="Convert the given LATEX_CODE to unicode text instead of reading "
+        "from FILE or standard input.  You cannot specify FILEs if you use this "
+        "option, and any standard input is ignored.",
+    )
+
+    codegroup.add_argument(
+        "files",
+        metavar="FILE",
+        nargs="*",
+        help="Input files to read LaTeX code from. If no FILE(s) is/are specified, "
+        "LaTeX code is read from standard input unless --code is specified",
+    )
+
+    group = parser.add_argument_group("LatexWalker options")
+
+    group.add_argument(
+        "--parser-keep-inline-math",
+        action="store_const",
+        const=True,
+        dest="parser_keep_inline_math",
+        default=None,
+        help=argparse.SUPPRESS,
+    )
+    group.add_argument(
+        "--no-parser-keep-inline-math",
+        action="store_const",
+        const=False,
+        dest="parser_keep_inline_math",
+        help=argparse.SUPPRESS,
+    )
+
+    group.add_argument(
+        "--tolerant-parsing",
+        action="store_const",
+        const=True,
+        dest="tolerant_parsing",
+        default=True,
+    )
+    group.add_argument(
+        "--no-tolerant-parsing",
+        action="store_const",
+        const=False,
+        dest="tolerant_parsing",
+        help="Tolerate syntax errors when parsing, and attempt to continue (default yes)",
+    )
+
+    # I'm not sure this flag is useful and if it should be exposed at all.
+    # Accept it, but make it hidden.
+    parser.add_argument(
+        "--strict-braces",
+        action="store_const",
+        const=True,
+        dest="strict_braces",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
+    parser.add_argument(
+        "--no-strict-braces",
+        action="store_const",
+        const=False,
+        dest="strict_braces",
+        # help="Report errors for mismatching LaTeX braces (default no)"
+        help=argparse.SUPPRESS,
+    )
+
+    group = parser.add_argument_group("LatexNodes2Text options")
+
+    group.add_argument(
+        "--text-keep-inline-math",
+        action="store_const",
+        const=True,
+        dest="text_keep_inline_math",
+        default=None,
+        help=argparse.SUPPRESS,
+    )
+    group.add_argument(
+        "--no-text-keep-inline-math",
+        action="store_const",
+        const=False,
+        dest="text_keep_inline_math",
+        help=argparse.SUPPRESS,
+    )
+
+    group.add_argument(
+        "--math-mode",
+        action="store",
+        dest="math_mode",
+        choices=["text", "with-delimiters", "verbatim", "remove"],
+        default="text",
+        help="How to handle chunks of math mode LaTeX code. 'text' = convert "
+        "to text like the rest; 'with-delimiters' = same as 'text' but retain "
+        "the original math mode delimiters; 'verbatim' = keep verbatim LaTeX code; "
+        "'remove' = remove from input entirely",
+    )
+
+    group.add_argument(
+        "--fill-text",
+        dest="fill_text",
+        action="store",
+        nargs="?",
+        default=-1,
+        help="Attempt to wrap text to the given width, or 80 columns if option is "
+        "specified with no argument",
+    )
+
+    group.add_argument(
+        "--keep-comments",
+        action="store_const",
+        const=True,
+        dest="keep_comments",
+        default=False,
+    )
+    group.add_argument(
+        "--no-keep-comments",
+        action="store_const",
+        const=False,
+        dest="keep_comments",
+        help="Keep LaTeX comments in text output (default no)",
+    )
+
+    class ListWithHiddenItems(list):
+        def __init__(self, thelist, hiddenitems):
+            super(ListWithHiddenItems, self).__init__(thelist)
+            self.hiddenitems = hiddenitems
+
+        def __contains__(self, value):
+            return (
+                super(ListWithHiddenItems, self).__contains__(value)
+                or value in self.hiddenitems
+            )
+
+    strict_latex_spaces_choices = ListWithHiddenItems(
+        # the list
+        ["off", "on"]
+        + list(k for k in _strict_latex_spaces_predef.keys() if k != "default"),
+        # hidden items: Value is accepted, but not shown in list of choices
+        ["default"],
+    )
+    group.add_argument(
+        "--strict-latex-spaces",
+        choices=strict_latex_spaces_choices,
+        dest="strict_latex_spaces",
+        default="macros",
+        help="How to handle whitespace. See documentation for the class "
+        "LatexNodes2Text().",
+    )
+
+    group.add_argument(
+        "--keep-braced-groups",
+        action="store_const",
+        const=True,
+        dest="keep_braced_groups",
+        default=False,
+    )
+    group.add_argument(
+        "--no-keep-braced-groups",
+        action="store_const",
+        const=False,
+        dest="keep_braced_groups",
+        help="Keep LaTeX {braced groups} in text output (default no)",
+    )
+
+    group.add_argument(
+        "--keep-braced-groups-minlen",
+        type=int,
+        default=2,
+        dest="keep_braced_groups_minlen",
+        help="Only apply --keep-braced-groups to groups that contain at least "
+        "this many characters",
+    )
+
+    group = parser.add_argument_group("General options")
+
+    group.add_argument(
+        "-q",
+        "--quiet",
+        dest="logging_level",
+        action="store_const",
+        const=logging.ERROR,
+        default=logging.INFO,
+        help="Suppress warning messages",
+    )
+    group.add_argument(
+        "-v",
+        "--verbose",
+        dest="logging_level",
+        action="store_const",
+        const=logging.DEBUG,
+        help="Verbose output",
+    )
+    group.add_argument(
+        "--version",
+        action="version",
+        version="pylatexenc {}".format(version_str),
+        help="Show version information and exit",
+    )
+    group.add_argument(
+        "--help", action="help", help="Show this help information and exit"
+    )
+
+    args = parser.parse_args(argv)
+
+    logging.basicConfig()
+    logging.getLogger().setLevel(args.logging_level)
+    logger = logging.getLogger(__name__)
+
+    if (
+        args.parser_keep_inline_math is not None
+        or args.text_keep_inline_math is not None
+    ):
+        logger.warning(
+            "Options --parser-keep-inline-math and --text-keep-inline-math are "
+            "deprecated and no longer have any effect.  Please use "
+            "--math-mode=... instead."
+        )
+
+    latex = ""
+    if args.code:
+        if args.files:
+            logger.error(
+                "Cannot specify both FILEs and --code option. "
+                "Use --help option for more information."
+            )
+            sys.exit(1)
+        latex = args.code
+    else:
+        for line in fileinput.input(files=args.files):
+            latex += line
+
+    if args.fill_text != -1:
+        if args.fill_text is not None and len(args.fill_text):
+            fill_text = int(args.fill_text)
+        else:
+            fill_text = True
+    else:
+        fill_text = None
+
+    lw = latexwalker.LatexWalker(
+        latex, tolerant_parsing=args.tolerant_parsing, strict_braces=args.strict_braces
+    )
+
+    (nodelist, pos, len_) = lw.get_latex_nodes()
+
+    ln2t = LatexNodes2Text(
+        math_mode=args.math_mode,
+        keep_comments=args.keep_comments,
+        strict_latex_spaces=args.strict_latex_spaces,
+        keep_braced_groups=args.keep_braced_groups,
+        keep_braced_groups_minlen=args.keep_braced_groups_minlen,
+        fill_text=fill_text,
+    )
+
+    print(ln2t.nodelist_to_text(nodelist))
+
+
+def run_main():
+
+    try:
+
+        main()
+
+    except SystemExit:
+        raise
+    except:  # lgtm [py/catch-base-exception]
+        import pdb
+        import traceback
+
+        traceback.print_exc()
+        pdb.post_mortem()
+
+
+if __name__ == "__main__":
+
+    main()
+    # run_main() # debug

good_ai/src/sus/external/pylatexenc/latexencode/__init__.py

@@ -0,0 +1,331 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2018 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+r"""
+The `latexencode` module provides a set of routines that allows you to
+convert a unicode string to LaTeX escape sequences.
+
+For basic usage you can use the :py:func:`unicode_to_latex()` function
+directly::
+
+  >>> from pylatexenc.latexencode import unicode_to_latex
+  >>> print(unicode_to_latex('À votre santé'))
+  \`A votre sant\'e
+  >>> print(unicode_to_latex('The length of samples #3 & #4 is 3μm'))
+  The length of samples \#3 \& \#4 is 3\ensuremath{\mu}m
+
+The conversion is handled by the class :py:class:`UnicodeToLatexEncoder`.  If
+you are converting multiple strings, you may create an instance with the flags
+you like and invoke its method
+:py:meth:`~UnicodeToLatexEncoder.unicode_to_latex()` as many times as necessary::
+
+  >>> from pylatexenc.latexencode import UnicodeToLatexEncoder
+  >>> u = UnicodeToLatexEncoder(unknown_char_policy='replace')
+  >>> print(u.unicode_to_latex('À votre santé'))
+  \`A votre sant\'e
+  >>> print(u.unicode_to_latex('The length of samples #3 & #4 is 3μm'))
+  The length of samples \#3 \& \#4 is 3\ensuremath{\mu}m
+  >>> print(u.unicode_to_latex('À votre santé: 乾杯'))
+  No known latex representation for character: U+4E7E - ‘乾’
+  No known latex representation for character: U+676F - ‘杯’
+  \`A votre sant\'e: {\bfseries ?}{\bfseries ?}
+
+Example using custom conversion rules::
+
+  >>> from pylatexenc.latexencode import UnicodeToLatexEncoder, \
+  ...     UnicodeToLatexConversionRule, RULE_REGEX
+  >>> u = UnicodeToLatexEncoder(
+  ...     conversion_rules=[
+  ...         UnicodeToLatexConversionRule(rule_type=RULE_REGEX, rule=[
+  ...             (re.compile(r'-->'), r'\\textrightarrow'),
+  ...             (re.compile(r'<--'), r'\\textleftarrow'),
+  ...         ]),
+  ...         'defaults'
+  ...     ]
+  ... )
+  >>> print(u.unicode_to_latex("Cheers --> À votre santé"))
+  Cheers {\textrightarrow} \`A votre sant\'e
+
+See :py:class:`UnicodeToLatexEncoder` and
+:py:class:`UnicodeToLatexConversionRule`.  Note for regex rules, the replacement
+text is expanded like the second argument of `re.sub()` and backslashes need to
+be escaped even inside raw strings.
+
+.. versionadded:: 2.0
+
+   The class :py:class:`UnicodeToLatexEncoder` along with its helper functions
+   and classes were introduced in `pylatexenc 2.0`.
+
+   The earlier function :py:func:`utf8tolatex()` that was available in
+   `pylatexenc 1.x` is still provided unchanged, so code written for `pylatexenc
+   1.x` should work without changes.  New code is however strongly encouraged to
+   employ the new API.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import functools
+import itertools
+import logging
+import sys
+import unicodedata
+
+if sys.version_info.major > 2:
+    unicode = str  # need to support unicode() w/ no arguments
+    basestring = str
+    # use MappingProxyType for keeping
+    # inspect function argument names
+    from inspect import getfullargspec
+    from types import MappingProxyType as _MappingProxyType
+else:
+    _MappingProxyType = dict
+    # inspect function argument names -- simulate getfullargspec with getargspec (argh...)
+    from inspect import getargspec as getfullargspec
+
+logger = logging.getLogger(__name__)
+
+
+from .. import _util
+from ._partial_latex_encoder import PartialLatexToLatexEncoder
+from ._unicode_to_latex_encoder import (
+    RULE_CALLABLE,
+    RULE_DICT,
+    RULE_REGEX,
+    UnicodeToLatexConversionRule,
+    UnicodeToLatexEncoder,
+    get_builtin_conversion_rules,
+    get_builtin_uni2latex_dict,
+)
+
+# ------------------------------------------------
+
+
+# ------------------------------------------------
+
+
+# ------------------------------------------------
+
+
+_u2l_obj_cache = {}
+
+
+def unicode_to_latex(
+    s,
+    non_ascii_only=False,
+    replacement_latex_protection="braces",
+    unknown_char_policy="keep",
+    unknown_char_warning=True,
+):
+    r"""
+    Shorthand for constructing a :py:class:`UnicodeToLatexEncoder` instance and
+    calling its :py:meth:`~UnicodeToLatexEncoder.unicode_to_latex()` method.
+
+    The :py:class:`UnicodeToLatexEncoder` instances for given option settings
+    are cached, making repeated calls to :py:func:`unicode_to_latex()` possible
+    without creating a new instance upon each call.
+
+    The parameters `non_ascii_only`, `replacement_latex_protection`,
+    `unknown_char_policy`, and `unknown_char_warning` are directly passed on to
+    the :py:class:`UnicodeToLatexEncoder` constructor.  See the class doc for
+    :py:class:`UnicodeToLatexEncoder` for more information about what they do.
+
+    You may only use arguments to this function that are python hashable (like
+    `True`, `False`, or simple strings) to help us keep a cache of previously
+    constructed :py:class:`UnicodeToLatexEncoder` instances.  For instance, it
+    is not possible to provide a callable to `unknown_char_policy`.  It is also
+    not possible to specify custom conversion rules with this helper function.
+    If you need any of these features, simply create a
+    :py:class:`UnicodeToLatexEncoder` instance directly.
+    """
+
+    key = (
+        non_ascii_only,
+        replacement_latex_protection,
+        unknown_char_policy,
+        unknown_char_warning,
+    )
+
+    if key in _u2l_obj_cache:
+        u = _u2l_obj_cache[key]
+    else:
+        u = UnicodeToLatexEncoder(
+            non_ascii_only=non_ascii_only,
+            replacement_latex_protection=replacement_latex_protection,
+            unknown_char_policy=unknown_char_policy,
+            unknown_char_warning=unknown_char_warning,
+        )
+        _u2l_obj_cache[key] = u
+
+    return u.unicode_to_latex(s)
+
+
+# ------------------------------------------------------------------------------
+
+# Don't change pylatexenc 1.x function:
+
+
+def _get_deprecated_utf82latex():
+    #
+    # Don't issue a deprecation warning, because utf8tolatex() uses the
+    # `utf82latex` dict even if it isn't modified by the user.
+    #
+    #     _util.pylatexenc_deprecated_2(
+    #         "The module-level dictionary `pylatexenc.latexencode.utf82latex` is deprecated "
+    #         "and might be removed in a future version of `pylatexenc`.",
+    #     )
+
+    # return a copy of the dict so that the user can modify the module-level
+    # `utf82latex` dict without influencing the behavior of the new
+    # `unicode_to_latex()` routines. (E.g., if two python modules use
+    # pylatexenc.latexencode, we don't want one python module's use of
+    # `utf2tolatex()` to influence the behavior of another module's use of
+    # `unicode_to_latex()`.  If both modules use `utf8tolatex()`, we can't avoid
+    # this influence.)
+    from ._uni2latexmap import uni2latex as _uni2latex
+
+    return _uni2latex.copy()
+
+
+utf82latex = _util.LazyDict(generate_dict_fn=_get_deprecated_utf82latex)
+"""
+.. deprecated:: 2.0
+
+   Pylatexenc 1.x exposed the module-level dictionary `utf82latex` that could be
+   modified to alter the behavior of `utf8tolatex()`.
+
+   If you would like to obtain a copy of the built-in unicode to text
+   dictionary, see :py:func:`get_builtin_uni2latex_dict()`.  If you would like
+   to alter the behavior of :py:func:`utf8tolatex()`, you should use
+   :py:class:`UnicodeToLatexEncoder` which provides a rich interface for
+   specifying rules how to convert chars to LaTeX escapes.
+
+   For backwards compatibility, you can still modify the module-level dictionary
+   `utf82latex` (but you can't assign a new object to it) and this will directly
+   modify the global built-in dictionary of known latex escapes.  This is not
+   recommended however, and the `utf82latex` module-level dictionary might be
+   removed in the future.
+
+   .. warning::
+
+      Modifying the `utf82latex` module-level dictionary is not recommended.
+      Doing so will alter the behavior of the `utf8tolatex()` function also for
+      all other modules that also use `pylatexenc`!
+"""
+
+
+def utf8tolatex(
+    s,
+    non_ascii_only=False,
+    brackets=True,
+    substitute_bad_chars=False,
+    fail_bad_chars=False,
+):
+    """
+    .. note::
+
+       Since `pylatexenc 2.0`, it is recommended to use the the
+       :py:func:`unicode_to_latex()` function or the
+       :py:class:`UnicodeToLatexEncoder` class instead of the earlier function
+       `utf8tolatex()`.
+
+       The new routines provide much more flexibility and versatility.  For
+       instance, you can specify custom escape sequences for certain characters.
+       Some cheap benchmarks seem to indicate that the new routines are not
+       significantly slower than the `utf8tolatex()` function.  Also, the name
+       `utf8tolatex()` was poorly chosen, since the argument is in fact not
+       'utf-8'-encoded but rather a Python unicode string object.
+
+       The function `utf8tolatex()` is still provided unchanged from `pylatexenc
+       1.x`.  We do not plan to remove this function in the near future so it is
+       not (yet) considered as deprecated and we will continue to provide it in
+       near future versions of `pylatexenc`.  Bug reports, improvements, and new
+       features will however be directed to :py:func:`UnicodeToLatexEncoder()`.
+
+    Encode a UTF-8 string to a LaTeX snippet.
+
+    If `non_ascii_only` is set to `True`, then usual (ascii) characters such as ``#``,
+    ``{``, ``}`` etc. will not be escaped.  If set to `False` (the default), they are
+    escaped to their respective LaTeX escape sequences.
+
+    If `brackets` is set to `True` (the default), then LaTeX macros are enclosed in
+    brackets.  For example, ``sant\N{LATIN SMALL LETTER E WITH ACUTE}`` is replaced by
+    ``sant{\\'e}`` if `brackets=True` and by ``sant\\'e`` if `brackets=False`.
+
+    .. warning::
+        Using `brackets=False` might give you an invalid LaTeX string, so avoid
+        it! (for instance, ``ma\N{LATIN SMALL LETTER I WITH CIRCUMFLEX}tre`` will be
+        replaced incorrectly by ``ma\\^\\itre`` resulting in an unknown macro ``\\itre``).
+
+    If `substitute_bad_chars=True`, then any non-ascii character for which no LaTeX escape
+    sequence is known is replaced by a question mark in boldface. Otherwise (by default),
+    the character is left as it is.
+
+    If `fail_bad_chars=True`, then a `ValueError` is raised if we cannot find a
+    character substitution for any non-ascii character.
+
+    .. versionchanged:: 1.3
+
+        Added `fail_bad_chars` switch
+    """
+
+    s = unicode(s)  # make sure s is unicode
+    s = unicodedata.normalize("NFC", s)
+
+    if not s:
+        return ""
+
+    result = ""
+    for ch in s:
+        # logger.longdebug("Encoding char %r", ch)
+        if non_ascii_only and ord(ch) < 127:
+            result += ch
+        else:
+            # use the `utf82latex` dict -- not `_uni2latex` which should NOT be
+            # modified externally even for backwards-compatible code
+            lch = utf82latex.get(ord(ch), None)
+            if lch is not None:
+                # add brackets if needed, i.e. if we have a substituting macro.
+                # note: in condition, beware, that lch might be of zero length.
+                result += "{" + lch + "}" if brackets and lch[0:1] == "\\" else lch
+            elif (ord(ch) >= 32 and ord(ch) <= 127) or (ch in "\n\r\t"):
+                # ordinary printable ascii char, just add it
+                result += ch
+            else:
+                # non-ascii char
+                msg = "Character cannot be encoded into LaTeX: U+%04X - `%s'" % (
+                    ord(ch),
+                    ch,
+                )
+                if fail_bad_chars:
+                    raise ValueError(msg)
+
+                logger.warning(msg)
+                if substitute_bad_chars:
+                    result += r"{\bfseries ?}"
+                else:
+                    # keep unescaped char
+                    result += ch
+
+    return result

good_ai/src/sus/external/pylatexenc/latexencode/__main__.py

@@ -0,0 +1,148 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2019 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+
+import argparse
+import fileinput
+import logging
+import sys
+
+from ..latexencode import unicode_to_latex
+from ..version import version_str
+
+
+def main(argv=None):
+
+    if argv is None:
+        argv = sys.argv[1:]
+
+    parser = argparse.ArgumentParser(prog="latexencode", add_help=False)
+    parser.add_argument(
+        "files",
+        metavar="FILE",
+        nargs="*",
+        help="Input files (if none specified, read from stdandard input)",
+    )
+
+    parser.add_argument(
+        "--non-ascii-only",
+        action="store_const",
+        const=True,
+        dest="non_ascii_only",
+        default=False,
+    )
+    parser.add_argument(
+        "--no-non-ascii-only",
+        action="store_const",
+        const=False,
+        dest="non_ascii_only",
+        help="The option --non-ascii-only specifies that only non-ascii characters "
+        "are to be encoded into LaTeX sequences, and not characters like '$' "
+        "even though they might have a special LaTeX meaning.",
+    )
+
+    parser.add_argument(
+        "--replacement-latex-protection",
+        choices=(
+            "braces",
+            "braces-all",
+            "braces-almost-all",
+            "braces-after-macro",
+            "none",
+        ),
+        dest="replacement_latex_protection",
+        default="braces",
+        help=r"How to protect replacement latex code from producing invalid latex code "
+        r"when concatenated in a longer string.  One of 'braces', 'braces-all', "
+        r"'braces-almost-all', 'braces-after-macro', 'none'.  Example: using "
+        r"choice 'braces' we avoid the invalid replacement 'a→b' -> 'a\tob' "
+        r"with instead 'a{\to}b'.",
+    )
+
+    parser.add_argument(
+        "--unknown-char-policy",
+        choices=("keep", "replace", "ignore", "fail"),
+        dest="unknown_char_policy",
+        default="keep",
+        help="How to deal with nonascii characters with no known latex code equivalent.",
+    )
+
+    parser.add_argument(
+        "-q",
+        "--quiet",
+        dest="logging_level",
+        action="store_const",
+        const=logging.ERROR,
+        default=logging.INFO,
+        help="Suppress warning messages",
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version="pylatexenc {}".format(version_str),
+        help="Show version information and exit",
+    )
+    parser.add_argument(
+        "--help", action="help", help="Show this help information and exit"
+    )
+
+    args = parser.parse_args(argv)
+
+    logging.basicConfig()
+    logging.getLogger().setLevel(args.logging_level)
+
+    latex = ""
+    for line in fileinput.input(files=args.files):
+        latex += line
+
+    result = unicode_to_latex(
+        latex,
+        non_ascii_only=args.non_ascii_only,
+        replacement_latex_protection=args.replacement_latex_protection,
+        unknown_char_policy=args.unknown_char_policy,
+    )
+
+    sys.stdout.write(result)
+
+
+def run_main():
+    try:
+
+        main()
+
+    except SystemExit:
+        raise
+    except:  # lgtm [py/catch-base-exception]
+        import pdb
+        import traceback
+
+        traceback.print_exc()
+        pdb.post_mortem()
+
+
+if __name__ == "__main__":
+
+    # run_main()  ## DEBUG
+    main()

good_ai/src/sus/external/pylatexenc/latexencode/_partial_latex_encoder.py

@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2021 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+# import sys
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+from ._unicode_to_latex_encoder import (
+    RULE_CALLABLE,
+    UnicodeToLatexConversionRule,
+    UnicodeToLatexEncoder,
+)
+
+# if sys.version_info.major == 2:
+#     bytes = str
+#     str = unicode
+
+
+class PartialLatexToLatexEncoder(UnicodeToLatexEncoder):
+    r"""
+    Encode a string while preserving some (fuzzily detected) LaTeX constructs
+    that the input string already has (e.g. accent macros or inline math modes).
+
+    Sometimes you need to fully LaTeX-encode a string that already has some
+    LaTeX constructs.  For instance, titles of bibliographic entries might
+    include some inline math or accents, but they might also include unicode
+    characters that need to be encoded.  Using a
+    :py:class:`UnicodeToLatexEncoder` on such strings would result in ugly
+    doubly-escaped strings such as ``\textbackslash{}'\{e\}``.  Instead,
+    constructs such as ``\'{e}`` should be preserved while other characters
+    and/or constructs (say '&' or '%') as well as unicode characters should be
+    encoded.
+
+    This class offers a simple partial solution: Characters are encoded as per
+    the given `conversion_rules` (or the default conversion rules of
+    :py:class:`UnicodeToLatexEncoder` objects), except that the characters in
+    `keep_latex_chars` are to be interpreted as LaTeX and are not to be further
+    encoded.
+
+    .. versionadded: 2.10
+    """
+
+    def __init__(
+        self,
+        # keyword arguments:
+        keep_latex_chars=r"\${}^_",
+        conversion_rules=None,
+        **kwargs
+    ):
+
+        base_conversion_rules = conversion_rules
+        if base_conversion_rules is None:
+            base_conversion_rules = ["defaults"]
+
+        super(PartialLatexToLatexEncoder, self).__init__(
+            # only a single rule, our own special method that tries to parse
+            # partial latex.
+            conversion_rules=[
+                UnicodeToLatexConversionRule(
+                    rule_type=RULE_CALLABLE,
+                    rule=self._do_partial_latex_encode_step,
+                    replacement_latex_protection="none",
+                )
+            ]
+            + base_conversion_rules,
+            **kwargs
+        )
+
+        self.keep_latex_chars = keep_latex_chars
+
+    def _do_partial_latex_encode_step(self, s, pos):
+        r"""
+        This method is used as a "callable rule" for the
+        :py:class:`UnicodeToLatexEncoder` object.
+
+        The strategy is to see if we have something that looks like a LaTeX char
+        we want to keep.  If so, keep it as is; if not, return `None` so that
+        further rules can be considered by the base unicode encoder.
+        """
+
+        from ..latexwalker import LatexWalker
+
+        if s[pos] in self.keep_latex_chars:
+            # Read a token and if it is a macro, keep the full macro!
+            lw = LatexWalker(s, tolerant_parsing=False)
+
+            tok = lw.get_token(pos, environments=False)
+
+            tok_as_latex = tok.pre_space + s[tok.pos : tok.pos + tok.len]
+
+            # keep the LaTeX token as-is
+            return (tok.pos + tok.len - pos, tok_as_latex)
+
+        return None

good_ai/src/sus/external/pylatexenc/latexencode/_unicode_to_latex_encoder.py

@@ -0,0 +1,686 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2021 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import functools
+import itertools
+import logging
+import sys
+import unicodedata
+
+if sys.version_info.major > 2:
+    unicode = str  # need to support unicode() w/ no arguments
+    basestring = str
+    # use MappingProxyType for keeping
+    # inspect function argument names
+    from inspect import getfullargspec
+    from types import MappingProxyType as _MappingProxyType
+else:
+    _MappingProxyType = dict
+    # inspect function argument names -- simulate getfullargspec with getargspec (argh...)
+    from inspect import getargspec as getfullargspec
+
+logger = logging.getLogger(__name__)
+
+
+def get_builtin_uni2latex_dict():
+    r"""
+    Return a dictionary that contains the default collection of known LaTeX
+    escape sequences for unicode characters.
+
+    The keys of the dictionary are integers that correspond to unicode code
+    points (i.e., `ord(char)`).  The values are the corresponding LaTeX
+    replacement strings.
+
+    The returned dictionary may not be modified.  To alter the behavior of
+    :py:func:`unicode_to_latex()`, you should specify custom rules to a new
+    instance of :py:class:`UnicodeToLatexEncoder`.
+
+    .. versionadded:: 2.0
+
+       This function was introduced in `pylatexenc 2.0`.
+    """
+    from ._uni2latexmap import uni2latex as _uni2latex
+
+    return _MappingProxyType(_uni2latex)
+
+
+RULE_DICT = 0
+r"""
+Indicates a rule type that is a dictionary of unicode point values to
+replacement strings. See :py:class:`UnicodeToLatexConversionRule`.
+
+.. versionadded:: 2.0
+
+   This member was introduced in pylatexenc version 2.0.
+"""
+
+RULE_REGEX = 1
+r"""
+Indicates a rule type that is a list (or iterable) of pairs
+`(compiled_regular_expression, replacement_string)`.  See
+:py:class:`UnicodeToLatexConversionRule`.
+
+.. versionadded:: 2.0
+
+   This member was introduced in pylatexenc version 2.0.
+"""
+
+RULE_CALLABLE = 2
+r"""
+Indicates a rule type that is a custom callable.  See
+:py:class:`UnicodeToLatexConversionRule`.
+
+.. versionadded:: 2.0
+
+   This member was introduced in pylatexenc version 2.0.
+"""
+
+
+class UnicodeToLatexConversionRule:
+    r"""
+    Specify a rule how to convert unicode characters into LaTeX escapes.
+
+    .. py:attribute:: rule_type
+    
+       One of :py:data:`RULE_DICT`, :py:data:`RULE_REGEX`, or
+       :py:data:`RULE_CALLABLE`.
+
+    .. py:attribute:: rule
+
+       A specification of the rule itself.  The `rule` attribute is an object
+       that depends on what `rule_type` is set to.  See below.
+
+    .. py:attribute:: replacement_latex_protection
+
+       If non-`None`, then the setting here will override any
+       `replacement_latex_protection` set on
+       :py:class:`UnicodeToLatexConversionRule` objects.  By default the value
+       is `None`, and you can set a replacement_latex_protection globally for
+       all rules on the :py:class:`UnicodeToLatexEncoder` object.
+
+       The use of this attribute is mainly in case you have a fancy rule in
+       which you already guarantee that whatever you output is valid LaTeX even
+       if concatenated with the remainder of the string; in this case you can
+       set `replacement_latex_protection='none'` to avoid unnecessary or
+       unwanted braces around the generated code.
+
+       .. versionadded:: 2.10
+
+          The `replacement_latex_protection` attribute was introduced in
+          `pylatexenc 2.10`.
+
+
+    Constructor syntax::
+    
+        UnicodeToLatexConversionRule(RULE_XXX, <...>)
+        UnicodeToLatexConversionRule(rule_type=RULE_XXX, rule=<...>)
+
+        UnicodeToLatexConversionRule(..., replacement_latex_protection='none')
+
+    Note that you can get some built-in rules via the
+    :py:func:`get_builtin_conversion_rules()` function::
+
+        conversion_rules = get_builtin_conversion_rules('defaults') # all defaults
+
+
+    Rules types:
+    
+      - `RULE_DICT`: If `rule_type` is `RULE_DICT`, then `rule` should be a
+        dictionary whose keys are integers representing unicode code points
+        (e.g., `0x210F`), and whose values are corresponding replacement strings
+        (e.g., ``r'\hbar'``).  See :py:func:`get_builtin_uni2latex_dict()` for
+        an example.
+
+      - `RULE_REGEX`: If `rule_type` is `RULE_REGEX`, then `rule` should be an
+        iterable of tuple pairs `(compiled_regular_expression,
+        replacement_string)` where `compiled_regular_expression` was obtained
+        with `re.compile(...)` and `replacement_string` is anything that can be
+        specified as the second (`repl`) argument of `re.sub(...)`.  This can be
+        a replacement string that includes escapes (like ``\1, \2, \g<name>``)
+        for captured sub-expressions or a callable that takes a match object as
+        argument.
+
+        .. note::
+    
+           The replacement string is parsed like the second argument to
+           `re.sub()` and backslashes have a special meaning because they can
+           refer to captured sub-expressions.  For a literal backslash, use two
+           backslashes ``\\`` in raw strings, four backslashes in normal
+           strings.
+
+        Example::
+
+          regex_conversion_rule = UnicodeToLatexConversionRule(
+              rule_type=RULE_REGEX,
+              rule=[
+                  # protect acronyms of capital letters with braces,
+                  # e.g.: ABC -> {ABC}
+                  (re.compile(r'[A-Z]{2,}'), r'{\1}'),
+                  # Additional rules, e.g., "..." -> "\ldots"
+                  (re.compile(r'...'), r'\\ldots'), # note double \\
+              ]
+          )
+
+      - `RULE_CALLABLE`: If `rule_type` is `RULE_CALLABLE`, then `rule` should
+        be a callable that accepts two arguments, the unicode string and the
+        position in the string (an integer).  The callable will be called with
+        the original unicode string as argument and the position of the
+        character that needs to be encoded.  If this rule can encode the given
+        character at the given position, it should return a tuple
+        `(consumed_length, replacement_string)` where `consumed_length` is the
+        number of characters in the unicode string that `replacement_string`
+        represents.  If the character(s) at the given position can't be encoded
+        by this rule, the callable should return `None` to indicate that further
+        rules should be attempted.
+
+        If the callable accepts an additional argument called `u2lobj`, then the
+        :py:class:`UnicodeToLatexEncoder` instance is provided to that argument.
+
+        For example, the following callable should achieve the same effect as
+        the previous example with regexes::
+
+          def convert_stuff(s, pos):
+              m = re.match(r'[A-Z]{2,}', s, pos)
+              if m is not None:
+                  return (m.end()-m.start(), '{'+m.group()+'}')
+              if s.startswith('...', pos): # or  s[pos:pos+3] == '...'
+                  return (3, r'\ldots')
+              return None
+
+
+    .. versionadded:: 2.0
+
+       This class was introduced in `pylatexenc 2.0`.
+    """
+
+    def __init__(
+        self,
+        rule_type,
+        rule=None,
+        # keyword-only, please:
+        replacement_latex_protection=None,
+    ):
+        self.rule_type = rule_type
+        self.rule = rule
+        self.replacement_latex_protection = replacement_latex_protection
+
+    def __repr__(self):
+        return "{}(rule_type={!r}, rule=<{}>, replacement_latex_protection={})".format(
+            self.__class__.__name__,
+            self.rule_type,
+            type(self.rule).__name__,
+            repr(self.replacement_latex_protection),
+        )
+
+
+def get_builtin_conversion_rules(builtin_name):
+    r"""
+    Return a built-in set of conversion rules specified by a given name
+    `builtin_name`.
+
+    There are two builtin conversion rules, with the following names:
+
+      - `'defaults'`: the default conversion rules, a custom-curated list of
+        unicode chars to LaTeX escapes.
+
+      - `'unicode-xml'`: the conversion rules derived from the `unicode.xml` file
+        maintained at https://www.w3.org/TR/xml-entity-names/#source by David
+        Carlisle.
+
+    The return value is a list of :py:class:`UnicodeToLatexConversionRule`
+    objects that can be either directly specified to the `conversion_rules=`
+    argument of :py:class:`UnicodeToLatexEncoder`, or included in a larger list
+    that can be provided to that argument.
+
+    .. versionadded:: 2.0
+
+       This function was introduced in `pylatexenc 2.0`.
+    """
+    if builtin_name == "defaults":
+        return [
+            UnicodeToLatexConversionRule(
+                rule_type=RULE_DICT, rule=get_builtin_uni2latex_dict()
+            )
+        ]
+    if builtin_name == "unicode-xml":
+        from . import _uni2latexmap_xml
+
+        return [
+            UnicodeToLatexConversionRule(
+                rule_type=RULE_DICT, rule=_uni2latexmap_xml.uni2latex
+            )
+        ]
+    raise ValueError("Unknown builtin rule set: {}".format(builtin_name))
+
+
+class UnicodeToLatexEncoder(object):
+    r"""
+    Encode a string with unicode characters into a LaTeX snippet.
+
+    The following general attributes can be specified as keyword arguments to
+    the constructor.  Note: These attributes must be specified to the
+    constructor and may NOT be subsequently modified.  This is because in the
+    constructor we pre-compile some rules and flags to optimize calls to
+    :py:meth:`unicode_to_text()`.
+
+    .. py:attribute:: non_ascii_only
+
+       Whether we should convert only non-ascii characters into LaTeX sequences,
+       or also all known ascii characters with special LaTeX meaning such as
+       '\\\\', '$', '&', etc.
+
+       If `non_ascii_only` is set to `True` (the default is `False`), then
+       conversion rules are not applied at positions in the string where an
+       ASCII character is encountered.
+
+    .. py:attribute:: conversion_rules
+
+       The conversion rules, specified as a list of
+       :py:class:`UnicodeToLatexConversionRule` objects.  For each position in
+       the string, the rules will be applied in the given sequence until a
+       replacement string is found.
+
+       Instead of a :py:class:`UnicodeToLatexConversionRule` object you may also
+       specify a string specifying a built-in rule (e.g., 'defaults'), which
+       will be expanded to the corresponding rules according to
+       :py:func:`get_builtin_conversion_rules()`.
+
+       If you specify your own list of rules using this argument, you will
+       probably want to include presumably at the end of your list the element
+       'defaults' to include all built-in default conversion rules.  To override
+       built-in rules, simply add your custom rules earlier in the list.
+       Example::
+
+         conversion_rules = [
+             # our custom rules
+             UnicodeToLatexConversionRule(RULE_REGEX, [
+                 # double \\ needed, see UnicodeToLatexConversionRule
+                 ( re.compile(r'...'), r'\\ldots' ),
+                 ( re.compile(r'î'), r'\\^i' ),
+             ]),
+             # plus all the default rules
+             'defaults'
+         ]
+         u = UnicodeToLatexEncoder(conversion_rules=conversion_rules)
+
+    .. py:attribute:: replacement_latex_protection
+
+       How to "protect" LaTeX replacement text that looks like it could be
+       interpreted differently if concatenated to arbitrary strings before and
+       after.
+
+       Currently in the default scheme only one situation is recognized: if the
+       replacement string ends with a latex macro invocation with a non-symbol
+       macro name, e.g. ``\textemdash`` or ``\^\i``.  Indeed, if we naively
+       replace these texts in an arbitrary string (like ``maître``), we might
+       get an invalid macro invocation (like ``ma\^\itre`` which causes un known
+       macro name ``\itre``).
+
+       Possible protection schemes are:
+
+         - 'braces' (the default):  Any suspicious replacement text (that
+           might look fragile) is placed in curly braces ``{...}``.
+
+         - 'braces-all':  All replacement latex escapes are surrounded in
+           protective curly braces ``{...}``, regardless of whether or not they
+           might be deemed "fragile" or "unsafe".
+
+         - 'braces-almost-all':  Almost all replacement latex escapes are
+           surrounded in protective curly braces ``{...}``.  This option
+           emulates closely the behavior of `brackets=True` of the function
+           `utf8tolatex()` in `pylatexenc 1.x`, though I'm not sure it is really
+           useful.  [Specifically, all those replacement strings that start with
+           a backslash are surrounded by curly braces].
+
+         - 'braces-after-macro':  In the situation where the replacement latex
+           code ends with a string-named macro, then a pair of empty braces is
+           added at the end of the replacement text to protect the macro.
+
+         - 'none': No protection is applied, even in "unsafe" cases.  This is
+           not recommended, as this will likely result in invalid LaTeX
+           code. (Note this is the string 'none', not Python's built-in `None`.)
+
+         - any callable object: The callable should take a single argument, the
+           replacement latex string associated with a piece of the input (maybe
+           a special character) that has been encoded; it should return the
+           actual string to append to the output string.
+
+         .. versionadded:: 2.10
+
+            You can specify a callable object to `replacement_latex_protection`
+            since `pylatexenc 2.10`.
+
+    .. py:attribute:: unknown_char_policy
+
+       What to do when a non-ascii character is encountered without any known
+       substitution macro.  The attribute `unknown_char_policy` can be set to one of:
+
+         - 'keep': keep the character as is;
+
+         - 'replace': replace the character by a boldface question mark;
+
+         - 'ignore': ignore the character from the input entirely and don't
+           output anything for it;
+
+         - 'fail': raise a `ValueError` exception;
+
+         - 'unihex': output the unicode hexadecimal code (U+XXXX) of the
+           character in typewriter font;
+
+         - a Python callable --- will be called with argument the character that
+           could not be encoded.  (If the callable accepts a second argument
+           called 'u2lobj', then the `UnicodeToLatexEncoder` instance is
+           provided to that argument.)  The return value of the callable is used
+           as LaTeX replacement code.
+
+    .. py:attribute:: unknown_char_warning
+
+       In addition to the `unknown_char_policy`, this attribute indicates
+       whether or not (`True` or `False`) one should generate a warning when a
+       nonascii character without any known latex representation is
+       encountered. (Default: True)
+
+    .. py:attribute:: latex_string_class
+
+       The return type of :py:meth:`unicode_to_latex()`.  Normally this is a
+       simple unicode string (`str` on `Python 3` or `unicode` on `Python 2`).
+
+       But you can specify your custom string type via the `latex_string_class`
+       argument.  The `latex_string_class` will be invoked with no arguments to
+       construct an empty object (so `latex_string_class` can be either an
+       object that can be constructed with no arguments or it can be a function
+       with no arguments that return a fresh object instance).  The object must
+       support the operation "+=", i.e., you should overload the ``__iadd__()``
+       method.
+
+       For instance, you can record the chunks that would have been appended
+       into a single string as follows::
+
+           class LatexChunkList:
+               def __init__(self):
+                   self.chunks = []
+
+               def __iadd__(self, s):
+                   self.chunks.append(s)
+                   return self
+
+           u = UnicodeToLatexEncoder(latex_string_class=LatexChunkList,
+                                     replacement_latex_protection='none')
+           result = u.unicode_to_latex("é → α")
+           # result.chunks == [ r"\'e", ' ', r'\textrightarrow', ' ',
+           #                    r'\ensuremath{\alpha}' ]
+
+    .. warning::
+
+       None of the above attributes should be modified after constructing the
+       object.  The values specified to the class constructor are final and
+       cannot be changed.  [Indeed, the class constructor "compiles" these
+       attribute values into a data structure that makes
+       :py:meth:`unicode_to_text()` slightly more efficient.]
+
+    .. versionadded:: 2.0
+
+       This class was introduced in `pylatexenc 2.0`.
+    """
+
+    def __init__(self, **kwargs):
+        self.non_ascii_only = kwargs.pop("non_ascii_only", False)
+        self.conversion_rules = kwargs.pop("conversion_rules", ["defaults"])
+        self.replacement_latex_protection = kwargs.pop(
+            "replacement_latex_protection", "braces"
+        )
+        self.unknown_char_policy = kwargs.pop("unknown_char_policy", "keep")
+        self.unknown_char_warning = kwargs.pop("unknown_char_warning", True)
+        self.latex_string_class = kwargs.pop("latex_string_class", unicode)
+
+        if kwargs:
+            logger.warning(
+                "Ignoring unknown keyword arguments: %s", ",".join(kwargs.keys())
+            )
+
+        super(UnicodeToLatexEncoder, self).__init__(**kwargs)
+
+        # build generator that expands built-in conversion rules
+        expanded_conversion_rules = itertools.chain.from_iterable(
+            (get_builtin_conversion_rules(r) if isinstance(r, basestring) else [r])
+            for r in self.conversion_rules
+        )
+
+        #
+        # now "pre-compile" some stuff so that calls to unicode_to_latex() can
+        # hopefully execute faster
+        #
+
+        # "pre-compile" rules and check rule types:
+        self._compiled_rules = []
+        for rule in expanded_conversion_rules:
+            if rule.rule_type == RULE_DICT:
+                self._compiled_rules.append(
+                    functools.partial(self._apply_rule_dict, rule.rule, rule)
+                )
+            elif rule.rule_type == RULE_REGEX:
+                self._compiled_rules.append(
+                    functools.partial(self._apply_rule_regex, rule.rule, rule)
+                )
+            elif rule.rule_type == RULE_CALLABLE:
+                thecallable = rule.rule
+                if "u2lobj" in getfullargspec(thecallable)[0]:
+                    thecallable = functools.partial(rule.rule, u2lobj=self)
+                self._compiled_rules.append(
+                    functools.partial(self._apply_rule_callable, thecallable, rule)
+                )
+            else:
+                raise TypeError("Invalid rule type: {}".format(rule.rule_type))
+
+        # bad char policy:
+        if isinstance(self.unknown_char_policy, basestring):
+            self._do_unknown_char = self._get_method_fn(
+                "do_unknown_char", self.unknown_char_policy, what="unknown_char_policy"
+            )
+        elif callable(self.unknown_char_policy):
+            fn = self.unknown_char_policy
+            if "u2lobj" in getfullargspec(fn)[0]:
+                self._do_unknown_char = functools.partial(
+                    self.unknown_char_policy, u2lobj=self
+                )
+            else:
+                self._do_unknown_char = self.unknown_char_policy
+        else:
+            raise TypeError(
+                "Invalid argument for unknown_char_policy: {!r}".format(
+                    self.unknown_char_policy
+                )
+            )
+
+        # bad char warning:
+        if not self.unknown_char_warning:
+            self._do_warn_unknown_char = lambda ch: None  # replace method by no-op
+
+        # set a method that will skip ascii characters if required:
+        if self.non_ascii_only:
+            self._maybe_skip_ascii = self._check_do_skip_ascii
+        else:
+            self._maybe_skip_ascii = lambda s, p: False
+
+        # set a method to protect replacement latex code, if necessary:
+        self._apply_protection = self._get_replacement_latex_fn(
+            self.replacement_latex_protection
+        )
+
+    def _get_method_fn(self, base, name, what):
+        selfmethname = "_" + base + "_" + name.replace("-", "_")
+        if not hasattr(self, selfmethname):
+            raise ValueError("Invalid {}: {}".format(what, name))
+        return getattr(self, selfmethname)
+
+    def _get_replacement_latex_fn(self, replacement_latex_protection):
+        if callable(replacement_latex_protection):
+            return replacement_latex_protection
+        return self._get_method_fn(
+            "apply_protection",
+            replacement_latex_protection,
+            what="replacement_latex_protection",
+        )
+
+    def unicode_to_latex(self, s):
+        """
+        Convert unicode characters in the string `s` into latex escape sequences,
+        according to the rules and options given to the constructor.
+        """
+
+        s = unicode(s)  # make sure s is unicode
+        s = unicodedata.normalize("NFC", s)
+
+        class _NS:
+            pass
+
+        p = _NS()
+        p.latex = self.latex_string_class()
+        p.pos = 0
+
+        while p.pos < len(s):
+
+            if self._maybe_skip_ascii(s, p):
+                continue
+
+            for compiledrule in self._compiled_rules:
+                if compiledrule(s, p):
+                    break
+            else:
+                # for-else, see
+                # https://docs.python.org/2/tutorial/controlflow.html\
+                # #break-and-continue-statements-and-else-clauses-on-loops
+                ch = s[p.pos]
+                o = ord(ch)
+                if (o >= 32 and o <= 127) or (ch in "\n\r\t"):
+                    p.latex += ch
+                    p.pos += 1
+                else:
+                    self._do_warn_unknown_char(ch)
+                    p.latex += self._do_unknown_char(ch)
+                    p.pos += 1
+
+        return p.latex
+
+    def _check_do_skip_ascii(self, s, p):
+        if ord(s[p.pos]) < 127:
+            # skip, we only want to convert non-ascii chars
+            p.latex += s[p.pos]
+            p.pos += 1
+            return True
+        return False
+
+    def _apply_rule_dict(self, ruledict, rule, s, p):
+        o = ord(s[p.pos])
+        if o in ruledict:
+            self._apply_replacement(p, ruledict[o], 1, rule)
+            return True
+        return None
+
+    def _apply_rule_regex(self, ruleregexes, rule, s, p):
+        for regex, repl in ruleregexes:
+            m = regex.match(s, p.pos)
+            if m is not None:
+                if callable(repl):
+                    replstr = repl(m)
+                else:
+                    replstr = m.expand(repl)
+                self._apply_replacement(p, replstr, m.end() - m.start(), rule)
+                return True
+        return None
+
+    def _apply_rule_callable(self, rulecallable, rule, s, p):
+        res = rulecallable(s, p.pos)
+        if res is None:
+            return None
+        (consumed, repl) = res
+        self._apply_replacement(p, repl, consumed, rule)
+        return True
+
+    def _apply_replacement(self, p, repl, numchars, ruleobj):
+        # check for possible replacement latex protection, like braces.
+
+        protect_fn = self._apply_protection
+
+        # maybe the rule object has overridden the replacement_latex_protection to use.
+        if ruleobj.replacement_latex_protection is not None:
+            protect_fn = self._get_replacement_latex_fn(
+                ruleobj.replacement_latex_protection
+            )
+
+        repl = protect_fn(repl)
+        p.latex += repl
+        p.pos += numchars
+
+    def _apply_protection_none(self, repl):
+        # no protection
+        return repl
+
+    def _apply_protection_braces(self, repl):
+        k = repl.rfind("\\")
+        if k >= 0 and repl[k + 1 :].isalpha():
+            # has dangling named macro, apply protection.
+            return "{" + repl + "}"
+        return repl
+
+    def _apply_protection_braces_almost_all(self, repl):
+        if repl[0:1] == "\\":
+            return "{" + repl + "}"
+        return repl
+
+    def _apply_protection_braces_all(self, repl):
+        return "{" + repl + "}"
+
+    def _apply_protection_braces_after_macro(self, repl):
+        k = repl.rfind("\\")
+        if k >= 0 and repl[k + 1 :].isalpha():
+            # has dangling named macro, apply protection.
+            return repl + "{}"
+        return repl
+
+    # policies for "bad chars":
+    def _do_unknown_char_keep(self, ch):
+        return ch
+
+    def _do_unknown_char_replace(self, ch):
+        return r"{\bfseries ?}"
+
+    def _do_unknown_char_ignore(self, ch):
+        return ""
+
+    def _do_unknown_char_fail(self, ch):
+        raise ValueError(
+            "No known latex representation for character: U+%04X - ‘%s’" % (ord(ch), ch)
+        )
+
+    def _do_unknown_char_unihex(self, ch):
+        return r"\ensuremath{\langle}\texttt{U+%04X}\ensuremath{\rangle}" % (ord(ch))
+
+    def _do_warn_unknown_char(self, ch):
+        logger.warning(
+            "No known latex representation for character: U+%04X - ‘%s’", ord(ch), ch
+        )

good_ai/src/sus/external/pylatexenc/latexwalker/__main__.py

@@ -0,0 +1,230 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2018 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+import argparse
+import fileinput
+import json
+import logging
+import sys
+
+from ..latexwalker import LatexWalker, disp_node, make_json_encoder
+from ..version import version_str
+
+
+def main(argv=None):
+
+    if argv is None:
+        argv = sys.argv[1:]
+
+    parser = argparse.ArgumentParser(prog="latexwalker", add_help=False)
+
+    parser.add_argument(
+        "--output-format",
+        metavar="FORMAT",
+        dest="output_format",
+        choices=["human", "json"],
+        default="human",
+        help='Requested output format for the node tree ("human" or "json")',
+    )
+    parser.add_argument(
+        "--json-indent",
+        metavar="NUMSPACES",
+        dest="json_indent",
+        type=int,
+        default=2,
+        help="Indentation in JSON output (specify number of spaces "
+        "per indentation level)",
+    )
+    parser.add_argument(
+        "--json-compact",
+        dest="json_indent",
+        action="store_const",
+        const=None,
+        help="Output compact JSON",
+    )
+
+    parser.add_argument(
+        "--keep-inline-math",
+        action="store_const",
+        const=True,
+        dest="keep_inline_math",
+        default=True,
+        help=argparse.SUPPRESS,
+    )
+    parser.add_argument(
+        "--no-keep-inline-math",
+        action="store_const",
+        const=False,
+        dest="keep_inline_math",
+        help=argparse.SUPPRESS,
+    )
+
+    parser.add_argument(
+        "--tolerant-parsing",
+        action="store_const",
+        const=True,
+        dest="tolerant_parsing",
+        default=True,
+    )
+    parser.add_argument(
+        "--no-tolerant-parsing",
+        action="store_const",
+        const=False,
+        dest="tolerant_parsing",
+        help="Tolerate syntax errors when parsing, and attempt "
+        "to continue (default yes)",
+    )
+
+    # I'm not sure this flag is useful and if it should be exposed at all.
+    # Accept it, but make it hidden.
+    parser.add_argument(
+        "--strict-braces",
+        action="store_const",
+        const=True,
+        dest="strict_braces",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
+    parser.add_argument(
+        "--no-strict-braces",
+        action="store_const",
+        const=False,
+        dest="strict_braces",
+        # help="Report errors for mismatching LaTeX braces (default no)"
+        help=argparse.SUPPRESS,
+    )
+
+    parser.add_argument(
+        "-q",
+        "--quiet",
+        dest="logging_level",
+        action="store_const",
+        const=logging.ERROR,
+        default=logging.INFO,
+        help="Suppress warning messages",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        dest="logging_level",
+        action="store_const",
+        const=logging.DEBUG,
+        help="Verbose output",
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version="pylatexenc {}".format(version_str),
+        help="Show version information and exit",
+    )
+    parser.add_argument(
+        "--help", action="help", help="Show this help information and exit"
+    )
+
+    parser.add_argument(
+        "--code",
+        "-c",
+        action="store",
+        default=None,
+        metavar="LATEX_CODE",
+        help="Convert the given LATEX_CODE to unicode text instead of reading "
+        "from FILE or standard input.  You cannot specify FILEs if you use this "
+        "option, and any standard input is ignored.",
+    )
+
+    parser.add_argument(
+        "files",
+        metavar="FILE",
+        nargs="*",
+        help="Input files (if none specified, read from stdandard input)",
+    )
+
+    args = parser.parse_args(argv)
+
+    logging.basicConfig()
+    logging.getLogger().setLevel(args.logging_level)
+    logger = logging.getLogger(__name__)
+
+    latex = ""
+    if args.code:
+        if args.files:
+            logger.error(
+                "Cannot specify both FILEs and --code option. "
+                "Use --help option for more information."
+            )
+            sys.exit(1)
+        latex = args.code
+    else:
+        for line in fileinput.input(files=args.files):
+            latex += line
+
+    latexwalker = LatexWalker(
+        latex, tolerant_parsing=args.tolerant_parsing, strict_braces=args.strict_braces
+    )
+
+    (nodelist, pos, len_) = latexwalker.get_latex_nodes()
+
+    if args.output_format == "human":
+        print("\n--- NODES ---\n")
+        for n in nodelist:
+            disp_node(n)
+        print("\n-------------\n")
+        return
+
+    if args.output_format == "json":
+        json.dump(
+            {
+                "nodelist": nodelist,
+            },
+            sys.stdout,
+            cls=make_json_encoder(latexwalker),
+            indent=args.json_indent,
+        )
+        sys.stdout.write("\n")
+        return
+
+    raise ValueError("Invalid output format: " + args.output_format)
+
+
+def run_main():
+
+    try:
+
+        main()
+
+    except SystemExit:
+        raise
+    except:  # lgtm [py/catch-base-exception]
+        import pdb
+        import traceback
+
+        traceback.print_exc()
+        pdb.post_mortem()
+
+
+if __name__ == "__main__":
+
+    # run_main() # debug
+    main()

good_ai/src/sus/external/pylatexenc/latexwalker/_defaultspecs.py

@@ -0,0 +1,459 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2019 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+
+# Internal module. May change without notice.
+
+
+from ..macrospec import (
+    EnvironmentSpec,
+    MacroSpec,
+    MacroStandardArgsParser,
+    VerbatimArgsParser,
+    std_environment,
+    std_macro,
+    std_specials,
+)
+
+specs = [
+    #
+    # CATEGORY: latex-base
+    #
+    (
+        "latex-base",
+        {
+            "macros": [
+                std_macro("documentclass", True, 1),
+                std_macro("usepackage", True, 1),
+                std_macro("RequirePackage", True, 1),
+                std_macro("selectlanguage", True, 1),
+                std_macro("setlength", True, 2),
+                std_macro("addlength", True, 2),
+                std_macro("setcounter", True, 2),
+                std_macro("addcounter", True, 2),
+                std_macro("newcommand", "*{[[{"),
+                std_macro("renewcommand", "*{[[{"),
+                std_macro("providecommand", "*{[[{"),
+                std_macro("newenvironment", "*{[[{{"),
+                std_macro("renewenvironment", "*{[[{{"),
+                std_macro("provideenvironment", "*{[[{{"),
+                std_macro("DeclareMathOperator", "*{{"),
+                std_macro("hspace", "*{"),
+                std_macro("vspace", "*{"),
+                MacroSpec(
+                    "mbox",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                # \title, \author, \date
+                MacroSpec("title", "{"),
+                MacroSpec("author", "{"),
+                MacroSpec("date", "{"),
+                # (Note: single backslash) end of line with optional no-break ('*') and
+                # additional vertical spacing, e.g. \\*[2mm]
+                #
+                # Special for this command: don't allow an optional spacing argument
+                # [2mm] to be separated by spaces from the rest of the macro.  This
+                # emulates the behavior in AMS environments, and avoids some errors;
+                # e.g. in "\begin{align} A=0 \\ [C,D]=0 \end{align}" the "[C,D]"
+                # does not get captured as an optional macro argument.
+                MacroSpec(
+                    "\\",
+                    args_parser=MacroStandardArgsParser(
+                        "*[", optional_arg_no_space=True
+                    ),
+                ),
+                std_macro("item", True, 0),
+                # \input{someotherfile}
+                std_macro("input", False, 1),
+                std_macro("include", False, 1),
+                std_macro("includegraphics", True, 1),
+                std_macro("chapter", "*[{"),
+                std_macro("section", "*[{"),
+                std_macro("subsection", "*[{"),
+                std_macro("subsubsection", "*[{"),
+                std_macro("pagagraph", "*[{"),
+                std_macro("subparagraph", "*[{"),
+                std_macro("bibliography", "{"),
+                std_macro("emph", False, 1),
+                MacroSpec(
+                    "textrm",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "textit",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "textbf",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "textmd",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "textsc",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "textsf",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "textsl",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "texttt",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "textup",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "text",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                std_macro("mathrm", False, 1),  # only allowed in math mode anyway
+                std_macro("mathbb", False, 1),  # only allowed in math mode anyway
+                std_macro("mathbf", False, 1),
+                std_macro("mathit", False, 1),
+                std_macro("mathsf", False, 1),
+                std_macro("mathtt", False, 1),
+                std_macro("mathcal", False, 1),
+                std_macro("mathscr", False, 1),
+                std_macro("mathfrak", False, 1),
+                std_macro("label", False, 1),
+                std_macro("ref", False, 1),
+                std_macro("autoref", False, 1),
+                std_macro("cref", False, 1),
+                std_macro("Cref", False, 1),
+                std_macro("eqref", False, 1),
+                std_macro("url", False, 1),
+                std_macro("hypersetup", False, 1),
+                std_macro("footnote", True, 1),
+                std_macro("keywords", False, 1),
+                std_macro("hphantom", True, 1),
+                std_macro("vphantom", True, 1),
+                std_macro("'", False, 1),
+                std_macro("`", False, 1),
+                std_macro('"', False, 1),
+                std_macro("c", False, 1),
+                std_macro("^", False, 1),
+                std_macro("~", False, 1),
+                std_macro("H", False, 1),
+                std_macro("k", False, 1),
+                std_macro("=", False, 1),
+                std_macro("b", False, 1),
+                std_macro(".", False, 1),
+                std_macro("d", False, 1),
+                std_macro("r", False, 1),
+                std_macro("u", False, 1),
+                std_macro("v", False, 1),
+                MacroSpec(
+                    "ensuremath",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[True]),
+                ),
+                std_macro("not", False, 1),
+                std_macro("vec", False, 1),
+                std_macro("dot", False, 1),
+                std_macro("hat", False, 1),
+                std_macro("check", False, 1),
+                std_macro("breve", False, 1),
+                std_macro("acute", False, 1),
+                std_macro("grave", False, 1),
+                std_macro("tilde", False, 1),
+                std_macro("bar", False, 1),
+                std_macro("ddot", False, 1),
+                std_macro("frac", False, 2),
+                std_macro("nicefrac", False, 2),
+                std_macro("sqrt", True, 1),
+                MacroSpec("overline", "{"),
+                MacroSpec("underline", "{"),
+                MacroSpec("widehat", "{"),
+                MacroSpec("widetilde", "{"),
+                MacroSpec("wideparen", "{"),
+                MacroSpec("overleftarrow", "{"),
+                MacroSpec("overrightarrow", "{"),
+                MacroSpec("overleftrightarrow", "{"),
+                MacroSpec("underleftarrow", "{"),
+                MacroSpec("underrightarrow", "{"),
+                MacroSpec("underleftrightarrow", "{"),
+                MacroSpec("overbrace", "{"),
+                MacroSpec("underbrace", "{"),
+                MacroSpec("overgroup", "{"),
+                MacroSpec("undergroup", "{"),
+                MacroSpec("overbracket", "{"),
+                MacroSpec("underbracket", "{"),
+                MacroSpec("overlinesegment", "{"),
+                MacroSpec("underlinesegment", "{"),
+                MacroSpec("overleftharpoon", "{"),
+                MacroSpec("overrightharpoon", "{"),
+                MacroSpec("xleftarrow", "[{"),
+                MacroSpec("xrightarrow", "[{"),
+                std_macro("ket", False, 1),
+                std_macro("bra", False, 1),
+                std_macro("braket", False, 2),
+                std_macro("ketbra", False, 2),
+                std_macro("texorpdfstring", False, 2),
+                # xcolor commands
+                MacroSpec("definecolor", "[{{{"),
+                MacroSpec("providecolor", "[{{{"),
+                MacroSpec("colorlet", "[{[{"),
+                MacroSpec("color", "[{"),
+                MacroSpec("textcolor", "[{{"),
+                MacroSpec("pagecolor", "[{"),
+                MacroSpec("nopagecolor", ""),
+                MacroSpec("colorbox", "[{{"),
+                MacroSpec("fcolorbox", "[{[{{"),
+                MacroSpec("boxframe", "{{{"),
+                MacroSpec("rowcolors", "*[{{{"),
+            ],
+            "environments": [
+                # NOTE: Starred variants (as in \begin{equation*}) are not specified as
+                # for macros with an argspec='*'.  Rather, we need to define a separate
+                # spec for the starred variant as the star really is part of the
+                # environment name.  If you specify argspec='*', the parser will try to
+                # look for an expression of the form '\begin{equation}*'
+                std_environment("figure", "["),
+                std_environment("figure*", "["),
+                std_environment("table", "["),
+                std_environment("table*", "["),
+                std_environment("abstract", None),
+                std_environment("tabular", "{"),
+                std_environment("tabular*", "{{"),
+                std_environment("tabularx", "{[{"),
+                std_environment("array", "[{"),
+                std_environment("equation", None, is_math_mode=True),
+                std_environment("equation*", None, is_math_mode=True),
+                std_environment("eqnarray", None, is_math_mode=True),
+                std_environment("eqnarray*", None, is_math_mode=True),
+                # AMS environments
+                std_environment("align", None, is_math_mode=True),
+                std_environment("align*", None, is_math_mode=True),
+                std_environment("gather", None, is_math_mode=True),
+                std_environment("gather*", None, is_math_mode=True),
+                std_environment("flalign", None, is_math_mode=True),
+                std_environment("flalign*", None, is_math_mode=True),
+                std_environment("multline", None, is_math_mode=True),
+                std_environment("multline*", None, is_math_mode=True),
+                std_environment("alignat", "{", is_math_mode=True),
+                std_environment("alignat*", "{", is_math_mode=True),
+                std_environment("split", None, is_math_mode=True),
+            ],
+            "specials": [
+                std_specials("&"),
+                # TODO --- for this, we need to parse their argument but don't use
+                #          the standard args parser because we need to be able to
+                #          accept arguments like "x_\mathrm{initial}"
+                #
+                # std_specials('^'),
+                # std_specials('_'),
+            ],
+        },
+    ),
+    #
+    # CATEGORY: nonascii-specials
+    #
+    (
+        "nonascii-specials",
+        {
+            "macros": [],
+            "environments": [],
+            "specials": [
+                std_specials("~"),
+                # cf. https://tex.stackexchange.com/a/439652/32188 "fake ligatures":
+                std_specials("``"),
+                std_specials("''"),
+                std_specials("--"),
+                std_specials("---"),
+                std_specials("!`"),
+                std_specials("?`"),
+            ],
+        },
+    ),
+    #
+    # CATEGORY: verbatim
+    #
+    (
+        "verbatim",
+        {
+            "macros": [
+                MacroSpec(
+                    "verb",
+                    args_parser=VerbatimArgsParser(verbatim_arg_type="verb-macro"),
+                ),
+            ],
+            "environments": [
+                EnvironmentSpec(
+                    "verbatim",
+                    args_parser=VerbatimArgsParser(
+                        verbatim_arg_type="verbatim-environment"
+                    ),
+                ),
+            ],
+            "specials": [
+                # optionally users could include the specials "|" like in latex-doc
+                # for verbatim |\like \this|...
+            ],
+        },
+    ),
+    #
+    # CATEGORY: theorems
+    #
+    (
+        "theorems",
+        {
+            "macros": [],
+            "environments": [
+                std_environment("theorem", "["),
+                std_environment("proposition", "["),
+                std_environment("lemma", "["),
+                std_environment("corollary", "["),
+                std_environment("definition", "["),
+                std_environment("conjecture", "["),
+                std_environment("remark", "["),
+                #
+                std_environment("proof", "["),
+                # short names
+                std_environment("thm", "["),
+                std_environment("prop", "["),
+                std_environment("lem", "["),
+                std_environment("cor", "["),
+                std_environment("conj", "["),
+                std_environment("rem", "["),
+                std_environment("defn", "["),
+            ],
+            "specials": [],
+        },
+    ),
+    #
+    # CATEGORY: enumitem
+    #
+    (
+        "enumitem",
+        {
+            "macros": [],
+            "environments": [
+                std_environment("enumerate", "["),
+                std_environment("itemize", "["),
+                std_environment("description", "["),
+            ],
+            "specials": [],
+        },
+    ),
+    #
+    # CATEGORY: natbib
+    #
+    (
+        "natbib",
+        {
+            "macros": [
+                std_macro("cite", "*[[{"),
+                std_macro("citet", "*[[{"),
+                std_macro("citep", "*[[{"),
+                std_macro("citealt", "*[[{"),
+                std_macro("citealp", "*[[{"),
+                std_macro("citeauthor", "*[[{"),
+                std_macro("citefullauthor", "[[{"),
+                std_macro("citeyear", "[[{"),
+                std_macro("citeyearpar", "[[{"),
+                std_macro("Citet", "*[[{"),
+                std_macro("Citep", "*[[{"),
+                std_macro("Citealt", "*[[{"),
+                std_macro("Citealp", "*[[{"),
+                std_macro("Citeauthor", "*[[{"),
+                std_macro("citetext", "{"),
+                std_macro("citenum", "{"),
+                std_macro("defcitealias", "{{"),
+                std_macro("citetalias", "[[{"),
+                std_macro("citepalias", "[[{"),
+            ],
+            "environments": [],
+            "specials": [],
+        },
+    ),
+    #
+    # CATEGORY: latex-ethuebung
+    #
+    (
+        "latex-ethuebung",
+        {
+            "macros": [
+                # ethuebung
+                std_macro("UebungLoesungFont", False, 1),
+                std_macro("UebungHinweisFont", False, 1),
+                std_macro("UebungExTitleFont", False, 1),
+                std_macro("UebungSubExTitleFont", False, 1),
+                std_macro("UebungTipsFont", False, 1),
+                std_macro("UebungLabel", False, 1),
+                std_macro("UebungSubLabel", False, 1),
+                std_macro("UebungLabelEnum", False, 1),
+                std_macro("UebungLabelEnumSub", False, 1),
+                std_macro("UebungSolLabel", False, 1),
+                std_macro("UebungHinweisLabel", False, 1),
+                std_macro("UebungHinweiseLabel", False, 1),
+                std_macro("UebungSolEquationLabel", False, 1),
+                std_macro("UebungTipsLabel", False, 1),
+                std_macro("UebungTipsEquationLabel", False, 1),
+                std_macro("UebungsblattTitleSeries", False, 1),
+                std_macro("UebungsblattTitleSolutions", False, 1),
+                std_macro("UebungsblattTitleTips", False, 1),
+                std_macro("UebungsblattNumber", False, 1),
+                std_macro("UebungsblattTitleFont", False, 1),
+                std_macro("UebungTitleCenterVSpacing", False, 1),
+                std_macro("UebungAttachedSolutionTitleTop", False, 1),
+                std_macro("UebungAttachedSolutionTitleFont", False, 1),
+                std_macro("UebungAttachedSolutionTitle", False, 1),
+                std_macro("UebungTextAttachedSolution", False, 1),
+                std_macro("UebungDueByLabel", False, 1),
+                std_macro("UebungDueBy", False, 1),
+                std_macro("UebungLecture", False, 1),
+                std_macro("UebungProf", False, 1),
+                std_macro("UebungLecturer", False, 1),
+                std_macro("UebungSemester", False, 1),
+                std_macro("UebungLogoFile", False, 1),
+                std_macro("UebungLanguage", False, 1),
+                std_macro("UebungStyle", False, 1),
+                #
+                std_macro("uebung", "{["),
+                std_macro("exercise", "{["),
+                std_macro("keywords", False, 1),
+                std_macro("subuebung", False, 1),
+                std_macro("subexercise", False, 1),
+                std_macro("pdfloesung", True, 1),
+                std_macro("pdfsolution", True, 1),
+                std_macro("exenumfulllabel", False, 1),
+                std_macro("hint", False, 1),
+                std_macro("hints", False, 1),
+                std_macro("hinweis", False, 1),
+                std_macro("hinweise", False, 1),
+            ],
+            "environments": [],
+            "specials": [],
+        },
+    ),
+]

good_ai/src/sus/external/pylatexenc/macrospec/__init__.py

@@ -0,0 +1,785 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2019 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+r"""
+Provides classes and helper functions to describe a LaTeX context of known
+macros and environments, specifying how they should be parsed by
+:py:mod:`pylatexenc.latexwalker`.
+
+.. versionadded:: 2.0
+
+   The entire module :py:mod:`pylatexenc.macrospec` was introduced in
+   `pylatexenc 2.0`.
+"""
+
+
+import sys
+
+if sys.version_info.major > 2:
+    # Py3
+    def unicode(s):
+        return s
+
+    _basestring = str
+    _str_from_unicode = lambda x: x
+    _unicode_from_str = lambda x: x
+else:
+    # Py2
+    _basestring = basestring
+    _str_from_unicode = lambda x: unicode(x).encode("utf-8")
+    _unicode_from_str = lambda x: x.decode("utf-8")
+
+
+# ------------------------------------------------------------------------------
+
+from ._argparsers import (
+    MacroStandardArgsParser,
+    ParsedMacroArgs,
+    ParsedVerbatimArgs,
+    VerbatimArgsParser,
+)
+
+# ------------------------------------------------------------------------------
+
+
+class MacroSpec(object):
+    r"""
+    Stores the specification of a macro.
+
+    This stores the macro name and instructions on how to parse the macro
+    arguments.
+
+    .. py:attribute:: macroname
+
+       The name of the macro, without the leading backslash.
+
+    .. py:attribute:: args_parser
+
+       The parser instance that can understand this macro's arguments.  For
+       standard LaTeX macros this is usually a
+       :py:class:`MacroStandardArgsParser` instance.
+
+       If you specify a string, then for convenience this is interpreted as an
+       argspec argument for :py:class:`MacroStandardArgsParser` and such an
+       instance is automatically created.
+    """
+
+    def __init__(self, macroname, args_parser=MacroStandardArgsParser(), **kwargs):
+        super(MacroSpec, self).__init__(**kwargs)
+        self.macroname = macroname
+        if isinstance(args_parser, _basestring):
+            self.args_parser = MacroStandardArgsParser(args_parser)
+        else:
+            self.args_parser = args_parser
+
+    def parse_args(self, *args, **kwargs):
+        r"""
+        Shorthand for calling the :py:attr:`args_parser`\ 's `parse_args()` method.
+        See :py:class:`MacroStandardArgsParser`.
+        """
+        return self.args_parser.parse_args(*args, **kwargs)
+
+    def __repr__(self):
+        return "MacroSpec(macroname=%r, args_parser=%r)" % (
+            self.macroname,
+            self.args_parser,
+        )
+
+
+class EnvironmentSpec(object):
+    r"""
+    Stores the specification of a LaTeX environment.
+
+    This stores the environment name and instructions on how to parse any
+    arguments provided after ``\begin{environment}<args>``.
+
+    .. py:attribute:: environmentname
+
+       The name of the environment, i.e., the argument of ``\begin{...}`` and
+       ``\end{...}``.
+
+    .. py:attribute:: args_parser
+
+       The parser instance that can understand this environment's arguments.
+       For standard LaTeX environment this is usually a
+       :py:class:`MacroStandardArgsParser` instance.
+
+       If you specify a string, then for convenience this is interpreted as an
+       argspec argument for :py:class:`MacroStandardArgsParser` and such an
+       instance is automatically created.
+
+    .. py:attribute:: is_math_mode
+
+       A boolean that indicates whether or not the contents is to be interpreted
+       in Math Mode.  This would be True for environments like
+       ``\begin{equation}``, ``\begin{align}``, etc., but False for
+       ``\begin{figure}``, etc.
+
+    .. note::
+
+       Starred variants of environments (as in ``\begin{equation*}``) must not
+       be specified using an argspec as for macros (e.g., `argspec='*'`).
+       Rather, we need to define a separate environment spec for the starred
+       variant with the star in the name itself (``EnvironmentSpec('equation*',
+       None)``) because the star really is part of the environment name.  If you
+       happened to use ``EnvironmentSpec('equation', '*')``, then the parser
+       would recognize the expression ``\begin{equation}*`` but not
+       ``\begin{equation*}``.
+    """
+
+    def __init__(
+        self,
+        environmentname,
+        args_parser=MacroStandardArgsParser(),
+        is_math_mode=False,
+        **kwargs
+    ):
+        super(EnvironmentSpec, self).__init__(**kwargs)
+        self.environmentname = environmentname
+        if isinstance(args_parser, _basestring):
+            self.args_parser = MacroStandardArgsParser(args_parser)
+        else:
+            self.args_parser = args_parser
+        self.is_math_mode = is_math_mode
+
+    def parse_args(self, *args, **kwargs):
+        r"""
+        Shorthand for calling the :py:attr:`args_parser`\ 's `parse_args()` method.
+        See :py:class:`MacroStandardArgsParser`.
+        """
+        return self.args_parser.parse_args(*args, **kwargs)
+
+    def __repr__(self):
+        return (
+            "EnvironmentSpec(environmentname=%r, args_parser=%r, is_math_mode=%r)"
+            % (self.environmentname, self.args_parser, self.is_math_mode)
+        )
+
+
+class SpecialsSpec(object):
+    r"""
+    Specification of a LaTeX "special char sequence": an active char, a
+    ligature, or some other non-macro char sequence that has a special meaning.
+
+    For instance, '&', '~', and '``' are considered as "specials".
+
+    .. py:attribute:: specials_chars
+
+       The string (one or several characters) that has a special meaning. E.g.,
+       '&', '~', '``', etc.
+
+    .. py:attribute:: args_parser
+
+       A parser (e.g. :py:class:`MacroStandardArgsParser`) that is invoked when
+       the specials is encountered.  Can/should be set to `None` if the specials
+       should not parse any arguments (e.g. '~').
+    """
+
+    def __init__(self, specials_chars, args_parser=None, **kwargs):
+        super(SpecialsSpec, self).__init__(**kwargs)
+        self.specials_chars = specials_chars
+        self.args_parser = args_parser
+
+    def parse_args(self, *args, **kwargs):
+        r"""
+        Basically a shorthand for calling the :py:attr:`args_parser`\ 's
+        `parse_args()` method.  See :py:class:`MacroStandardArgsParser`.
+
+        If however the py:attr:`args_parser` attribute is `None`, then this
+        method returns `None`.
+        """
+        if self.args_parser is None:
+            return None
+        return self.args_parser.parse_args(*args, **kwargs)
+
+    def __repr__(self):
+        return "SpecialsSpec(specials_chars=%r, args_parser=%r)" % (
+            self.specials_chars,
+            self.args_parser,
+        )
+
+
+# ------------------------------------------------------------------------------
+
+
+def std_macro(macname, *args, **kwargs):
+    r"""
+    Return a macro specification for the given macro.  Syntax::
+
+      spec = std_macro(macname, argspec)
+      #  or
+      spec = std_macro(macname, optarg, numargs)
+      #  or
+      spec = std_macro( (macname, argspec), )
+      #  or
+      spec = std_macro( (macname, optarg, numargs), )
+      #  or
+      spec = std_macro( spec ) # spec is already a `MacroSpec` -- no-op
+
+    - `macname` is the name of the macro, without the leading backslash.
+
+    - `argspec` is a string either characters "\*", "{" or "[", in which star
+      indicates an optional asterisk character (e.g. starred macro variants),
+      each curly brace specifies a mandatory argument and each square bracket
+      specifies an optional argument in square brackets.  For example, "{{\*[{"
+      expects two mandatory arguments, then an optional star, an optional
+      argument in square brackets, and then another mandatory argument.
+
+      `argspec` may also be `None`, which is the same as ``argspec=''``.
+
+    - `optarg` may be one of `True`, `False`, or `None`, corresponding to these
+      possibilities:
+
+      + if `True`, the macro expects as first argument an optional argument in
+        square brackets. Then, `numargs` specifies the number of additional
+        mandatory arguments to the command, given in usual curly braces (or
+        simply as one TeX token like a single macro)
+
+      + if `False`, the macro only expects a number of mandatory arguments given
+        by `numargs`. The mandatory arguments are given in usual curly braces
+        (or simply as one TeX token like a single macro)
+
+      + if `None`, then `numargs` is a string like `argspec` above.  I.e.,
+        ``std_macro(macname, None, argspec)`` is the same as
+        ``std_macro(macname, argspec)``.
+
+    - `numargs`: depends on `optarg`, see above.
+
+    To make environment specifications (:py:class:`EnvironmentSpec`) instead of
+    a macro specification, use the function :py:func:`std_environment()`
+    instead.
+
+    The helper function :py:func:`std_environment()` is a shorthand for calling
+    this function with additional keyword arguments.  An optional keyword
+    argument `make_environment_spec=True` to the present function may be
+    specified to return an `EnvironmentSpec` instead of a `MacroSpec`.  In this
+    case, you can further specify the `environment_is_math_mode=True|False` to
+    specify whether of not the environment represents a math mode.
+    """
+
+    if isinstance(macname, tuple):
+        if len(args) != 0:
+            raise TypeError(
+                "No positional arguments expected if first argument is a tuple"
+            )
+        args = tuple(macname[1:])
+        macname = macname[0]
+
+    if isinstance(macname, MacroSpec):
+        if len(args) != 0:
+            raise TypeError(
+                "No positional arguments expected if first argument is a MacroSpec"
+            )
+        return macname
+
+    if isinstance(macname, EnvironmentSpec):
+        if len(args) != 0:
+            raise TypeError(
+                "No positional arguments expected if first argument is a EnvironmentSpec"
+            )
+        return macname
+
+    if len(args) == 1:
+        # std_macro(macname, argspec)
+        argspec = args[0]
+    elif len(args) != 2:
+        raise TypeError(
+            "Wrong number of arguments for std_macro, macname={!r}, args={!r}".format(
+                macname, args
+            )
+        )
+    elif not args[0] and isinstance(args[1], _basestring):
+        # argspec given in numargs
+        argspec = args[1]
+    else:
+        argspec = ""
+        if args[0]:
+            argspec = "["
+        argspec += "{" * args[1]
+
+    if kwargs.get("make_environment_spec", False):
+        return EnvironmentSpec(
+            macname,
+            args_parser=MacroStandardArgsParser(argspec),
+            is_math_mode=kwargs.get("environment_is_math_mode", False),
+        )
+    return MacroSpec(macname, args_parser=MacroStandardArgsParser(argspec))
+
+
+def std_environment(envname, *args, **kwargs):
+    r"""
+    Return an environment specification for the given environment.  Syntax::
+
+      spec = std_environment(envname, argspec, is_math_mode=True|False)
+      #  or
+      spec = std_environment(envname, optarg, numargs, is_math_mode=True|False)
+      #  or
+      spec = std_environment( (envname, argspec), is_math_mode=True|False)
+      #  or
+      spec = std_environment( (envname, optarg, numargs), is_math_mode=True|False)
+      #  or
+      spec = std_environment( spec ) # spec is already a `EnvironmentSpec` -- no-op
+
+    - `envname` is the name of the environment, i.e., the argument to
+      ``\begin{...}``.
+
+    - `argspec` is a string either characters "\*", "{" or "[", in which star
+      indicates an optional asterisk character (e.g. starred environment
+      variants), each curly brace specifies a mandatory argument and each square
+      bracket specifies an optional argument in square brackets.  For example,
+      "{{\*[{" expects two mandatory arguments, then an optional star, an
+      optional argument in square brackets, and then another mandatory argument.
+
+      `argspec` may also be `None`, which is the same as ``argspec=''``.
+
+    .. note::
+
+       See :py:class:`EnvironmentSpec` for an important remark about starred
+       variants for environments.  TL;DR: a starred verison of an environment is
+       defined as a separate `EnvironmentSpec` with the star in the name and
+       *not* using an ``argspec='*'``.
+
+    - `optarg` may be one of `True`, `False`, or `None`, corresponding to these
+      possibilities:
+
+      + if `True`, the environment expects as first argument an optional argument in
+        square brackets. Then, `numargs` specifies the number of additional
+        mandatory arguments to the command, given in usual curly braces (or
+        simply as one TeX token like a single environment)
+
+      + if `False`, the environment only expects a number of mandatory arguments given
+        by `numargs`. The mandatory arguments are given in usual curly braces
+        (or simply as one TeX token like a single environment)
+
+      + if `None`, then `numargs` is a string like `argspec` above.  I.e.,
+        ``std_environment(envname, None, argspec)`` is the same as
+        ``std_environment(envname, argspec)``.
+
+    - `numargs`: depends on `optarg`, see above.
+
+    - `is_math_mode`: if set to True, then the environment represents a math
+      mode environment (e.g., 'equation', 'align', 'gather', etc.), i.e., whose
+      contents should be parsed in an appropriate math mode.  Note that
+      `is_math_mode` *must* be given as a keyword argument, in contrast to all
+      other arguments which must be positional (non-keyword) arguments.
+    """
+    is_math_mode = kwargs.pop("is_math_mode", False)
+    kwargs2 = dict(kwargs)
+    kwargs2.update(make_environment_spec=True, environment_is_math_mode=is_math_mode)
+    return std_macro(envname, *args, **kwargs2)
+
+
+def std_specials(specials_chars):
+    r"""
+    Return a latex specials specification for the given character sequence.  Syntax::
+
+      spec = std_specials(specials_chars)
+
+    where `specials_chars` is the sequence of characters that has a special
+    LaTeX meaning, e.g. ``&`` or ``''``.
+
+    This helper function only allows to create specs for simple specials without
+    any argument parsing.  For more complicated specials, you can instantiate a
+    :py:class:`SpecialsSpec` directly.
+    """
+    return SpecialsSpec(specials_chars, args_parser=None)
+
+
+# ------------------------------------------------------------------------------
+
+
+class LatexContextDb(object):
+    r"""
+    Store a database of specifications of known macros, environments, and other
+    latex specials.  This might be, e.g., how many arguments a macro accepts, or
+    how to determine the text representation of a macro or environment.
+
+    When used with :py:class:`pylatexenc.latexwalker.LatexWalker`, the
+    specifications describe mostly rules for parsing arguments of macros and
+    environments, and which sequences of characters to consider as "latex
+    specials".  Specifications for macros, environments, and other specials are
+    stored as :py:class:`MacroSpec`, :py:class:`EnvironmentSpec`, and
+    :py:class:`SpecialsSpec` instances, respectively.
+    When used with :py:class:`pylatexenc.latex2text.LatexNodes2Text`, the
+    specifications for macros, environments, and other specials are stored as
+    :py:class:`pylatexenc.latex2text.MacroTextSpec` ,
+    :py:class:`pylatexenc.latex2text.EnvironmentTextSpec`, and
+    :py:class:`pylatexenc.latex2text.SpecialsTextSpec` instances, respectively.
+
+    In fact, the objects stored in this database may be of any type, except that
+    macro specifications must have an attribute `macroname`, environment
+    specifications must have an attribute `environmentname`, and specials
+    specification must have an attribute `specials_chars`.
+
+    The `LatexContextDb` instance is meant to be (pseudo-)immutable.  Once
+    constructed and all the definitions added with
+    :py:meth:`add_context_category()`, one should refrain from modifying it
+    directly after providing it to, e.g., a
+    :py:class:`~pylatexenc.latexwalker.LatexWalker` object.  The reason is that
+    the latex walker keeps track of what the latex context was when parsing
+    nodes, and modifying the context will modify that stored information, too.
+    Instead of being tempted to modify the object, create a new one with
+    :py:meth:`filter_context()`.
+
+    See :py:func:`pylatexenc.latexwalker.get_default_latex_context_db()` for the
+    default latex context for `latexwalker` with a default collection of known
+    latex macros and environments.
+    See :py:func:`pylatexenc.latex2text.get_default_latex_context_db()` for the
+    default latex context for `latex2text` with a set of text replacements for a
+    collection of known macros and environments.
+    """
+
+    def __init__(self, **kwargs):
+        super(LatexContextDb, self).__init__(**kwargs)
+
+        self.category_list = []
+        self.d = {}
+
+        self.unknown_macro_spec = None
+        self.unknown_environment_spec = None
+        self.unknown_specials_spec = None
+
+    def add_context_category(
+        self,
+        category,
+        macros=[],
+        environments=[],
+        specials=[],
+        prepend=False,
+        insert_before=None,
+        insert_after=None,
+    ):
+        r"""
+        Register a category of macro and environment specifications in the context
+        database.
+
+        The category name `category` must not already exist in the database.
+
+        The argument `macros` is an iterable (e.g., a list) of macro
+        specification objects.  The argument `environments` is an iterable
+        (e.g., a list) of environment spec objects.  Similarly, the `specials`
+        argument is an iterable of latex specials spec instances.
+
+        If you specify `prepend=True`, then macro and environment lookups will
+        prioritize this category over other categories.  Categories are normally
+        searched for in the order they are registered to the database; if you
+        specify `prepend=True`, then the new category is prepended to the
+        existing list so that it is searched first.
+
+        If `insert_before` is not `None`, then it must be a string; the
+        definitions are inserted in the category list immediately before the
+        given category name, or at the beginning of the list if the given
+        category doesn't exist.  If `insert_after` is not `None`, then it must
+        be a string; the definitions are inserted in the category list
+        immediately after the given category name, or at the end of the list if
+        the given category doesn't exist.
+
+        You may only specify one of `prepend=True`, `insert_before='...'` or
+        `insert_after='...'`.
+        """
+
+        if category in self.category_list:
+            raise ValueError(
+                "Category {} is already registered in the context database".format(
+                    category
+                )
+            )
+
+        # ensure only one of these options is set
+        if len([x for x in (prepend, insert_before, insert_after) if x]) > 1:
+            raise TypeError(
+                "add_context_category(): You may only specify one of "
+                "prepend=True, insert_before=... or insert_after=..."
+            )
+
+        if prepend:
+            self.category_list.insert(0, category)
+        elif insert_before:
+            if insert_before in self.category_list:
+                i = self.category_list.index(insert_before)
+            else:
+                i = 0
+            self.category_list.insert(i, category)
+        elif insert_after:
+            if insert_after in self.category_list:
+                i = (
+                    self.category_list.index(insert_after) + 1
+                )  # insert after found category
+            else:
+                i = len(self.category_list)
+            self.category_list.insert(i, category)
+        else:
+            self.category_list.append(category)
+
+        self.d[category] = {
+            "macros": dict((m.macroname, m) for m in macros),
+            "environments": dict((e.environmentname, e) for e in environments),
+            "specials": dict((s.specials_chars, s) for s in specials),
+        }
+
+    def set_unknown_macro_spec(self, macrospec):
+        r"""
+        Set the macro spec to use when encountering a macro that is not in the
+        database.
+        """
+        self.unknown_macro_spec = macrospec
+
+    def set_unknown_environment_spec(self, environmentspec):
+        r"""
+        Set the environment spec to use when encountering a LaTeX environment that
+        is not in the database.
+        """
+        self.unknown_environment_spec = environmentspec
+
+    def set_unknown_specials_spec(self, specialsspec):
+        r"""
+        Set the latex specials spec to use when encountering a LaTeX environment
+        that is not in the database.
+        """
+        self.unknown_specials_spec = specialsspec
+
+    def categories(self):
+        r"""
+        Return a list of valid category names that are registered in the current
+        database context.
+        """
+        return list(self.category_list)
+
+    def get_macro_spec(self, macroname):
+        r"""
+        Look up a macro specification by macro name.  The macro name is searched for
+        in all categories one by one and the first match is returned.
+
+        Returns a macro spec instance that matches the given `macroname`.  If
+        the macro name was not found, we return the default macro specification
+        set by :py:meth:`set_unknown_macro_spec()` or `None` if no such spec was
+        set.
+        """
+        for cat in self.category_list:
+            # search categories in the given order
+            if macroname in self.d[cat]["macros"]:
+                return self.d[cat]["macros"][macroname]
+        return self.unknown_macro_spec
+
+    def get_environment_spec(self, environmentname):
+        r"""
+        Look up an environment specification by environment name.  The environment
+        name is searched for in all categories one by one and the first match is
+        returned.
+
+        Returns the environment spec.  If the environment name was not found, we
+        return the default environment specification set by
+        :py:meth:`set_unknown_environment_spec()` or `None` if no such spec was
+        set.
+        """
+        for cat in self.category_list:
+            # search categories in the given order
+            if environmentname in self.d[cat]["environments"]:
+                return self.d[cat]["environments"][environmentname]
+        return self.unknown_environment_spec
+
+    def get_specials_spec(self, specials_chars):
+        r"""
+        Look up a "latex specials" specification by character sequence.  The
+        sequence name is searched for in all categories one by one and the first
+        match is returned.
+
+        If you are parsing a chunk of LaTeX code, you should use
+        :py:meth:`test_for_specials()` instead.  Unlike
+        :py:meth:`test_for_specials()`, :py:meth:`get_specials_spec()` returns
+        the first match regardless of matched length.  [Rationale: we only need
+        to worry about matching the longest specials sequence when parsing LaTeX
+        code.  Calling `get_specials_spec()` means one has already parsed the
+        sequence and one is looking up additional specs on it.]
+
+        Returns the specials spec.  If the latex specials was not found, we
+        return the default latex specials specification set by
+        :py:meth:`set_unknown_specials_spec()` or `None` if no such spec was
+        set.
+        """
+        for cat in self.category_list:
+            # search categories in the given order
+            if specials_chars in self.d[cat]["specials"]:
+                return self.d[cat]["specials"][specials_chars]
+        return self.unknown_specials_spec
+
+    def test_for_specials(self, s, pos, parsing_state=None):
+        r"""
+        Test the given position in the string for any LaTeX specials.  The lookup
+        proceeds by searching for in all categories one by one and the first
+        match is returned, except that the longest match accross all categories
+        is returned.  For instance, a match of '``' in a later category will
+        take precedence over a match of '`' in a earlier-searched category.
+
+        Returns a specials spec instance, or `None` if no specials are detected
+        at the position `pos`.
+        """
+        best_match_len = 0
+        best_match_s = None
+        for cat in self.category_list:
+            # search categories in the given order
+            for specials_chars in self.d[cat]["specials"].keys():
+                if len(specials_chars) > best_match_len and s.startswith(
+                    specials_chars, pos
+                ):
+                    best_match_s = self.d[cat]["specials"][specials_chars]
+                    best_match_len = len(specials_chars)
+
+        return best_match_s  # this is None if no match
+
+    def iter_macro_specs(self, categories=None):
+        r"""
+        Yield the macro specs corresponding to all macros in the given categories.
+
+        If `categories` is `None`, then the known macro specs from all
+        categories are provided in one long iterable sequence.  Otherwise,
+        `categories` should be a list or iterable of category names (e.g.,
+        'latex-base') of macro specs to return.
+
+        The macro specs from the different categories specified are concatenated
+        into one long sequence which is yielded spec by spec.
+        """
+
+        if categories is None:
+            categories = self.category_list
+
+        for c in categories:
+            if c not in self.category_list:
+                raise ValueError(
+                    "Invalid latex macro spec db category: {!r} (Expected one of {!r})".format(
+                        c, self.category_list
+                    )
+                )
+            for spec in self.d[c]["macros"].values():
+                yield spec
+
+    def iter_environment_specs(self, categories=None):
+        r"""
+        Yield the environment specs corresponding to all environments in the given
+        categories.
+
+        If `categories` is `None`, then the known environment specs from all
+        categories are provided in one long iterable sequence.  Otherwise,
+        `categories` should be a list or iterable of category names (e.g.,
+        'latex-base') of environment specs to return.
+
+        The environment specs from the different categories specified are
+        concatenated into one long sequence which is yielded spec by spec.
+        """
+
+        if categories is None:
+            categories = self.category_list
+
+        for c in categories:
+            if c not in self.category_list:
+                raise ValueError(
+                    "Invalid latex environment spec db category: {!r} (Expected one of {!r})".format(
+                        c, self.category_list
+                    )
+                )
+            for spec in self.d[c]["environments"].values():
+                yield spec
+
+    def iter_specials_specs(self, categories=None):
+        r"""
+        Yield the specials specs corresponding to all environments in the given
+        categories.
+
+        If `categories` is `None`, then the known specials specs from all
+        categories are provided in one long iterable sequence.  Otherwise,
+        `categories` should be a list or iterable of category names (e.g.,
+        'latex-base') of specials specs to return.
+
+        The specials specs from the different categories specified are
+        concatenated into one long sequence which is yielded spec by spec.
+        """
+
+        if categories is None:
+            categories = self.category_list
+
+        for c in categories:
+            if c not in self.category_list:
+                raise ValueError(
+                    "Invalid latex environment spec db category: {!r} (Expected one of {!r})".format(
+                        c, self.category_list
+                    )
+                )
+            for spec in self.d[c]["specials"].values():
+                yield spec
+
+    def filter_context(self, keep_categories=[], exclude_categories=[], keep_which=[]):
+        r"""
+        Return a new :py:class:`LatexContextDb` instance where we only keep
+        certain categories of macro and environment specifications.
+
+        If `keep_categories` is set to a nonempty list, then the returned
+        context will not contain any definitions that do not correspond to the
+        specified categories.
+
+        If `exclude_categories` is set to a nonempty list, then the returned
+        context will not contain any definitions that correspond to the
+        specified categories.
+
+        It is explicitly fine to have category names in `keep_categories` and
+        `exclude_categories` that don't exist in the present object
+        (cf. :py:meth:`categories()`).
+
+        The argument `keep_which`, if non-empty, specifies which definitions to
+        keep.  It should be a subset of the list ['macros', 'environments',
+        'specials'].
+
+        The returned context will make a copy of the dictionaries that store the
+        macro and environment specifications, but the specification classes (and
+        corresponding argument parsers) might correspond to the same instances.
+        I.e., the returned context is not a full deep copy.
+        """
+
+        new_context = LatexContextDb()
+
+        new_context.unknown_macro_spec = self.unknown_macro_spec
+        new_context.unknown_environment_spec = self.unknown_environment_spec
+        new_context.unknown_specials_spec = self.unknown_specials_spec
+
+        keep_macros = not keep_which or "macros" in keep_which
+        keep_environments = not keep_which or "environments" in keep_which
+        keep_specials = not keep_which or "specials" in keep_which
+
+        for cat in self.category_list:
+            if keep_categories and cat not in keep_categories:
+                continue
+            if exclude_categories and cat in exclude_categories:
+                continue
+
+            # include this category
+            new_context.add_context_category(
+                cat,
+                macros=self.d[cat]["macros"].values() if keep_macros else [],
+                environments=self.d[cat]["environments"].values()
+                if keep_environments
+                else [],
+                specials=self.d[cat]["specials"].values() if keep_specials else [],
+            )
+
+        return new_context

good_ai/src/sus/external/pylatexenc/macrospec/_argparsers.py

@@ -0,0 +1,497 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2019 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+
+# Internal module. Internal API may move, disappear or otherwise change at any
+# time and without notice.
+
+
+import sys
+
+if sys.version_info.major > 2:
+    # Py3
+    def unicode(s):
+        return s
+
+    _basestring = str
+    _str_from_unicode = lambda x: x
+    _unicode_from_str = lambda x: x
+else:
+    # Py2
+    _basestring = basestring
+    _str_from_unicode = lambda x: unicode(x).encode("utf-8")
+    _unicode_from_str = lambda x: x.decode("utf-8")
+
+
+class ParsedMacroArgs(object):
+    r"""
+    Parsed representation of macro arguments.
+
+    The base class provides a simple way of storing the arguments as a list of
+    parsed nodes.
+
+    This base class can be subclassed to store additional information and
+    provide more advanced APIs to access macro arguments for certain categories
+    of macros.
+
+    Arguments:
+
+      - `argnlist` is a list of latexwalker nodes that represent macro
+        arguments.  If the macro arguments are too complicated to store in a
+        list, leave this as `None`.  (But then code that uses the latexwalker
+        must be aware of your own API to access the macro arguments.)
+
+        The difference between `argnlist` and the legacy `nodeargs` is that all
+        options, regardless of optional or mandatory, are stored in the list
+        `argnlist` with possible `None`\ 's at places where optional arguments
+        were not provided.  Previously, whether a first optional argument was
+        included in `nodeoptarg` or `nodeargs` depended on how the macro
+        specification was given.
+
+      - `argspec` is a string or a list that describes how each corresponding
+        argument in `argnlist` represents.  If the macro arguments are too
+        complicated to store in a list, leave this as `None`.  For standard
+        macros and parsed arguments this is a string with characters '*', '[',
+        '{' describing an optional star argument, an optional
+        square-bracket-delimited argument, and a mandatory argument.
+
+    Attributes:
+
+    .. py:attribute:: argnlist
+
+       The list of latexwalker nodes that was provided to the constructor
+
+    .. py:attribute:: argspec
+
+       Argument type specification provided to the constructor
+
+    .. py:attribute:: legacy_nodeoptarg_nodeargs
+
+       A tuple `(nodeoptarg, nodeargs)` that should be exposed as properties in
+       :py:class:`~pylatexenc.latexwalker.LatexMacroNode` to provide (as best as
+       possible) compatibility with pylatexenc < 2.
+
+       This is either `(<1st optional arg node>, <list of remaining args>)` if
+       the first argument is optional and all remaining args are mandatory; or
+       it is `(None, <list of args>)` for any other argument structure.
+    """
+
+    def __init__(self, argnlist=[], argspec="", **kwargs):
+        super(ParsedMacroArgs, self).__init__(**kwargs)
+
+        self.argnlist = argnlist
+        self.argspec = argspec
+
+        # for LatexMacroNode to provide some kind of compatibility with pylatexenc < 2
+        self.legacy_nodeoptarg_nodeargs = self._get_legacy_attribs(
+            self.argspec, self.argnlist
+        )
+
+    def _get_legacy_attribs(self, argspec, argnlist):
+        nskip = 0
+        while argspec.startswith("*"):
+            argspec = argspec[1:]
+            nskip += 1
+        if argspec[0:1] == "[" and all(x == "{" for x in argspec[1:]):
+            return (argnlist[nskip], argnlist[nskip + 1 :])
+        else:
+            return (None, argnlist)
+
+    def to_json_object(self):
+        r"""
+        Called when we export the node structure to JSON when running latexwalker in
+        command-line.
+
+        Return a representation of the current parsed arguments in an object,
+        typically a dictionary, that can easily be exported to JSON.  The object
+        may contain latex nodes and other parsed-argument objects, as we use a
+        custom JSON encoder that understands these types.
+
+        Subclasses may
+        """
+
+        return dict(
+            argspec=self.argspec,
+            argnlist=self.argnlist,
+        )
+
+    def __repr__(self):
+        return "{}(argspec={!r}, argnlist={!r})".format(
+            self.__class__.__name__, self.argspec, self.argnlist
+        )
+
+
+class MacroStandardArgsParser(object):
+    r"""
+    Parses the arguments to a LaTeX macro.
+
+    This class parses a simple macro argument specification with a specified
+    arrangement of optional and mandatory arguments.
+
+    This class also serves as base class for more advanced argument parsers
+    (e.g. for a ``\verb+...+`` macro argument parser).  In such cases,
+    subclasses should attempt to provide the most suitable `argspec` (and
+    `argnlist` for the corresponding :py:class:`ParsedMacroArgs`) for their use,
+    if appropriate, or set them to `None`.
+
+    Arguments:
+
+      - `argspec`: must be a string in which each character corresponds to an
+        argument.  The character '{' represents a mandatory argument (single
+        token or LaTeX group) and the character '[' denotes an optional argument
+        delimited by braces.  The character '\*' denotes a possible star char at
+        that position in the argument list, a corresponding
+        ``latexwalker.LatexCharsNode('*')`` (or `None` if no star) will be
+        inserted in the argument node list.  For instance, the string '\*{[[{'
+        would be suitable to specify the signature of the '\\newcommand' macro.
+
+        Currently, the argspec string may only contain the characters '\*', '{'
+        and '['.
+
+        The `argspec` may also be `None`, which is the same as specifying an
+        empty string.
+
+      - `optional_arg_no_space`: If set to `True`, then an optional argument
+        cannot have any whitespace between the preceeding tokens and the '['
+        character.  Set this to `True` in cases such as for ``\\`` in AMS-math
+        environments, where AMS apparently introduced a patch to prevent a
+        bracket on a new line after ``\\`` from being interpreted as the
+        optional argument to ``\\``.
+
+      - `args_math_mode`: Either `None`, or a list of the same length as
+        `argspec`.  If a list is given, then each item must be `True`, `False`,
+        or `None`.  The corresponding argument (cf. `argspec`) is then
+        respectively parsed in math mode (`True`), in text mode (`False`), or
+        with the mode unchanged (`None`).  If `args_math_mode` is `None`, then
+        all arguments are parsed in the same mode as the current mode.
+
+      - additional unrecognized keyword arguments are passed on to superclasses
+        in case of multiple inheritance
+
+    Attributes:
+
+    .. py:attribute:: argspec
+
+       Argument type specification provided to the constructor.
+
+    .. py:attribute:: optional_arg_no_space
+
+       See the corresponding constructor argument.
+
+    .. py:attribute:: args_math_mode
+
+       See the corresponding constructor argument.
+    """
+
+    def __init__(
+        self, argspec=None, optional_arg_no_space=False, args_math_mode=None, **kwargs
+    ):
+        super(MacroStandardArgsParser, self).__init__(**kwargs)
+        self.argspec = argspec if argspec else ""
+        self.optional_arg_no_space = optional_arg_no_space
+        self.args_math_mode = args_math_mode
+        # catch bugs, make sure that argspec is a string with only accepted chars
+        if not isinstance(self.argspec, _basestring) or not all(
+            x in "*[{" for x in self.argspec
+        ):
+            raise TypeError(
+                "argspec must be a string containing chars '*', '[', '{{' only: {!r}".format(
+                    self.argspec
+                )
+            )
+        # non-documented attribute that makes us ignore any leading '*'.  We use
+        # this to emulate pylatexenc 1.x behavior when using the MacrosDef()
+        # function explicitly
+        self._like_pylatexenc1x_ignore_leading_star = False
+
+    def parse_args(self, w, pos, parsing_state=None):
+        r"""
+        Parse the arguments encountered at position `pos` in the
+        :py:class:`~pylatexenc.latexwalker.LatexWalker` instance `w`.
+
+        You may override this function to provide custom parsing of complicated
+        macro arguments (say, ``\verb+...+``).  The method will be called by
+        keyword arguments, so the argument names should not be altered.
+
+        The argument `w` is the :py:class:`pylatexenc.latexwalker.LatexWalker`
+        object that is currently parsing LaTeX code.  You can call methods like
+        `w.get_goken()`, `w.get_latex_expression()` etc., to parse and read
+        arguments.
+
+        The argument `parsing_state` is the current parsing state in the
+        :py:class:`~pylatexenc.latexwalker.LatexWalker` (e.g., are we currently
+        in math mode?).  See doc for
+        :py:class:`~pylatexenc.latexwalker.ParsingState`.
+
+        This function should return a tuple `(argd, pos, len)` where:
+
+        - `argd` is a :py:class:`ParsedMacroArgs` instance, or an instance of a
+          subclass of :py:class:`ParsedMacroArgs`.  The base `parse_args()`
+          provided here returns a :py:class:`ParsedMacroArgs` instance.
+
+        - `pos` is the position of the first parsed content.  It should be the
+          same as the `pos` argument, except if there is whitespace at that
+          position in which case the returned `pos` would have to be the
+          position where the argument contents start.
+
+        - `len` is the length of the parsed expression.  You will probably want
+          to continue parsing stuff at the index `pos+len` in the string.
+        """
+
+        from .. import latexwalker
+
+        if parsing_state is None:
+            parsing_state = w.make_parsing_state()
+
+        argnlist = []
+
+        if self.args_math_mode is not None and len(self.args_math_mode) != len(
+            self.argspec
+        ):
+            raise ValueError(
+                "Invalid args_math_mode={!r} for argspec={!r}!".format(
+                    self.args_math_mode, self.argspec
+                )
+            )
+
+        def get_inner_parsing_state(j):
+            if self.args_math_mode is None:
+                return parsing_state
+            amm = self.args_math_mode[j]
+            if amm is None or amm == parsing_state.in_math_mode:
+                return parsing_state
+            if amm == True:
+                return parsing_state.sub_context(in_math_mode=True)
+            return parsing_state.sub_context(in_math_mode=False)
+
+        p = pos
+
+        if self._like_pylatexenc1x_ignore_leading_star:
+            # ignore any leading '*' character
+            tok = w.get_token(p)
+            if tok.tok == "char" and tok.arg == "*":
+                p = tok.pos + tok.len
+
+        for j, argt in enumerate(self.argspec):
+            if argt == "{":
+                (node, np, nl) = w.get_latex_expression(
+                    p, strict_braces=False, parsing_state=get_inner_parsing_state(j)
+                )
+                p = np + nl
+                argnlist.append(node)
+
+            elif argt == "[":
+
+                if self.optional_arg_no_space and p < len(w.s) and w.s[p].isspace():
+                    # don't try to read optional arg, we don't allow space
+                    argnlist.append(None)
+                    continue
+
+                optarginfotuple = w.get_latex_maybe_optional_arg(
+                    p, parsing_state=get_inner_parsing_state(j)
+                )
+                if optarginfotuple is None:
+                    argnlist.append(None)
+                    continue
+                (node, np, nl) = optarginfotuple
+                p = np + nl
+                argnlist.append(node)
+
+            elif argt == "*":
+                # possible star.
+                tok = w.get_token(p)
+                if tok.tok == "char" and tok.arg.startswith("*"):
+                    # has star
+                    argnlist.append(
+                        w.make_node(
+                            latexwalker.LatexCharsNode,
+                            parsing_state=get_inner_parsing_state(j),
+                            chars="*",
+                            pos=tok.pos,
+                            len=1,
+                        )
+                    )
+                    p = tok.pos + 1
+                else:
+                    argnlist.append(None)
+
+            else:
+                raise LatexWalkerError(
+                    "Unknown macro argument kind for macro: {!r}".format(argt)
+                )
+
+        parsed = ParsedMacroArgs(
+            argspec=self.argspec,
+            argnlist=argnlist,
+        )
+
+        return (parsed, pos, p - pos)
+
+    def __repr__(self):
+        return (
+            "{}(argspec={!r}, optional_arg_no_space={!r}, args_math_mode={!r})".format(
+                self.__class__.__name__,
+                self.argspec,
+                self.optional_arg_no_space,
+                self.args_math_mode,
+            )
+        )
+
+
+class ParsedVerbatimArgs(ParsedMacroArgs):
+    r"""
+    Parsed representation of arguments to LaTeX verbatim constructs, such as
+    ``\begin{verbatim}...\end{verbatim}`` or ``\verb|...|``.
+
+    Instances of `ParsedVerbatimArgs` are returned by the args parser
+    :py:class:`VerbatimArgsParser`.
+
+    Arguments:
+
+      - `verbatim_chars_node` --- a properly initialized
+        :py:class:`pylatexenc.latexwalker.LatexCharsNode` that stores the
+        verbatim text provided.  It is used to initialize the base class
+        :py:class:`ParsedMacroArgs` to expose a single mandatory argument with
+        the given verbatim text.  The `verbatim_text` attribute is initialized
+        from this node, too.
+
+      - `verbatim_delimiters` --- a 2-item tuple of characters used to delimit
+        the verbatim arguemnt (in case of a ``\verb+...+`` macro) or `None`.
+
+    Attributes:
+
+    .. py:attribute:: verbatim_text
+
+       The verbatim text that was provided
+
+    .. py:attribute:: verbatim_delimiters
+
+       If the verbatim text was specified as an argument to ``\verb$...$``, then
+       this is set to a 2-item tuple that specifies the begin and end
+       delimiters.  Otherwise, the attribute is `None`.
+    """
+
+    def __init__(self, verbatim_chars_node, verbatim_delimiters=None, **kwargs):
+
+        # provide argspec/argnlist to the parent class so that any code that is
+        # not "verbatim environment-aware" sees this simply as the argument to
+        # an empty verbatim environment
+        super(ParsedVerbatimArgs, self).__init__(
+            argspec="{", argnlist=[verbatim_chars_node], **kwargs
+        )
+
+        self.verbatim_text = verbatim_chars_node.chars
+        self.verbatim_delimiters = verbatim_delimiters
+
+    def __repr__(self):
+        return "{}(verbatim_text={!r}, verbatim_delimiters={!r})".format(
+            self.__class__.__name__, self.verbatim_text, self.verbatim_delimiters
+        )
+
+
+class VerbatimArgsParser(MacroStandardArgsParser):
+    r"""
+    Parses the arguments to various LaTeX "verbatim" constructs such as
+    ``\begin{verbatim}...\end{verbatim}`` environment or ``\verb+...+``.
+
+    This class also serves to illustrate how to write custom parsers for
+    complicated macro arguments.  See also :py:class:`MacroStandardArgsParser`.
+
+    Arguments:
+
+    .. py:attribute:: verbatim_arg_type
+
+      One of 'verbatim-environment' or 'verb-macro'.
+    """
+
+    def __init__(self, verbatim_arg_type, **kwargs):
+        super(VerbatimArgsParser, self).__init__(argspec="{", **kwargs)
+        self.verbatim_arg_type = verbatim_arg_type
+
+    def parse_args(self, w, pos, parsing_state=None):
+
+        from .. import latexwalker
+
+        if self.verbatim_arg_type == "verbatim-environment":
+            # simply scan the string until we find '\end{verbatim}'.  That's
+            # exactly how LaTeX processes it.
+            endverbpos = w.s.find(r"\end{verbatim}", pos)
+            if endverbpos == -1:
+                raise latexwalker.LatexWalkerParseError(
+                    s=w.s, pos=pos, msg=r"Cannot find matching \end{verbatim}"
+                )
+            # do NOT include the "\end{verbatim}", latexwalker will expect to
+            # see it:
+            len_ = endverbpos - pos
+
+            argd = ParsedVerbatimArgs(
+                verbatim_chars_node=w.make_node(
+                    latexwalker.LatexCharsNode,
+                    parsing_state=parsing_state,
+                    chars=w.s[pos : pos + len_],
+                    pos=pos,
+                    len=len_,
+                )
+            )
+            return (argd, pos, len_)
+
+        if self.verbatim_arg_type == "verb-macro":
+            # read the next nonwhitespace char. This is the delimiter of the
+            # argument
+            while w.s[pos].isspace():
+                pos += 1
+                if pos >= len(w.s):
+                    raise latexwalker.LatexWalkerParseError(
+                        s=w.s, pos=pos, msg=r"Missing argument to \verb command"
+                    )
+            verbdelimchar = w.s[pos]
+            beginpos = pos + 1
+            endpos = w.s.find(verbdelimchar, beginpos)
+            if endpos == -1:
+                raise latexwalker.LatexWalkerParseError(
+                    s=w.s,
+                    pos=pos,
+                    msg=r"End of stream reached while reading argument to \verb command",
+                )
+
+            verbarg = w.s[beginpos:endpos]
+
+            argd = ParsedVerbatimArgs(
+                verbatim_chars_node=w.make_node(
+                    latexwalker.LatexCharsNode,
+                    parsing_state=parsing_state,
+                    chars=verbarg,
+                    pos=beginpos,
+                    len=endpos - beginpos,
+                ),
+                verbatim_delimiters=(verbdelimchar, verbdelimchar),
+            )
+
+            return (argd, pos, endpos + 1 - pos)  # include delimiters in pos/len
+
+    def __repr__(self):
+        return "{}(verbatim_arg_type={!r})".format(
+            self.__class__.__name__, self.verbatim_arg_type
+        )

good_ai/src/sus/external/pylatexenc/version.py

@@ -0,0 +1,58 @@
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2021 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+
+#
+# Self-note: Checklist
+#
+#   1) First some checks:
+#
+#       - Set below in this file ' version_str = "X.Xb" ' (beta version for next
+#         release) for the following tests.
+#
+#       - tests pass: https://travis-ci.org/github/phfaist/pylatexenc
+#
+#       - LGTM looks good: https://lgtm.com/projects/g/phfaist/pylatexenc/
+#
+#       - python package creation works: (python setup.py sdist, pip install
+#         dist/pylatexenc-xxx.tar.gz)
+#
+#   2) update change log (doc/changes.rst)
+#
+#   3) bump version number here
+#
+#   4) git commit any remaining changes
+#
+#   5) " git tag vX.X -am '<message>' "
+#
+#   6) " git push && git push --tags "
+#
+#   7) on github.com, fill in release details with a summary of changes etc.
+#
+#   8) create the source package for PyPI (" python3 setup.py sdist ")
+#
+#   8) upload package to PyPI (twine upload dist/pylatexenc-X.X.tar.gz -r realpypi)
+#
+
+version_str = "2.10"

good_ai/src/sus/get_sentences.py

@@ -0,0 +1,18 @@
+from typing import List
+
+from .data import sentence_ending_punctuations
+from .nlp import nlp
+
+
+def get_sentences(text: str, ignore_partial: bool = False) -> List[str]:
+    doc = nlp(text)
+    possible_sentences = [s.text for s in doc.sents]
+
+    if ignore_partial:
+        possible_sentences = [
+            s
+            for s in possible_sentences
+            if s[0].isupper() and s[-1] in sentence_ending_punctuations
+        ]
+
+    return possible_sentences

good_ai/src/sus/language/__init__.py

@@ -0,0 +1,3 @@
+from .english_name_of_language import english_name_of_language
+from .is_english import is_english
+from .predict_language import predict_language

good_ai/src/sus/language/english_name_of_language.py

@@ -0,0 +1,10 @@
+from typing import Optional
+
+from langcodes import Language
+
+
+def english_name_of_language(language_code: Optional[str]) -> str:
+    if not language_code:
+        language_code = "und"
+
+    return Language.get(language_code).display_name()

good_ai/src/sus/language/is_english.py

@@ -0,0 +1,11 @@
+from typing import Optional
+
+from langcodes import standardize_tag, tag_distance
+
+
+def is_english(language_code: Optional[str]) -> bool:
+    if not language_code:
+        language_code = "und"
+
+    language_code = standardize_tag(language_code)
+    return tag_distance(language_code, "en") < 15

good_ai/src/sus/language/predict_language.py

@@ -0,0 +1,16 @@
+from typing import Optional
+
+from langcodes import Language
+from langdetect import LangDetectException, detect
+
+
+def predict_language(text: Optional[str]) -> str:
+    if not text:
+        return Language.make().to_tag()
+
+    try:
+        language_code = detect(text)
+    except LangDetectException:
+        return Language.make().to_tag()
+
+    return Language.get(language_code).to_tag()

good_ai/src/sus/lemmatize_text.py

@@ -0,0 +1,21 @@
+from typing import List
+
+from .lemmatize_token import lemmatize_token
+from .nlp import nlp
+
+
+def lemmatize_text(
+    text: str,
+    add_negation: bool = False,
+    add_part_of_speech: bool = False,
+) -> List[str]:
+    doc = nlp(text)
+
+    return [
+        lemmatize_token(
+            t,
+            add_negation=add_negation,
+            add_part_of_speech=add_part_of_speech,
+        )
+        for t in doc
+    ]

good_ai/src/sus/lemmatize_token.py

@@ -0,0 +1,20 @@
+from spacy.tokens import Token
+
+from .data import american_spellings
+
+
+def lemmatize_token(
+    token: Token,
+    add_negation: bool = False,
+    add_part_of_speech: bool = False,
+) -> str:
+    lemma = token.lemma_.lower()
+
+    lemma = american_spellings.get(lemma, lemma)
+
+    if add_part_of_speech:
+        lemma = f"{lemma}_{token.pos_}"
+    if add_negation and token._.negex:
+        lemma = f"NOT_{lemma}"
+
+    return lemma

good_ai/src/sus/match_names/__init__.py

@@ -0,0 +1 @@
+from .match_names import get_name_match_score, get_name_parts, match_names

good_ai/src/sus/match_names/config.py

@@ -0,0 +1,5 @@
+last_name_weight = 1.5
+first_name_weight = 0.9
+infixes_weight = 0.5
+initials_weight = 0.6
+match_threshold = 1.51

good_ai/src/sus/match_names/match_names.py

@@ -0,0 +1,162 @@
+import functools
+import re
+from typing import List, Optional, Tuple
+
+from ..clean import clean
+from .config import (
+    first_name_weight,
+    infixes_weight,
+    initials_weight,
+    last_name_weight,
+    match_threshold,
+)
+from .name_parts import NameParts
+
+
+def match_names(n1: Optional[str], n2: Optional[str]) -> bool:
+    score = get_name_match_score(n1, n2)
+    return score > match_threshold
+
+
+def get_name_match_score(n1: Optional[str], n2: Optional[str]) -> float:
+    if not n1 or not n2:
+        return 0
+
+    p1 = get_name_parts(n1)
+    p2 = get_name_parts(n2)
+
+    for f1 in p1.first_names:
+        for f2 in p2.first_names:
+            if f1[0] == f2[0]:
+                p1.initials = [i for i in p1.initials if i != f1[0]]
+                p2.initials = [i for i in p2.initials if i != f1[0]]
+
+    last_name_score = last_name_weight * _match_percentage(p1.last_names, p2.last_names)
+    first_name_score = first_name_weight * _match_percentage(
+        p1.first_names, p2.first_names
+    )
+    initials_score = initials_weight * _match_percentage(p1.initials, p2.initials)
+    infixes_score = min(0, infixes_weight * _match_percentage(p1.infixes, p2.infixes))
+
+    return last_name_score + first_name_score + initials_score + infixes_score
+
+
+def get_name_parts(name: str) -> NameParts:
+    result = _get_name_parts(name)
+    return result.copy()
+
+
+@functools.lru_cache(maxsize=None)
+def _get_name_parts(name: str) -> NameParts:
+    name = _clean_name(name)
+
+    first_names, name = _get_parenthesized_first_names(name)
+    initials, name = _get_initials(name)
+
+    last_names = []
+    infixes, name = _get_infixes(name)
+    if "," in name:
+        [last_name, *rest] = name.split(",")
+        rest_combined = " ".join(rest)
+        last_names.extend(last_name.split(" "))
+        first_names.extend(rest_combined.split(" "))
+    else:
+        parts = name.strip().split(" ")
+        if parts:
+            last_names.append(parts[-1])
+            parts.pop(-1)
+            first_names.extend(parts)
+        else:
+            last_names.append(name)
+
+    first_names = [f for f in first_names if f]
+    for f in first_names:
+        if all(f[0] != i for i in initials):
+            initials.append(f[0])
+
+    return NameParts(
+        first_names=first_names,
+        initials=initials,
+        infixes=infixes,
+        last_names=[
+            name
+            for last_name in last_names
+            for name in (last_name.split("-") if "-" in last_name else [last_name])
+            if name
+        ],
+    )
+
+
+def _clean_name(name: str) -> str:
+    name = clean(name, convert_to_ascii=True)
+
+    name = re.sub(
+        r"""
+            (MD|PhD|Ir|dr|Dr|DR|MSc|MA|BSc|BA|Prof)
+            [,.]?
+            [ ]*
+        """,
+        "",
+        name,
+        flags=re.VERBOSE,
+    )
+
+    subs_made = 1
+    while subs_made:
+        name, subs_made = re.subn(r"([A-Z]\.)\s+([A-Z])\b", r"\g<1>\g<2>", name)
+
+    name = name.replace(r"\s+-\s+", "-")
+    name = " ".join(p for p in name.split(" ") if p)
+    name = re.sub(r"^([^,.]+) ([A-Z])$", r"\g<1>, \g<2>.", name)
+    return name
+
+
+def _get_parenthesized_first_names(name: str) -> Tuple[List[str], str]:
+    groups = re.search(r"\(([a-zA-Z ]+)\)", name)
+    if groups:
+        return (
+            groups.group(1).split(" "),
+            _remove_between_indices(name, groups.start(), groups.end()),
+        )
+
+    return [], name
+
+
+def _get_initials(n: str) -> Tuple[List[str], str]:
+    initials = []
+
+    groups = re.search(r"\b(?P<abr1>([A-Z]\.)*)(?P<abr2>[A-Z])(\.|$)", n)
+    if groups:
+        first_name_abbreviations = [
+            *groups.group("abr1").split("."),
+            groups.group("abr2"),
+        ]
+        first_name_abbreviations = [f for f in first_name_abbreviations if f != ""]
+        initials.extend(first_name_abbreviations)
+        n = _remove_between_indices(n, groups.start(), groups.end())
+
+    return initials, n
+
+
+def _get_infixes(n: str) -> Tuple[List[str], str]:
+    infixes = ["de", "van", "der", "den", "te", "ter", "ten"]
+
+    parts = n.split(" ")
+    return [p for p in parts if p.lower() in infixes], " ".join(
+        p for p in parts if p.lower() not in infixes
+    )
+
+
+def _remove_between_indices(s: str, start: int, end: int) -> str:
+    return s[:start] + s[end:]
+
+
+def _match_percentage(l1: List[str], l2: List[str]) -> float:
+    if not l1 or not l2:
+        return 0
+
+    s1 = set(l1)
+    s2 = set(l2)
+    common_q = 2 * len(s1 & s2) / (len(s1) + len(s2))
+    different_q = min(len(s1 - s2) / len(s1), len(s2 - s1) / len(s2))
+    return common_q - different_q

good_ai/src/sus/match_names/name_parts.py

@@ -0,0 +1,10 @@
+from typing import List
+
+from pydantic import BaseModel
+
+
+class NameParts(BaseModel):
+    first_names: List[str]
+    initials: List[str]
+    infixes: List[str]
+    last_names: List[str]

good_ai/src/sus/nlp.py

@@ -0,0 +1,21 @@
+try:
+    import en_core_web_lg
+except ImportError:
+    import subprocess
+
+    print("Spacy model en_core_web_lg not found locally, downloading...")
+
+    subprocess.call(
+        [
+            "pip",
+            "install",
+            "https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.2.0/en_core_web_lg-3.2.0-py3-none-any.whl",
+        ]
+    )
+    import en_core_web_lg
+
+from .external.negspacy import negation  # noqa: F401 it's important to import this
+
+nlp = en_core_web_lg.load()
+
+nlp.add_pipe("negex")

good_ai/src/sus/parallel_map.py

@@ -0,0 +1,34 @@
+from math import ceil
+from typing import Any, Callable, Iterable, List, Optional
+
+import multiprocess as mp
+import psutil
+from tqdm.auto import tqdm
+
+
+def parallel_map(
+    function: Callable[[Any], Any],
+    values: Iterable[Any],
+    chunk_size: Optional[int] = None,
+    concurrency: int = psutil.cpu_count(),
+    disable_progress: bool = False,
+) -> List[Any]:
+    assert concurrency > 0
+    assert chunk_size is None or chunk_size > 0
+
+    values = list(values)
+
+    if not chunk_size:
+        chunk_size = max(1, ceil(len(values) / concurrency / 10))
+
+    if concurrency == 1:
+        iterable = values if disable_progress else tqdm(values)
+        return [function(v) for v in iterable]
+
+    with mp.Pool(processes=concurrency) as pool:
+        if disable_progress:
+            return pool.map(function, values, chunksize=chunk_size)
+
+        return list(
+            tqdm(pool.imap(function, values, chunksize=chunk_size), total=len(values))
+        )

good_ai/src/sus/publication_tei/__init__.py

@@ -0,0 +1,2 @@
+from .models import *
+from .publication_tei import PublicationTEI

good_ai/src/sus/publication_tei/models/__init__.py

@@ -0,0 +1,5 @@
+from .affiliation import Affiliation
+from .author import Author
+from .element import Element, Meta, MetaType, Paragraph, Title
+from .publication_metadata import PublicationMetadata
+from .text import Text

good_ai/src/sus/publication_tei/models/affiliation.py

@@ -0,0 +1,11 @@
+from typing import Optional, Tuple
+
+from pydantic import BaseModel
+
+
+class Affiliation(BaseModel):
+    institutions: Tuple[str, ...]
+    departments: Tuple[str, ...]
+    laboratories: Tuple[str, ...]
+    country: Optional[str]
+    settlement: Optional[str]

good_ai/src/sus/publication_tei/models/author.py

@@ -0,0 +1,14 @@
+from typing import List, Optional
+
+from pydantic import BaseModel
+
+from .affiliation import Affiliation
+
+
+class Author(BaseModel):
+    name: Optional[str]
+    orcid: Optional[str]
+    email: Optional[str]
+    corresponding: bool
+    affiliations: List[Affiliation]
+    coordinates: Optional[str]

good_ai/src/sus/publication_tei/models/element.py

@@ -0,0 +1,30 @@
+from typing import List, Literal, Union
+
+from pydantic import BaseModel
+
+from .text import Text
+
+
+class Title(BaseModel):
+    text: Text
+
+
+class Paragraph(BaseModel):
+    sentences: List[Text]
+
+
+MetaType = Literal[
+    "abstract_start",
+    "abstract_end",
+    "acknowledgements_start",
+    "acknowledgements_end",
+    "annex_start",
+    "annex_end",
+]
+
+
+class Meta(BaseModel):
+    meta_type: MetaType
+
+
+Element = Union[Title, Paragraph, Meta]

good_ai/src/sus/publication_tei/models/publication_metadata.py

@@ -0,0 +1,14 @@
+from typing import List, Optional
+
+from pydantic import BaseModel
+
+
+class PublicationMetadata(BaseModel):
+    language: Optional[str]
+    title: Optional[str]
+    publisher: Optional[str]
+    doi: Optional[str]
+    md5: Optional[str]
+    publication_date: Optional[str]
+    keywords: List[str]
+    reference_count: int

good_ai/src/sus/publication_tei/models/text.py

@@ -0,0 +1,7 @@
+from pydantic import BaseModel
+
+
+class Text(BaseModel):
+    content: str
+    document_order: int
+    coordinates: str

good_ai/src/sus/publication_tei/publication_tei.py

@@ -0,0 +1,176 @@
+from functools import cached_property
+from typing import Any, List, Optional, Union
+
+from bs4 import BeautifulSoup
+from bs4.element import NavigableString, Tag
+from sus.publication_tei.models.element import Paragraph
+
+from ..clean import clean
+from .models import Affiliation, Author, Element, Meta, PublicationMetadata, Text, Title
+
+
+class PublicationTEI:
+    def __init__(self, tei: str):
+        self._document_order_counter = 0
+
+        if tei:
+            cleaned_xml = clean(tei, ignore_xml=True)
+            self.soup = BeautifulSoup(cleaned_xml, "xml")
+        else:
+            self.soup = BeautifulSoup()
+
+    @cached_property
+    def publication_metadata(self) -> PublicationMetadata:
+        publication_date = (
+            self.soup.publicationStmt.date.get("when")
+            if self.soup.publicationStmt and self.soup.publicationStmt.date
+            else None
+        )
+
+        keywords = (
+            [self._element_to_text(k) for k in self.soup.keywords.find_all("term")]
+            if self.soup.keywords
+            else []
+        )
+
+        return PublicationMetadata(
+            language=self.soup.teiHeader.get("xml:lang")
+            if self.soup.teiHeader
+            else None,
+            title=self._element_to_text(self.soup.title),
+            publisher=self._element_to_text(self.soup.publisher),
+            doi=self._element_to_text(self.soup.find("idno", type="DOI")),
+            md5=self._element_to_text(self.soup.find("idno", type="MD5")),
+            publication_date=publication_date,
+            keywords=keywords,
+            reference_count=self.get_reference_count(),
+        )
+
+    def get_reference_count(self) -> int:
+        references = self.soup.find("div", {"type": "references"})
+
+        if not references:
+            return 0
+
+        return len(references.findAll("biblStruct"))
+
+    @cached_property
+    def authors(self) -> List[Author]:
+        if not self.soup.analytic:
+            return []
+
+        return [
+            self._parse_author(author)
+            for author in self.soup.analytic.find_all("author")
+        ]
+
+    @cached_property
+    def content(self) -> List[Element]:
+        self._document_order_counter = 0
+        return self._get_elements(self.soup)
+
+    @cached_property
+    def sentences(self) -> List[Text]:
+        return [
+            sentence
+            for element in self.content
+            if isinstance(element, Paragraph)
+            for sentence in element.sentences
+        ]
+
+    def _parse_author(self, raw: Tag) -> Author:
+        return Author(
+            name=(
+                clean(" ".join(name.get_text() for name in raw.persName))
+                if raw.persName
+                else None
+            ),
+            orcid=self._element_to_text(raw.find(attrs={"type": "ORCID"})),
+            email=self._element_to_text(raw.email),
+            corresponding=raw.get("role") == "corresp",
+            affiliations=[
+                self._parse_affiliation(aff) for aff in raw.find_all("affiliation")
+            ],
+            coordinates=raw.persName.get("coords") if raw.persName else None,
+        )
+
+    def _parse_affiliation(self, raw: Tag) -> Affiliation:
+        return Affiliation(
+            institutions=[
+                self._element_to_text(v)
+                for v in raw.find_all("orgName", attrs={"type": "institution"})
+            ],
+            departments=[
+                self._element_to_text(v)
+                for v in raw.find_all("orgName", attrs={"type": "department"})
+            ],
+            laboratories=[
+                self._element_to_text(v)
+                for v in raw.find_all("orgName", attrs={"type": "laboratory"})
+            ],
+            country=self._element_to_text(raw.address.country)
+            if raw.address and raw.address.country
+            else None,
+            settlement=self._element_to_text(raw.address.settlement)
+            if raw.address and raw.address.settlement
+            else None,
+        )
+
+    def _get_elements(self, raw: Tag) -> List[Element]:
+        results: List[Element] = []
+
+        for r in raw.find_all(["abstract", "div", "head", "p"]):
+            if r.name == "abstract":
+                results.append(Meta(meta_type="abstract_start"))
+                results.extend(self._get_primitives(r))
+                results.append(Meta(meta_type="abstract_end"))
+            elif r.name == "div" and r.get("type") == "acknowledgement":
+                results.append(Meta(meta_type="acknowledgements_start"))
+                results.extend(self._get_primitives(r))
+                results.append(Meta(meta_type="acknowledgements_end"))
+            elif r.name == "div" and r.get("type") == "annex":
+                results.append(Meta(meta_type="annex_start"))
+                results.extend(self._get_primitives(r))
+                results.append(Meta(meta_type="annex_end"))
+            elif not r.find_parents(["abstract", "div"]):
+                results.extend(self._get_primitives(r))
+
+        return results
+
+    def _get_primitives(self, raw: Tag) -> List[Element]:
+        results: List[Element] = []
+
+        for r in raw.find_all(["head", "p"]):
+            if r.name == "head" and r.get("coords") and r.get_text():
+                results.append(Title(text=self._parse_text(r)))
+            elif r.name == "p" and r.find_all("s"):
+                results.append(
+                    Paragraph(
+                        sentences=[
+                            self._parse_text(sentence)
+                            for sentence in r.find_all("s")
+                            if sentence.get_text()
+                        ]
+                    )
+                )
+
+        return results
+
+    def _element_to_text(
+        self, element: Optional[NavigableString], default: Any = None
+    ) -> Union[str, Any]:
+        return (
+            clean(element.get_text(separator=" ", strip=True)) if element else default
+        )
+
+    def _parse_text(self, raw: Tag) -> Text:
+        return Text(
+            content=clean(raw.get_text()),
+            document_order=self._generate_document_order_id(),
+            coordinates=raw.get("coords"),
+        )
+
+    def _generate_document_order_id(self) -> int:
+        value = self._document_order_counter
+        self._document_order_counter += 1
+        return value

good_ai/src/sus/unique.py

@@ -0,0 +1,16 @@
+from typing import Any, Callable, Iterable, List
+
+
+def unique(
+    values: Iterable[Any], *, key: Callable[[Any], Any] = lambda v: v
+) -> List[Any]:
+    """Only keep first occurrences and maintian order"""
+
+    key_values = {}
+    for v in values:
+        k = key(v)
+        if k not in key_values:
+            # dicts maintin insertion order: https://mail.python.org/pipermail/python-dev/2017-December/151283.html
+            key_values[k] = v
+
+    return list(key_values.values())

good_ai/tests/open_s3/test_human_readable_to_byte.py

@@ -0,0 +1,26 @@
+import unittest
+
+from src.open_s3.helper import human_readable_to_byte
+
+
+class TestHumanReadableToByte(unittest.TestCase):
+    def test_simple_cases(self):
+        self.assertEqual(human_readable_to_byte("1KB"), 1024)
+        self.assertEqual(human_readable_to_byte("2KB"), 2048)
+
+    def test_fractions(self):
+        self.assertEqual(human_readable_to_byte("0.5KB"), 512)
+        self.assertEqual(human_readable_to_byte("20.5KB"), 1024 * 20 + 512)
+
+    def test_formating(self):
+        self.assertEqual(human_readable_to_byte(" 1MB"), 1024 * 1024)
+        self.assertEqual(human_readable_to_byte(" 2  MB"), 1024 * 1024 * 2)
+        self.assertEqual(human_readable_to_byte("    4   MB "), 1024 * 1024 * 4)
+        self.assertEqual(human_readable_to_byte("8MB    "), 1024 * 1024 * 8)
+        self.assertEqual(human_readable_to_byte(" 1.5   MB  "), 1024 * 1024 * 1.5)
+
+    def test_casing(self):
+        self.assertEqual(human_readable_to_byte("0.5GB"), 0.5 * 1024 * 1024 * 1024)
+        self.assertEqual(human_readable_to_byte("0.5gB"), 0.5 * 1024 * 1024 * 1024)
+        self.assertEqual(human_readable_to_byte("0.5Gb"), 0.5 * 1024 * 1024 * 1024)
+        self.assertEqual(human_readable_to_byte("0.5gb"), 0.5 * 1024 * 1024 * 1024)

good_ai/tests/open_s3/test_large_file.py

@@ -0,0 +1,130 @@
+from pathlib import Path
+import unittest
+from unittest.mock import Mock, create_autospec, patch
+
+import botocore.session
+
+PATH = Path(__file__).parent.resolve()
+
+
+from src.open_s3 import LargeFile
+
+credentials = {
+    "aws_region_name": "your_region_like_eu-west-2",
+    "aws_access_key_id": "YOUR_ACCESS_KEY_ID",
+    "aws_secret_access_key": "YOUR_VERY_SECRET_ACCESS_KEY",
+    "large_files_bucket_name": "create_a_bucket_and_put_its_name_here",
+    "other_key": 23,
+    "endpoint_url": "this is optional, for backblaze, use this: https://s3.us-west-002.backblazeb2.com",
+}
+
+
+class TestLargeFile(unittest.TestCase):
+    def test_uninitialized(self):
+        self.assertRaises(ValueError, LargeFile, "test-file")
+
+    def test_bad_file_modes(self):
+        self.assertRaises(ValueError, LargeFile, "test-file", "w", version=3)
+        self.assertRaises(ValueError, LargeFile, "test-file", "wb", version=3)
+        self.assertRaises(ValueError, LargeFile, "test-file", "w+r")
+        self.assertRaises(ValueError, LargeFile, "test-file", "test")
+
+    @patch("botocore.session")
+    def test_initialized_with_dict(self, session):
+        session_mock = Mock()
+        session.get_session = create_autospec(
+            botocore.session.get_session, return_value=session_mock
+        )
+
+        s3 = Mock()
+        s3.list_objects_v2 = Mock(
+            return_value={
+                "Contents": [
+                    {
+                        "Key": "test-file/0",
+                        "Size": 30,
+                    },
+                    {
+                        "Key": "test-file/1",
+                        "Size": 300,
+                    },
+                    {
+                        "Key": "test-file/2",
+                        "Size": 187,
+                    },
+                ]
+            }
+        )
+
+        session_mock.create_client = Mock(return_value=s3)
+
+        LargeFile.configure_credentials(**credentials)
+        lf = LargeFile("test-file")
+        session_mock.set_credentials.assert_called_once_with(
+            access_key=credentials["aws_access_key_id"],
+            secret_key=credentials["aws_secret_access_key"],
+        )
+        session_mock.create_client.assert_called_once_with(
+            "s3",
+            region_name=credentials["aws_region_name"],
+            endpoint_url=credentials["endpoint_url"],
+        )
+
+        s3.list_objects_v2.assert_called_once_with(
+            Bucket=credentials["large_files_bucket_name"], Prefix="test-file"
+        )
+
+        self.assertEqual(lf._version, 2)
+        self.assertEqual(lf._local_name, "test-file-2")
+        self.assertEqual(lf._s3_name, "test-file/2")
+
+    @patch("botocore.session")
+    def test_initialized_with_file(self, session):
+        session_mock = Mock()
+        session.get_session = create_autospec(
+            botocore.session.get_session, return_value=session_mock
+        )
+
+        s3 = Mock()
+        s3.list_objects_v2 = Mock(
+            return_value={
+                "Contents": [
+                    {
+                        "Key": "test-file/0",
+                        "Size": 30,
+                    },
+                    {
+                        "Key": "test-file/1",
+                        "Size": 300,
+                    },
+                    {
+                        "Key": "test-file/2",
+                        "Size": 187,
+                    },
+                ]
+            }
+        )
+
+        session_mock.create_client = Mock(return_value=s3)
+
+        LargeFile.configure_credentials_from_file(PATH / "../../example_secrets.ini")
+        lf = LargeFile("test-file")
+
+        session_mock.set_credentials.assert_called_once_with(
+            access_key=credentials["aws_access_key_id"],
+            secret_key=credentials["aws_secret_access_key"],
+        )
+        session_mock.create_client.assert_called_once_with(
+            "s3",
+            region_name=credentials["aws_region_name"],
+            endpoint_url=credentials["endpoint_url"],
+        )
+
+        assert s3.list_objects_v2.called
+        s3.list_objects_v2.assert_called_once_with(
+            Bucket=credentials["large_files_bucket_name"], Prefix="test-file"
+        )
+
+        self.assertEqual(lf._version, 2)
+        self.assertEqual(lf._local_name, "test-file-2")
+        self.assertEqual(lf._s3_name, "test-file/2")

good_ai/tests/sus/data/bad.tei.xml

@@ -0,0 +1,717 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<TEI xml:space="preserve" xmlns="http://www.tei-c.org/ns/1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.tei-c.org/ns/1.0 https://raw.githubusercontent.com/kermitt2/grobid/master/grobid-home/schemas/xsd/Grobid.xsd" xmlns:xlink="http://www.w3.org/1999/xlink">
+	<teiHeader xml:lang="en">
+		<fileDesc>
+			<publicationStmt>
+				<publisher>BMJ</publisher>
+				<availability status="unknown">
+					<p>Copyright BMJ</p>
+				</availability>
+			</publicationStmt>
+			<sourceDesc>
+				<biblStruct>
+					<analytic>
+						<author role="corresp">
+							<persName coords="1,166.78,184.02,67.25,10.43">
+								<roleName>Ms</roleName>
+								<forename type="first">Jolien</forename>
+								<surname>Pieters</surname>
+							</persName>
+							<email>j.pieters@maastrichtuniversity.nl</email>
+							<idno type="ORCID">0000-0002-9327-3977</idno>
+							<affiliation key="aff0">
+								<note type="raw_affiliation">
+									<label>1</label>
+									Department of Educational Development and Research, Faculty of Health, Medicine and Life Sciences, School of Health Professions Education, Maastricht University, Maastricht, The Netherlands
+								</note>
+								<orgName type="department" key="dep2">Faculty of Health, Medicine and Life Sciences</orgName>
+								<orgName type="department" key="dep3">School of Health Professions Education</orgName>
+								<address>
+									<country key="NL">The Netherlands</country>
+								</address>
+							</affiliation>
+						</author>
+						<author>
+							<persName coords="1,261.76,184.02,94.40,10.43">
+								<forename type="first">Daniëlle</forename>
+								<surname>Verstegen</surname>
+							</persName>
+							<affiliation key="aff0">
+								<note type="raw_affiliation">
+									<label>1</label>
+									Department of Educational Development and Research, Faculty of Health, Medicine and Life Sciences, School of Health Professions Education, Maastricht University, Maastricht, The Netherlands
+								</note>
+								<orgName type="department" key="dep1">Department of Educational Development and Research</orgName>
+								<orgName type="department" key="dep2">Faculty of Health, Medicine and Life Sciences</orgName>
+								<orgName type="department" key="dep3">School of Health Professions Education</orgName>
+								<orgName type="institution">Maastricht University</orgName>
+								<address>
+									<settlement>Maastricht</settlement>
+									<country key="NL">The Netherlands</country>
+								</address>
+							</affiliation>
+						</author>
+						<author>
+							<persName coords="1,368.73,184.02,74.73,10.43">
+								<forename type="first">Diana</forename>
+								<surname>Dolmans</surname>
+							</persName>
+							<affiliation key="aff0">
+								<note type="raw_affiliation">
+									<label>1</label>
+									Department of Educational Development and Research, Faculty of Health, Medicine and Life Sciences, School of Health Professions Education, Maastricht University, Maastricht, The Netherlands
+								</note>
+								<orgName type="department" key="dep1">Department of Educational Development and Research</orgName>
+								<orgName type="department" key="dep2">Faculty of Health, Medicine and Life Sciences</orgName>
+								<orgName type="department" key="dep3">School of Health Professions Education</orgName>
+								<orgName type="institution">Maastricht University</orgName>
+							</affiliation>
+						</author>
+						<author>
+							<persName coords="1,456.53,184.02,59.31,10.43">
+								<forename type="first">Evelien</forename>
+								<surname>Neis</surname>
+							</persName>
+							<affiliation key="aff0">
+								<note type="raw_affiliation">
+									<label>1</label>
+									Department of Educational Development and Research, Faculty of Health, Medicine and Life Sciences, School of Health Professions Education, Maastricht University, Maastricht, The Netherlands
+								</note>
+								<orgName type="department" key="dep1">Department of Educational Development and Research</orgName>
+								<orgName type="department" key="dep2">Faculty of Health, Medicine and Life Sciences</orgName>
+								<orgName type="department" key="dep3">School of Health Professions Education</orgName>
+								<orgName type="institution">Maastricht University</orgName>
+								<address>
+									<settlement>Maastricht</settlement>
+									<country key="NL">The Netherlands</country>
+								</address>
+							</affiliation>
+						</author>
+						<author>
+							<persName coords="1,166.77,197.02,108.77,10.43"></persName>
+							<affiliation key="aff0">
+								<note type="raw_affiliation">
+									<label>1</label>
+									Department of Educational Development and Research, Faculty of Health, Medicine and Life Sciences, School of Health Professions Education, Maastricht University, Maastricht, The Netherlands
+								</note>
+								<orgName type="department" key="dep1">Department of Educational Development and Research</orgName>
+								<orgName type="department" key="dep2">Faculty of Health, Medicine and Life Sciences</orgName>
+								<orgName type="department" key="dep3">School of Health Professions Education</orgName>
+								<orgName type="institution">Maastricht University</orgName>
+								<address>
+									<settlement>Maastricht</settlement>
+									<country key="NL">The Netherlands</country>
+								</address>
+							</affiliation>
+						</author>
+						<author>
+							<affiliation key="aff1">
+								<note type="raw_affiliation">
+									<label>2</label>
+									Centre of Expertise for Palliative Care, Maastricht UMC+, Maastricht, The Netherlands
+								</note>
+								<orgName type="department">Centre of Expertise for Palliative Care</orgName>
+								<orgName type="laboratory">UMC+</orgName>
+								<address>
+									<settlement>Maastricht, Maastricht</settlement>
+									<country key="NL">The Netherlands</country>
+								</address>
+							</affiliation>
+						</author>
+						<author>
+							<affiliation key="aff2">
+								<note type="raw_affiliation">Department of Educational Development and Research, Faculty of Health, Medicine and Life Sciences, Maastricht University, Maastricht 6229 ER, The Netherlands;</note>
+							</affiliation>
+						</author>
+						<title level="a" type="main">Spiritual dimension in palliative medicine: a qualitative study of learning tasks: medical students, teachers, educationalists</title>
+					</analytic>
+				</biblStruct>
+			</sourceDesc>
+		</fileDesc>
+		<encodingDesc></encodingDesc>
+		<profileDesc></profileDesc>
+	</teiHeader>
+	<facsimile>
+		<surface n="1" ulx="0.0" uly="0.0" lrx="595.276" lry="793.701" />
+		<surface n="2" ulx="0.0" uly="0.0" lrx="595.276" lry="793.701" />
+		<surface n="3" ulx="0.0" uly="0.0" lrx="595.276" lry="793.701" />
+		<surface n="4" ulx="0.0" uly="0.0" lrx="595.276" lry="793.701" />
+		<surface n="5" ulx="0.0" uly="0.0" lrx="595.276" lry="793.701" />
+		<surface n="6" ulx="0.0" uly="0.0" lrx="595.276" lry="793.701" />
+		<surface n="7" ulx="0.0" uly="0.0" lrx="595.276" lry="793.701" />
+	</facsimile>
+	<text xml:lang="en">
+		<body>
+			<div xmlns="http://www.tei-c.org/ns/1.0">
+				<head coords="1,360.28,486.04,71.51,9.24">INTRODUCTION</head>
+				<p>
+					<s coords="1,360.28,498.01,178.54,9.36;1,360.28,510.17,178.57,9.36;1,360.28,522.33,145.45,9.36">The need for palliative care is set to grow due to demographic changes, longer disease trajectories and higher comorbidity.</s>
+					<s coords="1,507.95,522.33,30.90,9.36;1,360.28,534.49,178.57,9.36;1,360.28,546.65,178.56,9.36;1,360.28,558.81,178.60,9.36;1,360.28,570.97,178.57,9.36;1,360.28,583.13,104.24,9.36;1,464.27,580.99,3.69,6.21">
+						Central to providing palliative care is the holistic, patient-centred and multidimensional approach, which addresses not only the physical, but also the psychological, social and spiritual dimension.
+						<ref type="bibr" coords="1,464.27,580.99,3.69,6.21" target="#b0">1</ref>
+					</s>
+					<s coords="1,473.03,583.13,65.74,9.36;1,360.27,595.29,178.56,9.36;1,360.27,607.45,178.58,9.36;1,360.27,619.61,27.60,9.36;1,387.67,617.47,10.12,6.21;1,401.99,619.61,136.83,9.36;1,360.28,631.77,178.53,9.36;1,360.28,643.93,146.67,9.36;1,506.67,641.79,3.69,6.21">
+						Providing palliative care is increasingly recognised as a universal responsibility of healthcare professionals
+						<ref type="bibr" coords="1,387.67,617.47,10.12,6.21">2 3</ref>
+						and all doctors will see patients with progressive life-limiting conditions at some point during their careers.
+						<ref type="bibr" coords="1,506.67,641.79,3.69,6.21" target="#b3">4</ref>
+					</s>
+					<s coords="1,513.91,643.93,24.87,9.36;1,360.28,656.09,178.57,9.36;1,360.28,668.25,178.57,9.36;1,360.28,680.41,111.83,9.36">Physicians, irrespective of specialism, should be both competent and confident in caring for the palliative care patient.</s>
+					<s coords="1,476.55,680.41,62.29,9.36;1,360.28,692.57,178.57,9.36;1,360.28,704.73,178.50,9.36;1,360.28,716.89,178.58,9.36;1,360.28,729.05,45.08,9.36;1,405.15,726.91,3.69,6.21">
+						Taking care of palliative care patients is typically associated with powerful and highly emotional situations affecting junior doctor's emotional well-being.
+						<ref type="bibr" coords="1,405.15,726.91,3.69,6.21" target="#b4">5</ref>
+					</s>
+					<s coords="1,413.24,729.05,125.59,9.36;2,60.50,24.52,85.87,9.48;2,56.50,45.11,232.01,9.36;2,56.50,57.11,155.00,9.36;2,211.23,54.96,3.69,6.21">
+						It is therefore important that Original research junior doctors develop the ability to guide palliative care patients during their medical training.
+						<ref type="bibr" coords="2,211.23,54.96,3.69,6.21" target="#b5">6</ref>
+					</s>
+					<s coords="2,65.50,69.11,222.87,9.36;2,56.50,81.11,231.89,9.36;2,56.50,93.11,101.07,9.36;2,157.56,90.96,3.69,6.21;2,165.88,93.11,122.50,9.36;2,56.50,105.11,231.88,9.36">
+						lthough there is a growing international movement to embed palliative care education in the undergraduate medical curricula,
+						<ref type="bibr" coords="2,157.56,90.96,3.69,6.21" target="#b4">5</ref>
+						this topic is not adequately addressed within all European medical universities.
+					</s>
+					<s coords="2,56.50,117.11,231.88,9.36;2,56.50,129.11,184.10,9.36;2,240.59,126.96,8.94,6.21">
+						Several studies demonstrate that medical students do not receive sufficient education in this area.
+						<ref type="bibr" coords="2,240.59,126.96,8.94,6.21">7 8</ref>
+					</s>
+					<s coords="2,251.89,129.11,36.49,9.36;2,56.50,141.11,117.08,9.36;2,173.58,138.96,14.37,6.21;2,192.97,141.11,95.43,9.36;2,56.50,153.11,231.90,9.36;2,56.50,165.11,148.73,9.36;2,205.23,162.96,7.37,6.21">
+						Students do not feel well prepared
+						<ref type="bibr" coords="2,173.58,138.96,3.59,6.21" target="#b8">[9]</ref>
+						<ref type="bibr" coords="2,177.18,138.96,3.59,6.21" target="#b9">[10]</ref>
+						<ref type="bibr" coords="2,177.18,138.96,3.59,6.21" target="#b10">[11]</ref>
+						<ref type="bibr" coords="2,180.77,138.96,7.19,6.21" target="#b11">[12]</ref>
+						and feel especially ill -prepared to raise and discuss the psychological, social and spiritual dimensions of care.
+						<ref type="bibr" coords="2,205.23,162.96,7.37,6.21" target="#b12">13</ref>
+					</s>
+					<s coords="2,217.75,165.11,70.64,9.36;2,56.50,177.11,231.90,9.36;2,56.50,189.11,224.52,9.36;2,281.01,186.96,7.37,6.21">
+						Their education primarily focuses on one dimension-the physicalwhile allowing the others to fall by the wayside.
+						<ref type="bibr" coords="2,281.01,186.96,7.37,6.21" target="#b13">14</ref>
+					</s>
+					<s coords="2,56.50,201.11,231.90,9.36;2,56.50,213.11,231.90,9.36;2,56.50,225.11,67.49,9.36;2,123.99,222.96,12.78,6.21">
+						tudents also report that self-care and reflection in the context of palliative care do not get much attention in their education.
+						<ref type="bibr" coords="2,123.99,222.96,12.78,6.21">9 13</ref>
+					</s>
+					<s coords="2,139.37,225.11,149.01,9.36;2,56.50,237.11,231.88,9.36;2,56.50,249.11,146.92,9.36;2,203.41,246.96,28.09,6.21">
+						In the Netherlands, the undergraduate medical education assigns only limited attention on palliative and end-of-life care.
+						<ref type="bibr" coords="2,203.41,246.96,28.09,6.21">13 15-17</ref>
+					</s>
+					<s coords="2,235.51,249.11,52.87,9.36;2,56.50,261.11,231.88,9.36;2,56.50,273.11,231.90,9.36;2,56.50,285.11,172.10,9.36;2,228.59,282.96,7.37,6.21">
+						This despite that the national competency framework states that the doctor should promote people's health and related quality of life, also in the palliative phase.
+						<ref type="bibr" coords="2,228.59,282.96,7.37,6.21" target="#b17">18</ref>
+					</s>
+					<s coords="2,237.86,285.11,50.53,9.36;2,56.50,297.11,231.89,9.36;2,56.50,309.11,231.89,9.36;2,56.50,321.11,168.01,9.36;2,224.50,318.96,7.37,6.21">
+						The competences that Dutch medical students need to acquire to provide good-quality palliative care have recently been set out in an educational framework.
+						<ref type="bibr" coords="2,224.50,318.96,7.37,6.21" target="#b18">19</ref>
+					</s>
+					<s coords="2,236.71,321.11,51.68,9.36;2,56.50,333.11,231.88,9.36;2,56.50,345.11,231.90,9.36;2,56.50,357.11,231.89,9.36;2,56.50,369.11,121.76,9.36">This framework specifies among others that the medical students should be able to talk to the patient and family about the incurable illness, prognosis and death, and discuss the four dimensions of care.</s>
+					<s coords="2,181.41,369.11,106.99,9.36;2,56.50,381.11,231.88,9.36;2,56.50,393.11,231.89,9.36;2,56.50,405.11,117.74,9.36">They should also be able to take care of their own well-being and reflect on their own spiritual needs, alongside their perceptions about life, death and dying.</s>
+				</p>
+				<p>
+					<s coords="2,65.50,417.11,222.89,9.36;2,56.50,429.11,231.91,9.36;2,56.50,441.11,182.38,9.36">To bridge the gap between what students should learn and actually learn about spiritual care, we developed a coherent set of eight learning tasks.</s>
+					<s coords="2,241.57,441.11,46.81,9.36;2,56.50,453.11,184.25,9.36">Addressing the spiritual dimension is a complex task.</s>
+					<s coords="2,244.73,453.11,43.66,9.36;2,56.50,465.11,231.90,9.36;2,56.50,477.11,231.90,9.36;2,56.50,489.11,80.12,9.36;2,136.62,486.96,7.37,6.21;2,144.00,489.11,144.40,9.36;2,56.50,501.11,231.89,9.36">
+						According to current educational principles, learning complex tasks can be supported by providing authentic or realistic learning tasks
+						<ref type="bibr" coords="2,136.62,486.96,7.37,6.21" target="#b19">20</ref>
+						, by using principles of reflective learning, and should be integrated in the curricula.
+					</s>
+					<s coords="2,56.50,513.11,231.89,9.36;2,56.50,525.11,194.45,9.36;2,250.94,522.96,7.37,6.21;2,262.50,525.11,25.89,9.36;2,56.50,537.11,231.90,9.36;2,56.50,549.11,24.87,9.36;2,81.36,546.96,7.37,6.21">
+						Authentic tasks allow students to acquire knowledge, skills and attitudes in an integrated fashion,
+						<ref type="bibr" coords="2,250.94,522.96,7.37,6.21" target="#b20">21</ref>
+						which improves the transfer of the curriculum to the workplace.
+						<ref type="bibr" coords="2,81.36,546.96,7.37,6.21" target="#b19">20</ref>
+					</s>
+					<s coords="2,93.25,549.11,195.14,9.36;2,56.50,561.11,231.90,9.36;2,56.50,573.11,84.74,9.36">These authentic learning tasks can be interwoven in existing curricula in a horizontal and vertical integration manner.</s>
+					<s coords="2,91.86,594.96,7.22,6.21;2,101.14,597.11,187.26,9.36;2,56.50,609.11,165.28,9.36;2,221.77,606.96,7.37,6.21;2,231.56,609.11,56.83,9.36;2,56.50,621.11,231.89,9.36;2,56.50,633.11,231.89,9.36;2,56.50,645.11,23.83,9.36;2,80.32,642.96,7.37,6.21">
+						3]
+						<ref type="bibr" coords="2,88.25,594.96,3.61,6.21" target="#b23">[24]</ref>
+						<ref type="bibr" coords="2,91.86,594.96,7.22,6.21" target="#b24">[25]</ref>
+						Through reflection, students are encouraged to think about their role as a physician,
+						<ref type="bibr" coords="2,221.77,606.96,7.37,6.21" target="#b21">22</ref>
+						foster professional growth, release the emotional burden of caring for palliative care patients and increase patient care skills.
+						<ref type="bibr" coords="2,80.32,642.96,7.37,6.21" target="#b23">24</ref>
+					</s>
+					<s coords="2,90.92,645.11,197.48,9.36;2,56.51,657.11,227.44,9.36;2,283.94,654.96,3.69,6.21">
+						Self-reflective training on the spiritual dimensions within the students' own lives is recommended.
+						<ref type="bibr" coords="2,283.94,654.96,3.69,6.21" target="#b1">2</ref>
+					</s>
+					<s coords="2,65.50,669.11,222.90,9.36;2,56.50,681.11,231.90,9.36;2,56.50,693.11,231.88,9.36;2,56.50,705.11,212.47,9.36">e developed a coherent set of realistic authentic learning tasks, in which students learn about and reflect on communication about the four dimensions of care, with a particular focus on the spiritual dimension.</s>
+					<s coords="2,271.54,705.11,16.86,9.36;2,56.50,717.11,231.88,9.36;2,56.50,729.11,231.90,9.36;2,306.89,45.11,231.90,9.36;2,306.89,57.11,157.54,9.36">The main aims of these learning tasks are that students learn about spiritual care, are able to talk about it with a palliative care patient, and to reflect on their spiritual experiences regarding life and death.</s>
+					<s coords="2,467.16,57.11,71.62,9.36;2,306.89,69.11,231.90,9.36;2,306.89,81.11,231.88,9.36;2,306.89,93.11,231.89,9.36;2,306.89,105.11,231.89,9.36;2,306.89,117.11,231.90,9.36;2,306.89,129.11,115.41,9.36">This article gives more insight into the usability and feasibility of these learning tasks from the stakeholders' perspectives, that is, medical students, teachers and educational scientists on the design of the learning tasks based on the educational principles of authentic educational scenarios, reflection and integration.</s>
+					<s coords="2,426.79,129.11,112.00,9.36;2,306.89,141.11,231.89,9.36;2,306.89,153.11,231.90,9.36;2,306.89,165.11,228.96,9.36">The research question is: How do medical students, teachers and educational scientists evaluate a set of coherent learning tasks focusing on the spiritual dimension of palliative care?</s>
+				</p>
+			</div>
+			<div xmlns="http://www.tei-c.org/ns/1.0">
+				<head coords="2,306.89,189.29,47.13,9.24">METHODS</head>
+			</div>
+			<div xmlns="http://www.tei-c.org/ns/1.0">
+				<head coords="2,306.89,200.84,26.58,7.82">Design</head>
+				<p>
+					<s coords="2,306.89,211.61,231.88,9.36;2,306.89,223.61,231.90,9.36;2,306.89,235.61,93.84,9.36">Three groups of stakeholders were asked to participate in this evaluation: medical students, teachers, and educational scientists.</s>
+					<s coords="2,404.50,235.61,134.28,9.36;2,306.89,247.61,70.68,9.36">The students were interviewed in focus groups.</s>
+					<s coords="2,381.54,247.61,157.23,9.36;2,306.89,259.61,231.89,9.36;2,306.89,271.61,92.69,9.36">The teachers and educational scientists were questioned in individual interviews, due to their busy schedules.</s>
+					<s coords="2,404.55,271.61,134.24,9.36;2,306.89,283.61,231.88,9.36;2,306.89,295.61,73.76,9.36">This qualitative approach was used to gather in-depth information and insights from our stakeholders.</s>
+				</p>
+			</div>
+			<div xmlns="http://www.tei-c.org/ns/1.0">
+				<head coords="2,306.89,319.34,27.97,7.82">Setting</head>
+				<p>
+					<s coords="2,306.89,330.11,231.88,9.36;2,306.89,342.11,20.24,9.36">In the Netherlands, it takes 6 years to qualify as a physician.</s>
+					<s coords="2,330.85,342.11,207.93,9.36;2,306.89,354.11,231.89,9.36;2,306.89,366.11,22.54,9.36">In the first 3 years, the Bachelor's program, the student primarily acquires theory and medical knowledge.</s>
+					<s coords="2,334.14,366.11,204.63,9.36;2,306.89,378.11,231.90,9.36;2,306.89,390.11,231.88,9.36;2,306.89,402.11,49.79,9.36">In the last 3 years, the Master's program, the focus is on the application of knowledge in the work setting by letting students rotate between different internships.</s>
+				</p>
+			</div>
+			<div xmlns="http://www.tei-c.org/ns/1.0">
+				<head coords="2,306.89,425.84,116.44,7.82">Coherent set of learning tasks</head>
+				<p>
+					<s coords="2,306.89,436.61,231.86,9.36;2,306.89,448.61,231.89,9.36;2,306.89,460.61,231.87,9.36;2,306.89,472.61,86.70,9.36">
+						We designed a set of eight learning tasks (table
+						<ref type="table" coords="2,513.72,436.61,4.45,9.36" target="#tab_0">1</ref>
+						; for a full description, see online supplemental appendix 1), designed to be integrated into the undergraduate medical curriculum.
+					</s>
+					<s coords="2,397.51,472.61,141.26,9.36;2,306.89,484.61,231.89,9.36;2,306.89,496.61,178.30,9.36">The designers included diversity and variations in teaching methods, diseases, treatment plans, age and gender of the patient.</s>
+					<s coords="2,487.66,496.61,51.11,9.36;2,306.89,508.61,202.74,9.36">The competencies to be acquired are described in box 1.</s>
+					<s coords="2,513.58,508.61,25.19,9.36;2,306.89,520.61,231.89,9.36;2,306.89,532.61,52.49,9.36;2,359.38,530.46,7.37,6.21;2,315.89,544.61,222.90,9.36;2,306.89,556.61,231.90,9.36;2,306.89,568.61,77.61,9.36">
+						These competences are a selection of the framework from Pieters et al.
+						<ref type="bibr" coords="2,359.38,530.46,7.37,6.21" target="#b18">19</ref>
+						The educational principles of authenticity, and reflection are incorporated into the set of learning tasks (see table
+						<ref type="table" coords="2,372.84,568.61,3.89,9.36" target="#tab_1">2</ref>
+						).
+					</s>
+				</p>
+			</div>
+			<div xmlns="http://www.tei-c.org/ns/1.0">
+				<head coords="3,448.91,24.52,85.87,9.48">Original research</head>
+			</div>
+			<div xmlns="http://www.tei-c.org/ns/1.0">
+				<head coords="3,56.50,225.44,36.35,7.82">Participants</head>
+				<p>
+					<s coords="3,56.50,236.21,231.88,9.36;3,56.50,248.21,231.90,9.36;3,56.50,260.21,95.79,9.36">Three groups of stakeholders were asked to participate in the evaluation: medical students, teachers and educational scientists.</s>
+					<s coords="3,158.01,260.21,130.37,9.36;3,56.50,272.21,231.88,9.36;3,56.50,284.21,71.47,9.36">The stakeholders came from faculties of medicine of four different universities in the Netherlands.</s>
+				</p>
+			</div>
+			<div xmlns="http://www.tei-c.org/ns/1.0">
+				<head coords="3,56.50,309.28,73.82,7.82">Medical students (N=9)</head>
+				<p>
+					<s coords="3,56.50,320.05,231.89,9.36;3,56.50,332.05,231.87,9.36;3,56.50,344.05,48.14,9.36">These stakeholders were asked to be interviewed in focus groups as they represented the educational users as learners.</s>
+					<s coords="3,107.86,344.05,180.53,9.36;3,56.50,356.05,231.88,9.36;3,56.50,368.05,87.16,9.36">Medical students in their final year of the Bachelor's program or studying for their Master's degree were invited.</s>
+					<s coords="3,146.62,368.05,141.78,9.36;3,56.50,380.05,231.87,9.36;3,56.50,392.05,231.90,9.36;3,56.50,404.05,135.74,9.36">These students have an informed opinion as to which tasks they deemed suitable for students and at what stage the tasks could best be implemented in the curriculum.</s>
+				</p>
+			</div>
+			<div xmlns="http://www.tei-c.org/ns/1.0">
+				<head coords="3,56.50,429.12,101.30,7.82">Teachers of palliative care (N=9)</head>
+				<p>
+					<s coords="3,56.50,439.89,231.87,9.36;3,56.50,451.89,231.88,9.36;3,56.50,463.89,73.68,9.36">These stakeholders were invited for their insight and experience in education and their substantive expertise in palliative care.</s>
+					<s coords="3,133.20,463.89,155.20,9.36;3,56.50,475.89,231.88,9.36;3,56.50,487.89,231.89,9.36;3,56.50,499.89,80.65,9.36">This group of stakeholders included medical specialists, mental healthcare providers and psychologists involved in teaching in undergraduate medical education.</s>
+				</p>
+				<p>
+					<s coords="3,306.89,44.93,87.73,7.82">Educational scientists (N=4)</s>
+				</p>
+				<p>
+					<s coords="3,306.89,55.70,231.90,9.36;3,306.89,67.70,231.90,9.36;3,306.89,79.70,231.89,9.36;3,306.89,91.70,211.09,9.36">These stakeholders were asked for their expertise in both educational design and the educational principles used in this learning program (Authentic learning, reflection and integration into existing courses).</s>
+					<s coords="3,521.90,91.70,16.86,9.36;3,306.89,103.70,212.67,9.36">The educational scientists worked at medical faculties.</s>
+				</p>
+			</div>
+			<div xmlns="http://www.tei-c.org/ns/1.0">
+				<head coords="3,306.89,129.24,46.29,7.82">Instruments</head>
+				<p>
+					<s coords="3,306.89,140.01,231.89,9.36;3,306.89,152.01,231.88,9.36;3,306.89,164.01,231.90,9.36;3,306.89,176.01,187.04,9.36">The teachers and educational scientists were interviewed individually, the students were interviewed in focus groups, using the same semistructured interview guide (see online supplemental appendix 2).</s>
+					<s coords="3,496.36,176.01,42.42,9.36;3,306.89,188.01,231.89,9.36;3,306.89,200.01,231.89,9.36;3,306.89,212.01,231.88,9.36;3,306.89,224.01,231.88,9.36;3,306.89,236.01,70.09,9.36">The interview guide asked for perceptions of the set of learning tasks, focusing on the educational learning principles that shaped them: authentic learning tasks, the principles of reflective learning, and the integration into the existing courses.</s>
+				</p>
+			</div>
+			<div xmlns="http://www.tei-c.org/ns/1.0">
+				<head coords="3,306.89,261.55,38.67,7.82">Procedure</head>
+				<p>
+					<s coords="3,306.89,272.32,231.89,9.36;3,306.89,284.32,231.91,9.36;3,306.89,296.32,105.21,9.36">Participants were recruited through purposeful sampling within the network of Pasemeco until data saturation was reached.</s>
+					<s coords="3,417.20,296.32,121.57,9.36;3,306.89,308.32,195.27,9.36">They were invited through an email that included the information letter.</s>
+					<s coords="3,505.29,308.32,33.50,9.36;3,306.89,320.32,231.90,9.36;3,306.89,332.32,231.89,9.36;3,306.89,344.32,231.88,9.36">Prior to their individual or focus group interview, the participants received a short explanatory video and a document with an overview of the set of learning tasks.</s>
+					<s coords="3,306.89,356.32,231.90,9.36;3,306.89,368.32,109.85,9.36">The interviews took no more than 45 min and were conducted by JP and EN.</s>
+					<s coords="3,419.96,368.32,118.81,9.36;3,306.89,380.32,111.65,9.36">The interviewees gave their written informed consent.</s>
+				</p>
+			</div>
+			<div xmlns="http://www.tei-c.org/ns/1.0">
+				<head coords="3,306.89,405.87,32.08,7.82">Analysis</head>
+				<p>
+					<s coords="3,306.89,416.64,215.11,9.36">The interviews were recorded and transcribed.</s>
+					<s coords="3,527.88,416.64,10.90,9.36;3,306.89,428.64,188.59,9.36">To support the qualitative data analysis, Atlas.</s>
+					<s coords="3,495.48,428.64,43.29,9.36;3,306.89,440.64,22.13,9.36">ti V.8 was used.</s>
+					<s coords="3,334.24,440.64,204.54,9.36;3,306.89,452.64,231.88,9.36;3,306.88,464.64,57.78,9.36;3,364.67,462.49,7.37,6.21;3,376.65,464.64,162.13,9.36;3,306.89,476.64,231.90,9.36;3,306.89,488.64,143.68,9.36">
+						The transcripts were analysed using template analysis, taking into account the step-by-step plan of Brooks et al.
+						<ref type="bibr" coords="3,364.67,462.49,7.37,6.21" target="#b25">26</ref>
+						Researchers JP, EN, and FW carried out the preliminary coding of the data, starting with determining the a priori themes.
+					</s>
+					<s coords="3,454.64,488.64,84.12,9.36;3,306.89,500.64,231.89,9.36;3,306.89,512.64,231.89,9.36;3,306.89,524.64,91.86,9.36">These themes were based on the components of the research question (authentic learning tasks; reflection and integration into the curriculum).</s>
+					<s coords="3,402.84,524.64,135.94,9.36;3,306.89,536.64,231.88,9.36;3,306.89,548.64,37.37,9.36">Two interview transcripts were read and coded independently by researchers JP, EN, and FW.</s>
+					<s coords="3,349.37,548.64,189.39,9.36;3,306.89,560.64,231.90,9.36;3,306.89,572.64,98.00,9.36">They discussed their findings, merged the themes into meaningful clusters, and provided an initial coding template.</s>
+					<s coords="3,407.19,572.64,131.59,9.36;3,306.89,584.64,231.89,9.36;3,306.89,596.64,47.56,9.36">This template was then applied independently by the three researchers to three other transcripts.</s>
+				</p>
+				<p>
+					<s coords="3,315.89,608.64,222.88,9.36;3,306.89,620.64,124.20,9.36">After consultation, the template was finalised and applied to the full data set.</s>
+					<s coords="3,435.75,620.64,103.02,9.36;3,306.89,632.64,231.90,9.36;3,306.89,644.64,23.64,9.36">Themes were discussed and elaborated on iteratively with the whole research team.</s>
+					<s coords="3,335.58,644.64,203.19,9.36;3,306.89,656.64,212.79,9.36">JP and DV coded the other eight transcripts independently and discussed the results afterward.</s>
+					<s coords="3,521.92,656.64,16.86,9.36;3,306.89,668.64,225.50,9.36">The results were then discussed among all of the authors.</s>
+				</p>
+			</div>
+			<div xmlns="http://www.tei-c.org/ns/1.0">
+				<head coords="3,306.89,694.18,41.27,7.82">Reflexivity</head>
+				<p>
+					<s coords="3,306.89,704.95,231.89,9.36;3,306.89,716.95,231.89,9.36;3,306.89,728.95,231.90,9.36">This learning program was developed by a multidisciplinary group that included educational scientists (DV and DD), researchers (JP and EN) and physicians</s>
+				</p>
+			</div>
+			<div xmlns="http://www.tei-c.org/ns/1.0">
+				<head coords="3,62.50,50.89,200.31,9.48;3,62.50,62.39,21.66,9.48">Box 1 : Intended competencies of the learning tasks</head>
+				<p>
+					<s coords="3,62.50,84.59,58.25,9.01">Competencies:</s>
+				</p>
+				<p>
+					<s coords="3,62.50,95.59,216.64,9.01;3,62.50,106.59,127.30,9.01">1) The student communicates in a respectful and empathetic manner with patients and relatives.</s>
+				</p>
+				<p>
+					<s coords="3,62.50,117.60,184.98,9.01;3,62.50,128.60,184.29,9.01;3,62.50,139.60,143.12,9.01">2) The student makes the four dimensions (somatic, psychological, social and spiritual) of palliative care discussable with the patient and family.</s>
+				</p>
+				<p>
+					<s coords="3,62.50,150.60,197.81,9.01;3,62.50,161.60,170.37,9.01">3) The student reflects on distance and closeness in the treatment relationship with a palliative patient.</s>
+					<s coords="3,62.50,172.60,203.60,9.01;3,62.50,183.60,102.51,9.01">4) The student reflects on his/her spiritual and existential views around life and death.</s>
+				</p>
+			</div>
+			<div xmlns="http://www.tei-c.org/ns/1.0">
+				<head coords="4,56.50,106.74,38.66,9.24">RESULTS</head>
+			</div>
+			<div xmlns="http://www.tei-c.org/ns/1.0">
+				<head coords="4,56.50,118.29,60.04,7.82">Authentic tasks</head>
+				<p>
+					<s coords="4,56.50,129.06,231.90,9.36;4,56.50,141.06,231.90,9.36;4,56.50,153.06,231.88,9.36;4,56.50,165.06,231.90,9.36">The stakeholders agreed that the educational materials, including the case descriptions, the video clips and short movies, showed the students a realistic view of what they could encounter in the workplace.</s>
+					<s coords="4,56.50,177.06,231.89,9.36;4,56.50,189.06,231.90,9.36;4,56.50,201.06,155.31,9.36">The learning tasks were authentic and presented in a logical and structured order, from paper cases to a fullfledged conversation with a patient.</s>
+					<s coords="4,56.50,177.06,231.89,9.36;4,56.50,189.06,231.90,9.36;4,56.50,201.06,155.31,9.36">The learning tasks were authentic and presented in a logical and structured order, from paper cases to a fullfledged conversation with a patient.</s>
+					<s coords="4,56.50,177.06,231.89,9.36;4,56.50,189.06,231.90,9.36;4,56.50,201.06,155.31,9.36">The learning tasks were authentic and presented in a logical and structured order, from paper cases to a fullfledged conversation with a patient.</s>
+					<s coords="4,215.03,201.06,73.37,9.36;4,56.50,213.06,231.90,9.36;4,56.50,225.06,231.89,9.36;4,56.50,237.06,231.90,9.36">The stakeholders commented that the learning tasks and materials could be further improved if they provided a better reflection of the cultural diversity within the patient population.</s>
+				</p>
+				<p>
+					<s coords="4,56.50,249.06,231.90,9.36;4,56.50,261.06,231.89,9.36;4,56.50,273.06,231.89,9.36;4,56.50,285.06,231.90,9.36;4,56.50,297.06,168.17,9.36">Students especially appreciated the clips in which real patients and doctors talked about palliative care and suggested that encouraging the possibility to discuss cases students have encountered in medical practice would further increase the authenticity.</s>
+				</p>
+			</div>
+			<div xmlns="http://www.tei-c.org/ns/1.0">
+				<head coords="4,56.50,322.24,72.59,7.82">Reflective learning</head>
+				<p>
+					<s coords="4,56.50,333.01,231.88,9.36;4,56.50,345.01,141.35,9.36">Stakeholders recognised reflection is a recurring element across the learning tasks.</s>
+					<s coords="4,200.31,345.01,88.06,9.36;4,56.50,357.01,100.91,9.36">They also consider it to be an important skill.</s>
+					<s coords="4,159.63,357.01,128.75,9.36;4,56.50,369.01,231.90,9.36;4,56.50,381.01,231.90,9.36;4,56.50,393.01,174.42,9.36">According to the stakeholders, the majority of the students should be able to reflect well on the various health dimensions, but the spiritual dimension can be emotionally charged.</s>
+					<s coords="4,235.46,393.01,52.94,9.36;4,56.50,405.01,231.89,9.36;4,56.50,417.01,198.88,9.36">Some stakeholders indicated that to be able to reflect on this properly, some life experiences would not go amiss.</s>
+					<s coords="4,257.91,417.01,30.47,9.36;4,56.50,429.01,231.88,9.36;4,56.50,441.01,192.96,9.36">Novice medical students' reflections lack the necessary depth in comparison with Master's degree students.</s>
+					<s coords="4,251.93,441.01,36.47,9.36;4,56.50,453.01,231.88,9.36">Both the teachers and students preferred oral group reflection.</s>
+				</p>
+				<p>
+					<s coords="4,56.50,465.01,231.90,9.36;4,56.50,477.01,231.89,9.36;4,56.50,489.01,168.94,9.36">The students indicated that they attached great value to the interaction with their fellow students and that it helps them express themselves better.</s>
+					<s coords="4,228.24,489.01,60.14,9.36;4,56.50,501.01,231.88,9.36;4,56.50,513.01,46.77,9.36">Student safety must be ensured in order for them to properly reflect in a group.</s>
+				</p>
+			</div>
+			<div xmlns="http://www.tei-c.org/ns/1.0">
+				<head coords="4,56.50,322.24,72.59,7.82">Reflective learning</head>
+				<head coords="4,56.50,322.24,72.59,7.82">Reflective learning</head>
+				<head coords="4,56.50,322.24,72.59,7.82">Reflective learning</head>
+				<head coords="4,56.50,538.19,43.55,7.82">Integration</head>
+				<p>
+					<s coords="4,56.50,548.96,231.88,9.36;4,56.50,560.96,231.88,9.36">The stakeholders believed that the learning tasks should be integrated across the curriculum in different years.</s>
+					<s coords="4,56.50,572.96,231.89,9.36;4,56.50,584.96,231.89,9.36;4,56.50,596.96,20.38,9.36">However, it is important that this does not become too fragmented and that students can clearly see the cohesion.</s>
+					<s coords="4,79.30,596.96,209.08,9.36;4,56.50,608.96,231.91,9.36;4,56.50,620.96,231.89,9.36;4,56.50,632.96,148.00,9.36">Almost all stakeholders mentioned that the initial tasks (tasks 1-5) would be best implemented in the Bachelor's program, as the cases in these tasks revolve around the basis of palliative care.</s>
+					<s coords="4,207.59,632.96,80.81,9.36;4,56.50,644.96,231.89,9.36;4,56.50,656.96,231.89,9.36;4,56.50,668.96,231.88,9.36;4,56.50,680.96,92.17,9.36">The other learning tasks (tasks 6-8) were considered more appropriate for more advanced students during the internships as these center around communication with a simulated or real patient.</s>
+					<s coords="4,153.08,680.96,135.29,9.36;4,56.50,692.96,231.88,9.36;4,56.50,704.96,181.77,9.36">According to the stakeholders, they required experience with patients that students only acquire during the Master's degree.</s>
+					<s coords="4,242.68,704.96,45.72,9.36;4,56.50,716.96,231.90,9.36;4,56.50,728.96,19.29,9.36">There was no consensus as to when to start the learning trajectory.</s>
+					<s coords="4,80.35,728.96,208.04,9.36;4,306.89,45.11,231.89,9.36;4,306.89,57.11,231.90,9.36;4,306.89,69.11,133.09,9.36">Some stakeholders believed that palliative care education should start in the very first year, so that students become immediately aware that palliative care is a part of routine care.</s>
+					<s coords="4,444.37,69.11,94.41,9.36;4,306.89,81.11,231.89,9.36;4,306.89,93.11,231.89,9.36;4,306.89,105.11,213.52,9.36">But others thought it better to wait until the end of the Bachelor's program, because students would not be open to the subject or not fully understand the subject matter until then.</s>
+					<s coords="4,315.89,117.11,222.89,9.36;4,306.89,129.11,142.94,9.36">The implementation should also take into account the differences between students.</s>
+					<s coords="4,452.97,129.11,85.81,9.36;4,306.89,141.11,231.89,9.36;4,306.89,153.11,81.89,9.36">In the final learning task, students conduct an unsupervised interview with a palliative patient.</s>
+					<s coords="4,391.67,153.11,147.11,9.36;4,306.89,165.11,231.88,9.36;4,306.89,177.11,231.90,9.36">According to the stakeholders, the learning program does prepare students well for this interview, but it is still an emotionally charged task.</s>
+					<s coords="4,306.89,189.11,231.89,9.36;4,306.89,201.11,231.90,9.36;4,306.89,213.11,124.16,9.36">In order to protect both the patients and the students, the students must be well prepared and be able to take the lead in such an interview.</s>
+					<s coords="4,433.77,213.11,105.03,9.36;4,306.89,225.11,147.84,9.36">
+						Some students may need more training or support (table
+						<ref type="table" coords="4,443.07,225.11,3.89,9.36" target="#tab_2">3</ref>
+						).
+					</s>
+				</p>
+			</div>
+			<div xmlns="http://www.tei-c.org/ns/1.0">
+				<head coords="4,306.89,249.29,56.35,9.24">DISCUSSION</head>
+				<p>
+					<s coords="4,306.89,261.11,231.89,9.36;4,306.89,273.11,231.90,9.36;4,306.89,285.11,231.89,9.36;4,306.89,297.11,22.23,9.36">The medical students, teachers, and educational scientists agree that the set of learning tasks was designed in line with contemporary instructional design guidelines.</s>
+					<s coords="4,331.78,297.11,207.01,9.36;4,306.89,309.11,231.89,9.36;4,306.89,321.11,231.89,9.36;4,306.89,333.11,36.27,9.36">Learning was clearly organised around authentic learning tasks relevant to the later profession, using paper, video cases, as well as simulations and real patients.</s>
+					<s coords="4,347.41,333.11,191.36,9.36;4,306.89,345.11,231.88,9.36;4,306.89,357.11,69.73,9.36">The tasks encourage the students to reflect on the four dimensions of palliative care and their personal values.</s>
+					<s coords="4,381.04,357.11,157.74,9.36;4,306.89,369.11,231.87,9.36">There were various options offered for the integration of the tasks into existing curricula.</s>
+					<s coords="4,306.89,381.11,231.90,9.36;4,306.89,393.11,225.88,9.36">The stakeholders also offered a number of suggestions to take into account, which will be discussed further.</s>
+				</p>
+				<p>
+					<s coords="4,315.89,405.11,222.88,9.36;4,306.89,417.11,204.25,9.36">First, palliative care and the spiritual dimension in particular are emotionally charged subjects.</s>
+					<s coords="4,515.34,417.11,23.44,9.36;4,306.89,429.11,231.88,9.36">Extra attention to both student and patient safety is required.</s>
+					<s coords="4,306.89,441.11,231.88,9.36;4,306.89,453.11,231.87,9.36;4,306.89,465.11,231.87,9.36;4,306.89,477.11,178.53,9.36">Both the students and teachers attached great value to the group discussions and reflections, where a bond of trust and psychological safety between all of the attendees was considered to be essential.</s>
+					<s coords="4,489.22,477.11,49.57,9.36;4,306.89,489.11,231.90,9.36;4,306.89,501.11,231.90,9.36;4,306.89,513.11,166.83,9.36;4,473.72,510.96,12.90,6.21;4,315.89,525.11,222.88,9.36;4,306.89,537.11,231.89,9.36;4,306.89,549.11,231.89,9.36;4,306.89,561.11,231.89,9.36;4,306.89,573.11,134.15,9.36">
+						These findings are in line with the literature, where confidentially sharing experiences is needed and only possible in a group of people who know each other.
+						<ref type="bibr" coords="4,473.72,510.96,12.90,6.21">2 27</ref>
+						Second, the educational materials were considered authentic with variation in cases and learning methods, but to increase authenticity, the stakeholders suggested including more cultural diversity among the patient cases used in the learning tasks.
+					</s>
+					<s coords="4,443.75,573.11,95.04,9.36;4,306.89,585.11,231.90,9.36;4,306.89,597.11,39.97,9.36">Contemporary society is more culturally diverse than the current materials reflected.</s>
+					<s coords="4,350.66,597.11,188.12,9.36;4,306.89,609.11,231.90,9.36;4,306.89,621.11,231.88,9.36;4,306.89,633.11,150.06,9.36;4,456.93,630.96,7.37,6.21">
+						It is an important topic that should not be overlooked, given that many physicians are unfamiliar with the specific needs of ethnic minorities regarding palliative care and communication.
+						<ref type="bibr" coords="4,456.93,630.96,7.37,6.21" target="#b27">28</ref>
+					</s>
+					<s coords="4,315.89,645.11,222.89,9.36;4,306.89,657.11,231.89,9.36">hird, there was discussion on how to integrate learning tasks or palliative care education in general.</s>
+					<s coords="4,306.89,669.11,231.89,9.36;4,306.89,681.11,231.90,9.36;4,306.89,693.11,231.87,9.36;4,306.89,705.11,156.93,9.36">All participants agreed that this should start in the Bachelors and most agreed that it should be integrated vertically, throughout the undergraduate program and not as a stand-alone module.</s>
+					<s coords="4,469.68,705.11,69.11,9.36;4,306.89,717.11,231.90,9.36;4,306.89,729.11,231.89,9.36;4,567.14,376.44,8.32,39.01">Furthermore, it needs to be integrated horizontally, where relevant, for instance, in cardiology, oncology, humanities, and copyright.</s>
+				</p>
+				<p>
+					<s coords="4,576.14,582.02,8.33,10.01;4,576.14,594.53,8.33,32.01;4,576.14,629.05,8.32,12.51;4,576.14,644.06,8.33,20.02;4,576.14,666.58,8.33,9.50;4,576.14,678.58,8.33,24.52">on January 21, 2022 by guest.</s>
+					<s coords="4,576.14,705.60,8.33,38.52;4,576.14,746.62,8.33,9.50;4,576.14,490.99,8.33,88.53">Protected by http://spcare.bmj.com/</s>
+					<s coords="5,448.91,24.52,85.87,9.48;5,56.50,512.98,104.79,9.36">Original research communication training.</s>
+					<s coords="5,163.19,512.98,125.21,9.36;5,56.50,524.98,231.88,9.36;5,56.50,536.98,231.90,9.36;5,56.50,548.98,231.90,9.36;5,56.50,560.98,65.80,9.36">Integrating this set of learning tasks, therefore, mainly depends on the curriculum of a medical university and its identity as to where and when palliative care education can be implemented in the curriculum.</s>
+					<s coords="5,124.97,560.98,163.40,9.36;5,56.50,572.98,231.89,9.36;5,56.50,584.98,64.18,9.36;5,120.67,582.83,22.12,6.21">
+						The idea of vertically and horizontally integration is in line with recent educational research and guidelines.
+						<ref type="bibr" coords="5,120.67,582.83,22.12,6.21">4 29 30</ref>
+					</s>
+					<s coords="5,65.50,596.98,222.90,9.36;5,56.50,608.98,231.89,9.36;5,56.50,620.98,231.89,9.36;5,56.50,632.98,231.90,9.36">ome stakeholders argued that for learning about spiritual and palliative care in general, students should have some life experience and, therefore, it would be more suitable to start the program later in the bachelor.</s>
+					<s coords="5,56.50,644.98,231.89,9.36;5,56.50,656.98,231.89,9.36;5,56.50,668.98,20.34,9.36">Others argue that palliative care should be normalised for the students and should, therefore, start in the first year.</s>
+					<s coords="5,79.00,668.98,209.39,9.36;5,56.50,680.98,231.89,9.36;5,56.50,692.98,231.89,9.36;5,56.50,704.98,128.64,9.36;5,185.13,702.83,17.54,6.21">
+						The concern that early exposure may be emotionally challenging for young students, is the main reason why curricula integrate palliative care education in general later in the program.
+						<ref type="bibr" coords="5,185.13,702.83,17.54,6.21">31 32</ref>
+					</s>
+					<s coords="5,206.90,704.98,81.49,9.36;5,56.50,716.98,231.89,9.36;5,56.50,728.98,231.87,9.36;5,306.89,512.98,231.91,9.36;5,306.89,524.98,56.36,9.36;5,363.24,522.83,34.36,6.21">
+						However, research shows that direct experience with palliative care for first-year students is associated with positive effects on the students' attitudes regarding caring for palliative care patients.
+						<ref type="bibr" coords="5,363.24,522.83,34.36,6.21">29 31 33 34</ref>
+					</s>
+					<s coords="5,400.07,524.98,138.71,9.36;5,306.89,536.98,231.89,9.36;5,306.89,548.98,231.90,9.36;5,306.89,560.98,164.04,9.36;5,470.92,558.83,7.37,6.21">
+						Junior doctors who were trained earlier in palliative care have enhanced competencies of psychosocial and spiritual aspects of palliative care, communication, and self-awareness.
+						<ref type="bibr" coords="5,470.92,558.83,7.37,6.21" target="#b33">34</ref>
+					</s>
+					<s coords="5,486.00,560.98,52.77,9.36;5,306.88,572.98,231.89,9.36;5,306.88,584.98,231.89,9.36;5,306.88,596.98,206.08,9.36;5,512.96,594.83,25.81,6.21">
+						Thus, early exposure helps to normalise death and dying and the complex emotions that students and physicians can encounter while treating palliative care patients.
+						<ref type="bibr" coords="5,512.96,594.83,25.81,6.21">31 33 34</ref>
+					</s>
+					<s coords="5,306.89,608.98,231.89,9.36;5,306.89,620.98,231.88,9.36;5,306.89,632.98,231.87,9.36;5,306.89,644.98,112.32,9.36">nterestingly, in this study, students themselves had lively discussions on this topic too, but they mainly discussed more practical obstacles: when is there time for this in the curriculum?</s>
+				</p>
+				<p>
+					<s coords="5,315.89,656.98,141.26,9.36">This study had some limitations.</s>
+					<s coords="5,460.51,656.98,78.29,9.36;5,306.89,668.98,231.88,9.36;5,306.89,680.98,46.39,9.36">To begin with, we evaluated the set of learning tasks prior to its implementation.</s>
+					<s coords="5,357.66,680.98,181.12,9.36;5,306.89,692.98,231.89,9.36;5,306.89,704.98,137.26,9.36">This meant that we could not reflect on the implementation process itself, leaving this to be researched at a later stage.</s>
+					<s coords="5,449.30,704.98,89.46,9.36;5,306.89,716.98,231.89,9.36;5,306.89,728.98,195.29,9.36">We also focused on the stakeholders in the educational setting: medical students, teachers, and educational scientists.</s>
+					<s coords="5,505.64,728.98,33.14,9.36;5,108.50,156.07,221.01,8.06">Patients 'I would also pay more attention to a more diversified background,(…).</s>
+					<s coords="5,330.77,156.07,189.69,8.06;5,108.50,165.57,422.62,8.06;5,108.50,175.07,12.26,8.06">The impression here is that it might be more about the classic white Dutch native than perhaps the Surinamese or Moroccan who gets into these problems and needs an essentially different approach' -I7</s>
+				</p>
+				<p>
+					<s coords="5,108.50,203.57,426.67,8.06;5,108.50,213.08,59.04,8.06">'(the clips, eds.)in the sense that the doctors who were talking, for example a [person] I have met in the clinic as well so that appeals more to the imagination.</s>
+					<s coords="5,168.80,213.08,153.12,8.06">That actually makes it even more authentic' -FG2</s>
+				</p>
+				<p>
+					<s coords="5,108.50,241.58,429.85,8.06;5,108.50,251.08,30.85,8.06">'That there are not only videos but also something like a link to a wishlist, and a Volkskrant [newspaper] article so I think that's already very authentic.</s>
+					<s coords="5,140.60,251.08,334.78,8.06;5,60.50,263.46,29.61,8.06;5,60.50,272.96,25.03,8.06;5,108.50,263.46,419.16,8.06;5,108.50,272.96,418.97,8.06;5,108.50,282.46,386.39,8.06;5,108.50,310.96,424.83,8.06;5,108.50,320.47,76.74,8.06">You might want to make it a little more authentic by having students contribute cases they encounter' -FG2 Reflective learning 'I believe, indeed, that in real life that might be better than on paper, especially because you sometimes have to put something on paper and then you are not really sure or you feel differently or have not enough time and if you are really working in such a group, then ideas emerge or feelings that other people then evoke in you or you hear it and you think, oh that's right, I agree or I do not' -FG1 'What is also important here is that in your educational groups, or in the subgroup where you would discuss this, you have a bond of trust as well as feel safe' -I11</s>
+				</p>
+			</div>
+			<div xmlns="http://www.tei-c.org/ns/1.0">
+				<head coords="5,60.50,332.84,33.70,8.06">Integration</head>
+				<p>
+					<s coords="5,108.50,332.84,277.17,8.06">And what might also be good timing,(…), is the second half of the third year of medicine.</s>
+					<s coords="5,387.39,332.84,141.06,8.06;5,108.50,342.35,429.36,8.06;5,108.50,351.85,44.36,8.06">Students are then very receptive to everything that is useful in their internships, so to speak, because then it is kind of a big thing that you will have to tackle soon and so you have to get started' -FG2</s>
+				</p>
+				<p>
+					<s coords="5,108.50,380.35,181.60,8.06">'I see no reason not to do it (integration as of year 1, eds.).</s>
+					<s coords="5,291.36,380.35,238.27,8.06;5,108.50,389.85,71.45,8.06">And another advantage is that students will get an idea of -oh, yes, so this is part of the job as well?</s>
+					<s coords="5,181.99,389.85,349.34,8.06;5,108.50,399.35,409.56,8.06;5,108.50,408.86,305.78,8.06">It soon makes it more normal, because students often enrol with a strong idea of: we are going to make everyone better… but accepting that you can't make someone better will become the reality… but it is not really part of the idea of beginning students so, in that sense, I think: maybe it's also good to create some kind of awareness here' -I12</s>
+				</p>
+				<p>
+					<s coords="5,108.50,437.36,316.08,8.06">'I believe you could use that simulation interview as a go/no-go for having an interview with a patient.</s>
+					<s coords="5,425.84,437.36,97.24,8.06;5,108.50,446.86,414.15,8.06;5,108.50,456.36,425.00,8.06;5,108.50,465.87,102.09,8.06;5,60.50,478.19,90.18,8.06">And, once again, they really are vulnerable patients, often in the final stage of life, who are often quite willing to contribute to the training, but who should also not be burdened disproportionately by oafs or students who are not interested or cut corners, and that such a patient is about to go home with a bad feeling after 45 minutes' -I8 FG, Focus Group; I, Interview.</s>
+					<s coords="6,56.50,45.11,231.87,9.36;6,56.50,57.12,231.87,9.36;6,56.50,69.13,150.15,9.36">were not involved in the evaluation, although they were involved in the overarching project within which the learning tasks were developed.</s>
+					<s coords="6,210.12,69.13,78.29,9.36;6,56.50,81.14,231.87,9.36;6,56.50,93.15,159.03,9.36">Last but not least, the educational principles within this learning program were tailored to the Dutch situation.</s>
+					<s coords="6,218.82,93.15,69.57,9.36;6,56.50,105.16,231.89,9.36;6,56.50,117.17,231.90,9.36;6,56.50,129.18,39.88,9.36">The educational principles and general setup of the design are internationally transferable, if tailored to the national situation.</s>
+					<s coords="6,100.44,129.18,187.95,9.36;6,56.50,141.19,231.90,9.36;6,56.50,153.20,77.26,9.36">Following the authenticity principle means that it is important to paint a realistic picture of the professional field.</s>
+					<s coords="6,137.41,153.20,150.97,9.36;6,56.50,165.21,231.89,9.36;6,56.50,177.22,127.15,9.36">If authenticity is to be guaranteed, learning tasks will have to be reviewed and adapted to be used in other countries.</s>
+					<s coords="6,186.79,177.22,101.60,9.36;6,56.50,189.23,133.47,9.36">This also regards to the principle of reflective learning.</s>
+					<s coords="6,193.19,189.23,95.19,9.36;6,56.50,201.24,231.89,9.36;6,56.50,213.25,231.88,9.36;6,56.50,225.26,49.64,9.36">The students involved in this study are already familiar with reflecting and sharing their opinion, since it plays a vital role in their curriculum.</s>
+					<s coords="6,108.94,225.26,179.46,9.36;6,56.50,237.27,231.89,9.36">On a national level there is a lively debate on end-of-life care, but not on the spiritual dimension.</s>
+					<s coords="6,56.51,249.28,231.88,9.36;6,56.51,261.29,47.17,9.36">Therefore, an emphasis was placed on the spiritual dimension.</s>
+				</p>
+				<p>
+					<s coords="6,65.51,273.30,196.78,9.36">This study has several practical implications.</s>
+					<s coords="6,266.39,273.30,22.01,9.36;6,56.51,285.31,231.87,9.36;6,56.51,297.32,160.16,9.36">First, it pays off to integrate the program horizontally and vertically into the curriculum.</s>
+					<s coords="6,222.91,297.32,65.48,9.36;6,56.51,309.33,231.90,9.36;6,56.51,321.34,181.09,9.36">It depends on the specific tasks and the curriculum itself how and where the integration can best take place.</s>
+					<s coords="6,240.93,321.34,47.47,9.36;6,56.51,333.35,231.90,9.36;6,56.51,345.36,231.87,9.36;6,56.51,357.37,109.08,9.36">Some tasks will fit in well with education regarding communication or where clinical conditions are discussed such as oncology or lung disease.</s>
+					<s coords="6,168.62,357.37,119.77,9.36;6,56.51,369.38,231.90,9.36;6,56.51,381.39,231.88,9.36;6,56.51,393.40,231.89,9.36;6,56.51,405.41,231.88,9.36;6,56.51,417.42,153.03,9.36">To avoid fragmentation and guarantee that all aspects are covered, however, it is also important that someone has an overview of where and how palliative care is addressed in the curriculum, for example, a curriculum coordinator or someone specifically assigned with this task.</s>
+					<s coords="6,213.64,417.42,74.76,9.36;6,56.51,429.43,213.40,9.36">Furthermore, the tasks can always be adapted to specific contexts.</s>
+					<s coords="6,273.72,429.43,14.69,9.36;6,56.51,441.44,231.89,9.36;6,56.51,453.45,119.44,9.36">For example, it might be difficult to arrange interviews with palliative care patients.</s>
+					<s coords="6,178.45,453.45,109.95,9.36;6,56.51,465.46,231.90,9.36;6,56.51,477.47,32.57,9.36">Students could then interview a chronically ill patient instead of a palliative patient.</s>
+					<s coords="6,91.84,477.47,196.56,9.36;6,56.51,489.48,231.88,9.36;6,56.51,501.49,209.89,9.36">With explicit attention to communication and spiritual care education, it is possible to better prepare students for working in the professional field.</s>
+					<s coords="6,271.55,501.49,16.86,9.36;6,56.51,513.50,231.89,9.36;6,56.51,525.51,112.93,9.36">The spiritual dimension of care deserves explicit attention in the medical curriculum.</s>
+				</p>
+			</div>
+			<figure xmlns="http://www.tei-c.org/ns/1.0" xml:id="fig_0" coords="5,567.14,376.44,8.32,39.01;5,576.14,582.02,8.33,10.01;5,576.14,594.53,8.33,32.01;5,576.14,629.05,8.32,12.51;5,576.14,644.06,8.33,20.02;5,576.14,666.58,8.33,9.50;5,576.14,678.58,8.33,24.52;5,576.14,705.60,8.33,38.52;5,576.14,746.62,8.33,9.50;5,576.14,490.99,8.33,88.53;6,60.50,24.52,85.87,9.48">
+				<head></head>
+				<label></label>
+				<figDesc>
+					<div>
+						<p>
+							<s coords="5,581.69,629.05,2.77,12.51;5,576.14,644.06,8.33,20.02;5,576.14,666.58,8.33,9.50;5,576.14,678.58,8.33,24.52">, 2022 by guest.</s>
+							<s coords="5,576.14,705.60,8.33,38.52;5,576.14,746.62,8.33,9.50;5,576.14,490.99,8.33,88.53">Protected by http://spcare.bmj.com/</s>
+							<s coords="6,60.50,24.52,85.87,9.48">Original research</s>
+						</p>
+					</div>
+				</figDesc>
+			</figure>
+			<figure xmlns="http://www.tei-c.org/ns/1.0" type="table" xml:id="tab_0" coords="2,310.89,376.44,273.57,379.68">
+				<head>Table 1</head>
+				<label>1</label>
+				<figDesc>
+					<div>
+						<p>
+							<s coords="2,348.73,604.84,81.98,9.01">The eight learning tasks</s>
+						</p>
+					</div>
+				</figDesc>
+				<table coords="2,310.89,376.44,264.57,357.42">
+					<row>
+						<cell></cell>
+						<cell>copyright.</cell>
+					</row>
+					<row>
+						<cell>1</cell>
+						<cell>Learning about the four dimensions: physical, psychological, social</cell>
+					</row>
+					<row>
+						<cell></cell>
+						<cell>and emotional</cell>
+					</row>
+					<row>
+						<cell>2</cell>
+						<cell>Discussing and reflecting on palliative patient in the context of life</cell>
+					</row>
+					<row>
+						<cell>3</cell>
+						<cell>Reflection on personal views on life, death and dying</cell>
+					</row>
+					<row>
+						<cell>4</cell>
+						<cell>Recognising the spiritual dimension</cell>
+					</row>
+					<row>
+						<cell>5</cell>
+						<cell>Learning to integrate the spiritual dimension into counselling</cell>
+					</row>
+					<row>
+						<cell>6</cell>
+						<cell>To conduct an interview on meaning and purpose and work with</cell>
+					</row>
+					<row>
+						<cell></cell>
+						<cell>the diamond model</cell>
+					</row>
+					<row>
+						<cell>7</cell>
+						<cell>Simulation interview with a palliative patient</cell>
+					</row>
+					<row>
+						<cell>8</cell>
+						<cell>Unsupervised interview with a palliative patient</cell>
+					</row>
+				</table>
+			</figure>
+			<figure xmlns="http://www.tei-c.org/ns/1.0" type="table" xml:id="tab_1" coords="3,60.50,376.44,523.96,379.68">
+				<head>Table 2</head>
+				<label>2</label>
+				<figDesc>
+					<div>
+						<p>
+							<s coords="3,98.34,541.72,188.43,9.01;3,60.50,552.72,47.85,9.01">The incorporated educational principles in the different learning tasks</s>
+						</p>
+					</div>
+				</figDesc>
+				<table coords="3,60.50,376.44,523.96,379.68">
+					<row>
+						<cell></cell>
+						<cell>copyright.</cell>
+						<cell></cell>
+					</row>
+					<row>
+						<cell>Authentic tasks Reflecting Integration</cell>
+						<cell>The materials provided are based on life-like, real cases encountered by professionals in their work. The learning tasks have been developed in close consultation with various care professionals. Furthermore, the student interviews a real patient they have met in the workplace. Reflection is a vital part of almost every learning task. Reflection is emphasised in several areas: ► Reflection on personal values, standards, and allocation of meaning and purpose (learning tasks 2, 3 and 5) ► Reflection on the influence of the four dimensions on the patient and the care trajectory (learning task 1, 4, 5 and 8) The learning tasks are developed for integration into existing courses. authentic tasks. The video clips used are based on</cell>
+						<cell>on January 21, 2022 by guest. Protected by http://spcare.bmj.com/</cell>
+					</row>
+				</table>
+			</figure>
+			<figure xmlns="http://www.tei-c.org/ns/1.0" type="table" xml:id="tab_2" coords="5,60.50,50.36,475.29,85.26">
+				<head>Table 3</head>
+				<label>3</label>
+				<figDesc>
+					<div>
+						<p>
+							<s coords="5,98.34,50.36,134.78,9.01;5,145.80,80.06,206.18,8.06">Themes and quotes of the stakeholders this(authenticity, eds.)in so far that patients tell you a lot of stories.</s>
+							<s coords="5,353.71,80.06,182.09,8.06;5,108.50,89.56,161.34,8.06;5,108.50,118.06,189.47,8.06">So, in that sense, they are all very authentic stories and this is how patients present themselves to doctors' -I10 'If you look at the learning tasks, you do see a clear structure.</s>
+							<s coords="5,299.70,118.06,231.07,8.06;5,108.50,127.57,130.78,8.06">From watching to an increasingly active role to eventually actually having a conversation with a patient, of course' -I8</s>
+						</p>
+					</div>
+				</figDesc>
+				<table coords="5,60.50,65.69,83.26,31.93">
+					<row>
+						<cell>Theme</cell>
+						<cell>Quotes</cell>
+					</row>
+					<row>
+						<cell>Authentic</cell>
+						<cell>'I recognise</cell>
+					</row>
+					<row>
+						<cell>tasks</cell>
+						<cell></cell>
+					</row>
+				</table>
+			</figure>
+			<note xmlns="http://www.tei-c.org/ns/1.0" place="foot">BMJ Support Palliat Care: first published as 10.1136/bmjspcare-2021-003026 on 20 July 2021. Downloaded from</note>
+			<note xmlns="http://www.tei-c.org/ns/1.0" place="foot">Pieters J, et al. BMJ Supportive &amp; Palliative Care 2021;0:1-7. doi:10.1136/bmjspcare-2021-003026</note>
+		</body>
+		<back>
+
+			<div type="acknowledgement">
+				<div xmlns="http://www.tei-c.org/ns/1.0">
+					<head coords="6,56.50,548.07,69.90,8.06">Acknowledgements</head>
+					<p>
+						<s coords="6,130.65,548.07,129.88,7.96;6,56.50,557.58,218.13,7.96">We thank Judith Westen and Jimmy Frerejean for their support in designing these learning tasks.</s>
+						<s coords="6,276.96,557.58,11.43,7.96;6,56.50,567.09,222.78,7.96;6,56.50,576.60,31.41,7.96">We further thank the involved students, teachers and educational scientist.</s>
+					</p>
+				</div>
+			</div>
+
+			<div type="annex">
+				<div xmlns="http://www.tei-c.org/ns/1.0">
+					<p>
+						<s coords="6,56.50,590.11,227.18,8.07;6,56.50,599.62,124.24,7.96">Contributors DV and FW developed the learning tasks; JP and EN conducted the data collection.</s>
+						<s coords="6,183.11,599.62,97.60,7.96;6,56.50,609.13,31.66,7.96">JP, EN and FW interpreted the data.</s>
+						<s coords="6,90.53,609.13,179.27,7.96">JP drafted the manuscript; DV, MvdBvE and DD.</s>
+						<s coords="6,56.50,618.64,105.25,7.96">supervised the study process.</s>
+						<s coords="6,164.11,618.64,119.80,7.96;6,56.50,628.15,137.67,7.96">All authors have read and agreed to the final version of the manuscript.</s>
+					</p>
+					<p>
+						<s coords="6,56.50,641.66,231.68,8.07;6,56.50,651.17,70.40,7.96">Funding This work was supported by ZonMW (project number 80-84400-98-027).</s>
+					</p>
+				</div>
+				<div xmlns="http://www.tei-c.org/ns/1.0">
+					<head coords="6,56.50,664.68,132.07,8.07">Competing interests None declared.</head>
+					<p>
+						<s coords="6,56.50,678.18,164.07,8.07">Patient consent for publication Not required.</s>
+					</p>
+				</div>
+				<div xmlns="http://www.tei-c.org/ns/1.0">
+					<head coords="6,56.50,691.69,204.20,8.07;6,56.50,701.20,229.14,7.96">Ethics approval Ethical clearance was obtained from the Netherlands Association for Medical Education Ethical Review</head>
+					<p></p>
+					<p>
+						<s coords="6,306.88,45.02,215.24,8.07;6,306.88,54.66,53.42,7.96">Provenance and peer review Not commissioned; externally peer reviewed.</s>
+					</p>
+				</div>
+				<div xmlns="http://www.tei-c.org/ns/1.0"></div>
+				<div xmlns="http://www.tei-c.org/ns/1.0">
+					<head coords="6,306.88,91.08,203.26,8.07;6,306.88,100.22,207.66,7.96">Open access This is an open access article distributed in accordance with the Creative Commons Attribution Non</head>
+					<p>
+						<s coords="6,306.88,109.36,214.63,7.96;6,306.88,118.49,195.94,7.96;6,306.88,127.63,220.30,7.96;6,306.88,136.77,231.89,7.96;6,306.88,145.91,230.06,7.96;6,306.88,155.04,44.63,7.96">Commercial (CC BY-NC 4.0) license, which permits others to distribute, remix, adapt, build upon this work noncommercially, and license their derivative works on different terms, provided the original work is properly cited, appropriate credit is given, any changes made indicated, and the use is noncommercial.</s>
+						<s coords="6,353.87,155.04,105.68,7.96">See: http:// creativecommons.</s>
+						<s coords="6,459.54,155.04,73.63,7.96;6,306.88,164.18,9.45,7.96">org/ licenses/ by-nc/ 4. 0/.</s>
+					</p>
+				</div>
+
+			</div>
+
+		</back>
+	</text>
+</TEI>

good_ai/tests/sus/test_clean.py

@@ -0,0 +1,103 @@
+import unittest
+
+from src.sus.clean import clean
+
+
+class TestClean(unittest.TestCase):
+    def test_xml_handling(self) -> None:
+        xml = '<strong>Hi, </strong> my name<br/>is <span style="color: hotpink;"> András</span>! &lt;&#51; <> < ></><> &lt;&gt; <|'
+        clean_xml = "Hi, my name is András! <3 <> <|"
+        clean_xml_ascii = "Hi, my name is Andras! <3 <> <|"
+
+        self.assertEqual(clean(xml), clean_xml)
+        self.assertEqual(clean(xml, ignore_xml=True, ignore_latex=True), xml)
+        self.assertEqual(clean(xml, convert_to_ascii=True), clean_xml_ascii)
+
+    def test_simple_latex_handling(self) -> None:
+        latex = 'Bj\\"{o}rn is \\textit{happy} 🙂'
+        clean_latex = "Björn is happy 🙂"
+        clean_latex_ascii = "Bjorn is happy"
+
+        self.assertEqual(clean(latex), clean_latex)
+        self.assertEqual(clean(latex, ignore_latex=True), latex)
+        self.assertEqual(clean(latex, convert_to_ascii=True), clean_latex_ascii)
+
+    def test_cursed(self) -> None:
+        cursed = """
+        t̴̨̢̝̻͚͓͉̀̄̃́͜o̸͉̰̼͖͖̅̀̓̿̇̚͝ ̶̢͕̄͊̾͑̂̀̉͑ṱ̸̨̛͈̣̠̤͍̰̂̅͋̀́h̸̹̐͗̅̉͐͂͝e̶̜̳̞̍͘ ̴̟̗̻̤̤̮̒̂̋̓́͐͘͜m̵̺̫̥̙̣̥͐̌͜o̴̖͙̙̎̒͂o̷̬̤͑̆̂̉̊̂n̸̥͒̊
+        """
+        cleaned = "to the moon"
+
+        self.assertEqual(clean(cursed), cursed.strip())
+        self.assertEqual(clean(cursed, convert_to_ascii=True), cleaned)
+
+    def test_whitespace(self) -> None:
+        text = """
+        
+        word1
+
+          word2 \n\n\n\t
+        wo\t\f rd3
+
+
+        """  # noqa: W293
+        cleaned = "word1 word2 wo rd3"
+
+        self.assertEqual(clean(text, convert_to_ascii=True), cleaned)
+        self.assertEqual(clean(text, ignore_xml=True, ignore_latex=True), cleaned)
+        self.assertEqual(clean(text), cleaned)
+
+    def test_hyphens(self) -> None:
+        text = """
+            break - word
+            break- word
+            break -word
+            break    -word
+            break-\tword
+        """
+        cleaned = "break - word break-word break-word break-word break-word"
+
+        self.assertEqual(clean(text), cleaned)
+
+    def test_bracket_removal(self) -> None:
+        text = "something [0], and [frefe, ferf]"
+        cleaned = "something, and"
+
+        self.assertEqual(clean(text, remove_brackets=True), cleaned)
+        self.assertEqual(
+            clean(text, ignore_xml=True, ignore_latex=True, remove_brackets=True),
+            cleaned,
+        )
+        self.assertEqual(
+            clean(text, convert_to_ascii=True, remove_brackets=True), cleaned
+        )
+        self.assertEqual(clean(text), text)
+
+    def test_latex_math_handling(self) -> None:
+        latex = "An increase of 3% was achieved with $q_1 = \\frac{3}{5}$."
+        bad_latex = "An increase of 3% was achieved with $q_1 = \\frac{3}/5$."
+        clean_latex = "An increase of 3% was achieved with q_1 = 3/5."
+        clean_bad_latex = "An increase of 3% was achieved with q_1 = 3//5."
+
+        self.assertEqual(clean(latex), clean_latex)
+        self.assertEqual(clean(bad_latex), clean_bad_latex)
+        self.assertEqual(clean(latex, ignore_latex=True), latex)
+        self.assertEqual(clean(bad_latex, ignore_latex=True), bad_latex)
+
+    def test_everything(self) -> None:
+        text = """
+         Hi % 3 &gt;2   <h2 color="red">my paper</h2> there! cost- effective cost - effective cost -effective \\frac{1/2} hi \\frac{1}{2} \\textit{italic} \\`acc\\^ented text 北亰 fi æ IJ ij ff András öô 2132 rgrv \n\\ &nbsp; fd [32] [Bei et al., 2003]\n  🤡 🙄 😶‍🌫️ 🇲🇲
+        """
+        cleaned = "Hi% 3 >2 my paper there! cost-effective cost - effective cost-effective 1/2/hi 1/2 italic accented text Bei Jing fi ae IJ ij ff Andras oo 2132 rgrv fd"
+
+        self.assertEqual(
+            clean(text, convert_to_ascii=True, remove_brackets=True), cleaned
+        )
+
+    def test_empty(self) -> None:
+        self.assertEqual(clean("", convert_to_ascii=True), "")
+
+    def test_punctuation_fixing(self) -> None:
+        text = " Dear reader ( or  listener ) , welcome ! "
+        fixed = "Dear reader (or listener), welcome!"
+        self.assertEqual(clean(text), fixed)

good_ai/tests/sus/test_evaluate_ranking.py

@@ -0,0 +1,93 @@
+import unittest
+from pathlib import Path
+
+from src.sus.evaluate_ranking import evaluate_ranking
+
+
+class TestEvaluateRanking(unittest.TestCase):
+    def test_default(self) -> None:
+        results = evaluate_ranking(
+            ["a", "a", "b", "b", "c", "d", "d", "d"],
+            [10, 7, 11, 6, 8, 2, 7, 1],
+            target_recall=0.6,
+            reverse_order=True,
+        )
+
+        self.assertEqual(
+            results,
+            {"d": 1.0, "c": 0.6666666666666666, "b": 0.4},
+        )
+
+        results = evaluate_ranking(
+            ["a", "a", "b", "b", "c", "d", "d", "d"],
+            [10, 7, 11, 6, 8, 2, 7, 1],
+            target_recall=0.6,
+            reverse_order=False,
+            disable_interpolation=True,
+        )
+
+        self.assertEqual(
+            results,
+            {"a": 0.6666666666666666, "b": 0.3333333333333333, "c": 0.2857142857142857},
+        )
+
+    def test_mismatching_lengths(self) -> None:
+        self.assertRaises(
+            ValueError,
+            evaluate_ranking,
+            ["a", "a", "b", "b", "c"],
+            [10, 7, 11, 6, 8, 2, 7, 1],
+            target_recall=0.6,
+        )
+
+    def test_invalid_recalls(self) -> None:
+        self.assertRaises(
+            AssertionError,
+            evaluate_ranking,
+            ["a", "a", "b", "b"],
+            [10, 7, 11, 6],
+            target_recall=10.6,
+        )
+
+        self.assertRaises(
+            AssertionError,
+            evaluate_ranking,
+            ["a", "a", "b", "b"],
+            [10, 7, 11, 6],
+            target_recall=-0.0001,
+        )
+
+        self.assertRaises(
+            AssertionError,
+            evaluate_ranking,
+            ["a", "a", "b", "b"],
+            [10, 7, 11, 6],
+            target_recall=1.00001,
+        )
+
+    def test_empty(self) -> None:
+        evaluate_ranking(
+            [],
+            [],
+            target_recall=0.6,
+        )
+
+        evaluate_ranking(
+            ["a"],
+            [1],
+            target_recall=0.6,
+        )
+
+    def test_save(self) -> None:
+        path = Path("test.svg")
+        path.unlink(missing_ok=True)
+
+        evaluate_ranking(
+            ["a", "a", "b", "b", "c", "d", "d", "d"],
+            [10, 7, 11, 6, 8, 2, 7, 1],
+            target_recall=0.1,
+            output_svg=path,
+        )
+
+        self.assertTrue(path.exists())
+        path.unlink()

good_ai/tests/sus/test_get_sentences.py

@@ -0,0 +1,16 @@
+import unittest
+
+from src.sus.get_sentences import get_sentences
+
+
+class TestGetSentences(unittest.TestCase):
+    def test_default(self) -> None:
+        text = "This is a complete sentence. So is this. However this is n"  # ot.
+        expected = ["This is a complete sentence.", "So is this.", "However this is n"]
+
+        self.assertEqual(get_sentences(text), expected)
+        self.assertEqual(get_sentences(text, ignore_partial=True), expected[0:2])
+
+    def test_empty(self) -> None:
+        self.assertEqual(get_sentences(""), [])
+        self.assertEqual(get_sentences("", ignore_partial=True), [])

good_ai/tests/sus/test_language.py

@@ -0,0 +1,32 @@
+import unittest
+
+from src.sus.language import english_name_of_language, is_english, predict_language
+
+
+class TestLanguage(unittest.TestCase):
+    def test_predict_language(self) -> None:
+        self.assertEqual(predict_language("This is an English text."), "en")
+        self.assertEqual(predict_language("Ez egy magyar szöveg."), "hu")
+        self.assertEqual(predict_language("此處按原典，應為「黃武元年」，而電子稿此處為「黃初元年」。"), "zh-TW")
+        self.assertEqual(predict_language("32"), "und")
+        self.assertEqual(predict_language(""), "und")
+
+    def test_is_english(self) -> None:
+        self.assertTrue(is_english("en"))
+        self.assertTrue(is_english("en-US"))
+        self.assertFalse(is_english("hu"))
+        self.assertFalse(is_english("de"))
+        self.assertFalse(is_english("zh"))
+        self.assertFalse(is_english("zh-TW"))
+        self.assertFalse(is_english("und"))
+        self.assertFalse(is_english(""))
+        self.assertFalse(is_english(None))
+
+    def english_name_of_language(self) -> None:
+        self.assertEqual(english_name_of_language("en"), "English")
+        self.assertEqual(english_name_of_language("hu"), "Hungarian")
+        self.assertEqual(english_name_of_language("zh"), "Chinese")
+        self.assertEqual(english_name_of_language("zh-TW"), "Chinese")
+        self.assertEqual(english_name_of_language("und"), "Unknown language")
+        self.assertEqual(english_name_of_language(""), "Unknown language")
+        self.assertEqual(english_name_of_language(None), "Unknown language")

good_ai/tests/sus/test_lemmatize_text.py

@@ -0,0 +1,139 @@
+import unittest
+
+from src.sus.lemmatize_text import lemmatize_text
+
+
+class TestLemmatizeText(unittest.TestCase):
+    def test_simple(self) -> None:
+        text = "The state-of-the-art could not be improved, however we managed to create a less resource-intensive implementation of it."
+
+        lemmatized = [
+            "the",
+            "state",
+            "-",
+            "of",
+            "-",
+            "the",
+            "-",
+            "art",
+            "could",
+            "not",
+            "be",
+            "improve",
+            ",",
+            "however",
+            "we",
+            "manage",
+            "to",
+            "create",
+            "a",
+            "less",
+            "resource",
+            "-",
+            "intensive",
+            "implementation",
+            "of",
+            "it",
+            ".",
+        ]
+
+        lemmatized_pos = [
+            "the_DET",
+            "state_NOUN",
+            "-_PUNCT",
+            "of_ADP",
+            "-_PUNCT",
+            "the_DET",
+            "-_PUNCT",
+            "art_NOUN",
+            "could_AUX",
+            "not_PART",
+            "be_AUX",
+            "improve_VERB",
+            ",_PUNCT",
+            "however_ADV",
+            "we_PRON",
+            "manage_VERB",
+            "to_PART",
+            "create_VERB",
+            "a_DET",
+            "less_ADJ",
+            "resource_NOUN",
+            "-_PUNCT",
+            "intensive_ADJ",
+            "implementation_NOUN",
+            "of_ADP",
+            "it_PRON",
+            "._PUNCT",
+        ]
+
+        lemmatized_neg = [
+            "the",
+            "state",
+            "-",
+            "of",
+            "-",
+            "the",
+            "-",
+            "art",
+            "could",
+            "not",
+            "NOT_be",
+            "NOT_improve",
+            "NOT_,",
+            "however",
+            "we",
+            "manage",
+            "to",
+            "create",
+            "a",
+            "less",
+            "resource",
+            "-",
+            "intensive",
+            "implementation",
+            "of",
+            "it",
+            ".",
+        ]
+
+        lemmatized_pos_neg = [
+            "the_DET",
+            "state_NOUN",
+            "-_PUNCT",
+            "of_ADP",
+            "-_PUNCT",
+            "the_DET",
+            "-_PUNCT",
+            "art_NOUN",
+            "could_AUX",
+            "not_PART",
+            "NOT_be_AUX",
+            "NOT_improve_VERB",
+            "NOT_,_PUNCT",
+            "however_ADV",
+            "we_PRON",
+            "manage_VERB",
+            "to_PART",
+            "create_VERB",
+            "a_DET",
+            "less_ADJ",
+            "resource_NOUN",
+            "-_PUNCT",
+            "intensive_ADJ",
+            "implementation_NOUN",
+            "of_ADP",
+            "it_PRON",
+            "._PUNCT",
+        ]
+
+        self.assertEqual(lemmatize_text(text), lemmatized)
+        self.assertEqual(lemmatize_text(text, add_part_of_speech=True), lemmatized_pos)
+        self.assertEqual(lemmatize_text(text, add_negation=True), lemmatized_neg)
+        self.assertEqual(
+            lemmatize_text(text, add_negation=True, add_part_of_speech=True),
+            lemmatized_pos_neg,
+        )
+
+    def test_empty(self) -> None:
+        self.assertEqual(lemmatize_text(""), [])

good_ai/tests/sus/test_lemmatize_token.py

@@ -0,0 +1,20 @@
+import unittest
+
+from src.sus.lemmatize_text import lemmatize_token
+from src.sus.nlp import nlp
+
+
+class TestLemmatizeToken(unittest.TestCase):
+    def test_simple(self) -> None:
+        token = nlp("Center")[0]
+
+        self.assertEqual(lemmatize_token(token), "centre")
+        self.assertEqual(lemmatize_token(token, add_negation=True), "centre")
+        self.assertEqual(lemmatize_token(token, add_part_of_speech=True), "centre_NOUN")
+
+    def test_punctuation(self) -> None:
+        token = nlp("This.")[1]
+
+        self.assertEqual(lemmatize_token(token), ".")
+        self.assertEqual(lemmatize_token(token, add_negation=True), ".")
+        self.assertEqual(lemmatize_token(token, add_part_of_speech=True), "._PUNCT")

good_ai/tests/sus/test_match_names.py

@@ -0,0 +1,71 @@
+import unittest
+
+from src.sus.match_names.match_names import match_names
+
+
+class TestMatchNames(unittest.TestCase):
+    def test_grid(self) -> None:
+        names = [
+            ["Al-Nasiry, S.", "Al-Nasiry S"],
+            ["De Leo, A. (Andreina)"],
+            [
+                "DeBenedictis, J.N. (Julia)",
+                "DeBenedictis, N. (Julia)",
+                "DeBenedictis, J.",
+                "Julia DeBenedictis",
+            ],
+            ["Diesen, J.A.Y. van"],
+            ["Nio, C Yung"],
+            ["Uyen Chau Nguyen"],
+            ["Hagger M.S.", "Martin Hagger"],
+            ["Izabela-Cristina STANCU"],
+            ["Verborgh R.", "Ruben Verborgh"],
+            ["Droshout, D.T.G.G.M.L. (Dimitri)"],
+            ["El Demellawy"],
+            ["Sebastiaan van de Velde", "van de Velde, Sebastiaan"],
+            ["Sebastiaan Brand"],
+            ["Bertil FM Blok", "B.F.M. Blok"],
+            ["Bob Zadok Blok"],
+            ["Shannon Spruit"],
+            ["Shannon Kroes"],
+            [
+                "MSc Jérémie Decouchant",
+                "PhD, Jérémie Decouchant",
+                "PhD Jérémie Decouchant",
+                "Jérémie Decouchant",
+                "MD, PhD Jérémie Decouchant",
+            ],
+            ["Jeremie Gobeil"],
+            ["Wouters, B.B.R.E.F.", "Wouters B.B. R.  E.F."],
+            ["Elias Caldeira Dantas, A.M. (Aline)"],
+            ["Ed Harris"],
+            ["Ed Deprettere ", "    Ed   Deprettere "],
+            ["András Schmelczer"],
+            ["Enden, G. van den (Gitta)"],
+            ["Ruben van Dijk"],
+            ["Richard van Dijk"],
+            ["Xinping Guan", "X. Guan"],
+            ["Xiaohong Guan", "X. Guan"],
+            ["Pruimboom, Tim", "Pruimboom T."],
+            ["Sanchez-Faddeev H.", "Sanchez-Faddeev, Hernando"],
+            ["Duijnhoven - Jansen, E.M. van", "Emma M. van Jansen"],
+        ]
+
+        all_names = [n for t in names for n in t]
+
+        for n1 in all_names:
+            for n2 in all_names:
+                is_match = match_names(n1, n2)
+                groups = [t for t in names if n1 in t]
+                with self.subTest(n1=n1, n2=n2):
+                    if any(n2 in g for g in groups):
+                        self.assertTrue(is_match, "false negative")
+                    elif all(n2 not in g for g in groups):
+                        self.assertFalse(is_match, "false match")
+
+    def test_empty(self) -> None:
+        self.assertFalse(match_names("", ""))
+        self.assertFalse(match_names(None, ""))
+        self.assertFalse(match_names(None, None))
+        self.assertFalse(match_names("", None))
+        self.assertFalse(match_names("Oliver", None))