Add files

Signed-off-by: András Schmelczer <andras@schmelczer.dev>

good_ai/src/__init__.py

@@ -0,0 +1 @@
+

good_ai/src/open_s3/__init__.py

@@ -0,0 +1 @@
+from .large_file import LargeFile

good_ai/src/open_s3/__main__.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+
+import logging
+from pathlib import Path
+
+from large_file import LargeFile
+from parse_arguments import parse_arguments
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    parser, args = parse_arguments()
+
+    LargeFile.configure_credentials_from_file(args.secrets)
+
+    if not args.cache and not args.push and not args.delete:
+        logging.warn("No action required.")
+        parser.print_help()
+    try:
+        if args.cache:
+            for c in args.cache:
+                split = c.split(":")
+                file_name = split[0]
+                version = None if len(split) == 1 else int(split[1])
+                LargeFile(file_name, "r", version=version).get()
+
+        if args.push:
+            for p in args.push:
+                path = Path(p)
+                LargeFile(path.name, "w").push(path)
+
+        if args.delete:
+            for f in args.delete:
+                LargeFile(f).delete()
+    except Exception as e:
+        logging.error(e)

good_ai/src/open_s3/helper/__init__.py

@@ -0,0 +1,2 @@
+from .progress_bar import DownloadProgressBar, UploadProgressBar
+from .human_readable_to_byte import human_readable_to_byte

good_ai/src/open_s3/helper/human_readable_to_byte.py

@@ -0,0 +1,31 @@
+import re
+
+
+def human_readable_to_byte(size: str) -> int:
+    """Case is ignored, kb, kB, Kb, and KB are all treated as kilobyte."""
+
+    if size.strip() == "0":
+        return 0
+
+    possible_units = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
+    units_re = "|".join(possible_units)
+    regex = re.compile(
+        rf"""
+        \s*                         # trim
+        (?P<scalar>\d+(.\d+)?)      # get scalar, it might be a float
+        \s*                         # ignore optional whitespace
+        (?P<unit>{units_re})        # capture the unit
+    """,
+        flags=re.VERBOSE | re.IGNORECASE,
+    )
+
+    match = regex.match(size)
+    if not match:
+        raise ValueError(f'Could not find values in "{size}"')
+
+    results = match.groupdict()
+
+    scalar = float(results["scalar"])
+    idx = possible_units.index(results["unit"].upper())
+    factor = 1024 ** idx
+    return round(scalar * factor)

good_ai/src/open_s3/helper/progress_bar.py

@@ -0,0 +1,40 @@
+import os
+import threading
+from logging import Logger
+from typing import IO, Any, Optional
+
+from tqdm.auto import tqdm
+from pathlib import Path
+
+import os
+import sys
+import threading
+
+
+class ProgressBar:
+    def __init__(self, file_size: int, logger: Logger, prefix: str):
+        self._file_size = file_size
+        self._logger = logger
+        self._prefix=prefix
+
+        self._seen_so_far = 0
+        self._lock = threading.Lock()
+
+    def __call__(self, bytes_amount: int):
+        with self._lock:
+            self._seen_so_far += bytes_amount
+            percentage = (self._seen_so_far / float(self._file_size)) * 100
+            size_length = len(str(self._file_size))
+            progress = str(self._seen_so_far).rjust(size_length)
+            self._logger.info(f"{self._prefix} {progress}/{self._file_size} bytes ({percentage:.1f}%)")
+
+
+class DownloadProgressBar(ProgressBar):
+    def __init__(self, name: str, size: int, logger: Logger):
+        super().__init__(file_size=size, logger=logger, prefix=f'Downloading {name}')
+
+
+class UploadProgressBar(ProgressBar):
+    def __init__(self, path: Path, logger: Logger):
+        size = os.path.getsize(path)
+        super().__init__(file_size=size, logger=logger, prefix=f'Uploading {path.name}')

good_ai/src/open_s3/large_file.py

@@ -0,0 +1,346 @@
+import configparser
+import logging
+import os
+import shutil
+import tempfile
+from pathlib import Path
+from types import TracebackType
+from typing import IO, Any, Dict, List, Optional, Type, Union
+import boto3
+from botocore.exceptions import ClientError
+
+from helper import DownloadProgressBar, UploadProgressBar, human_readable_to_byte
+
+logger = logging.getLogger("open_s3")
+
+
+class LargeFile:
+    """
+    Store large files in S3. Use local cache for speed up.
+
+    Examples:
+
+    ```
+    with LargeFile("test.txt", "w", keep_last_n=3) as f:
+        for i in range(1000000):
+            f.write('test\n')
+
+    with LargeFile("test.txt", "r") as f:
+        print(f.readlines()[0])
+
+    path_to_cached_text_file = LargeFile("test.txt", version=0).get()
+    ```
+
+    By default, files are stored in the ".cache" folder and the
+    least recently use is deleted after the overall size reaches 30 GBs.
+
+    Change it with the following properties.
+
+    ```
+    LargeFile.cache_path = Path(".cache")
+    LargeFile.max_cache_size = "30GB"
+    ```
+    """
+
+    region_name = None
+    access_key_id = None
+    secret_access_key = None
+    bucket_name = None
+    endpoint_url = None
+
+    cache_path = Path(".cache")
+    max_cache_size: Optional[str] = "30GB"
+
+    def __init__(
+        self,
+        name: str,
+        mode: str = "r",
+        *,
+        buffering: int = -1,
+        encoding: Optional[str] = None,
+        errors: Optional[str] = None,
+        newline: Optional[str] = None,
+        version: Optional[int] = None,
+        keep_last_n: Optional[int] = None,
+        offline_mode: bool = False
+    ):
+        self._name: str = name
+        self._version = version
+        self._mode: str = mode
+        self._keep_last_n = keep_last_n
+        self._offline_mode = offline_mode
+
+        self._buffering = buffering
+        self._encoding = encoding
+        self._errors = errors
+        self._newline = newline
+
+        LargeFile.cache_path.mkdir(parents=True, exist_ok=True)
+
+        self._find_versions()
+        self._check_mode_and_set_version()
+        
+    @classmethod
+    def configure_credentials(
+        cls,
+        *,
+        aws_region_name: str,
+        aws_access_key_id: str,
+        aws_secret_access_key: str,
+        large_files_bucket_name: str,
+        endpoint_url: Optional[str] = None,
+        **_: Dict[str, Any],
+    ) -> None:
+        cls.region_name = aws_region_name
+        cls.access_key_id = aws_access_key_id
+        cls.secret_access_key = aws_secret_access_key
+        cls.bucket_name = large_files_bucket_name
+        cls.endpoint_url = endpoint_url
+
+    @classmethod
+    def configure_credentials_from_file(
+        cls,
+        secrets_path: Union[Path, str],
+    ) -> None:
+
+        if isinstance(secrets_path, str):
+            secrets_path = Path(secrets_path)
+
+        if not secrets_path.exists():
+            raise FileNotFoundError(secrets_path.resolve())
+
+        credentials = configparser.ConfigParser()
+        credentials.read(secrets_path)
+        credentials.default_section
+        cls.configure_credentials(**credentials[credentials.default_section])
+
+    def __enter__(self) -> IO:
+        self._file: IO[Any] = (
+            tempfile.NamedTemporaryFile(
+                mode=self._mode,
+                buffering=self._buffering,
+                encoding=self._encoding,
+                newline=self._newline,
+                errors=self._errors,
+                delete=False,
+                prefix="large-file-",
+            )
+            if "w" in self._mode
+            else open(
+                self.get(),
+                mode=self._mode,
+                buffering=self._buffering,
+                encoding=self._encoding,
+                newline=self._newline,
+                errors=self._errors,
+            )
+        )
+
+        return self._file
+
+    def __exit__(
+        self,
+        type: Optional[Type[BaseException]],
+        exc: Optional[BaseException],
+        traceback: Optional[TracebackType],
+    ) -> bool:
+        self._file.close()
+
+        if type is None:
+            if "w" in self._mode:
+                self.push(Path(self._file.name))
+                os.unlink(self._file.name)
+        else:
+            logger.exception("Could not finish operation.")
+
+        return True
+    
+    @property
+    def version_ids(self) -> List[int]:
+        return [self._get_version_from_key(key) for key in self._versions]
+
+    def get(self, hide_progress: bool = False) -> Path:
+        key = next(
+            key
+            for key in self._versions
+            if self._get_version_from_key(key) == self._version
+        )
+
+        destination = self.cache_path / self._local_name
+        if not destination.exists():
+            logger.info(
+                f"File {self._local_name} does not exist locally, starting download from S3"
+            )
+
+            with tempfile.TemporaryDirectory() as tmp:
+                tmp_file_archive = Path(tmp) / f"{self._local_name}.tar.gz"
+
+                size = self._client.head_object(Bucket=self.bucket_name, Key=key)['ContentLength']
+                self._client.download_file(Bucket=self.bucket_name, Key=key, Filename=str(tmp_file_archive), Callback=None if hide_progress else DownloadProgressBar(size=size, name=key, logger=logger))
+                logger.info(f"Decompressing {self._local_name}")
+                shutil.unpack_archive(str(tmp_file_archive), tmp, "gztar")
+                tmp_file = Path(tmp) / self._local_name
+                shutil.move(str(tmp_file), str(destination))
+        else:
+            logger.info(f"File {self._local_name} found in cache")
+
+        return destination
+
+    def push(self, path: Union[Path, str], hide_progress: bool = False) -> None:
+        if isinstance(path, str):
+            path = Path(path)
+
+        with tempfile.TemporaryDirectory() as tmp:
+            if path.is_file():
+                logger.info(f"Copying file for {self._local_name}")
+                copy = shutil.copy 
+            else:
+                logger.info(f"Copying directory for {self._local_name}")
+                copy = shutil.copytree
+
+            try:
+                # Make local copy in the cache
+                copy(str(path), str(self.cache_path / self._local_name))
+            except shutil.SameFileError:
+                pass  # No worries
+
+            copy(str(path), str(Path(tmp) / self._local_name))
+
+            logger.info(f"Compressing {self._local_name}")
+            shutil.make_archive(
+                str(Path(tmp) / self._local_name),
+                "gztar",
+                tmp,
+            )
+
+            logger.info(f"Uploading {self._local_name} to S3 from {path}")
+
+            file_to_be_uploaded = Path(tmp) / f"{self._local_name}.tar.gz"
+            self._client.upload_file(Filename=str(file_to_be_uploaded), Bucket=self.bucket_name, Key=self._s3_name, Callback=None if hide_progress else UploadProgressBar(file_to_be_uploaded, logger=logger))
+
+        self.clean_up()
+
+    def delete(self) -> None:
+        self._keep_last_n = 0
+        self._delete_old_versions_from_s3()
+
+    def clean_up(self) -> None:
+        self._delete_old_versions_from_s3()
+        self._delete_old_versions_from_disk()
+
+    def _create_client(self) -> None:
+        if (
+            self.region_name is None
+            or self.access_key_id is None
+            or self.secret_access_key is None
+            or self.bucket_name is None
+        ):
+            raise ValueError(
+                "Please configure the S3 access options by calling LargeFile.configure_credentials or set offline_mode=True in the constructor."
+            )
+
+        self._client = boto3.client('s3', aws_access_key_id=self.access_key_id, aws_secret_access_key=self.secret_access_key, region_name=self.region_name, endpoint_url=self.endpoint_url)
+        
+    def _find_versions(self) -> None:
+        if self._offline_mode:
+            self._fetch_versions_from_cache()
+        else:
+            self._create_client()
+            self._fetch_versions_from_s3()
+        
+        if self._versions:
+            logger.info(f"Found versions: {self.version_ids}")
+        else:
+            logger.info("No versions found")
+
+    def _fetch_versions_from_cache(self) -> None:
+        logger.info(f"Fetching offline versions of {self._name}")
+
+        self._versions = [
+            path
+            for path in self.cache_path.glob(f'{self._local_name}-*')
+        ]
+
+    def _fetch_versions_from_s3(self) -> None:
+        logger.info(f"Fetching online versions of {self._name}")
+        found_objects = self._client.list_objects_v2(
+            Bucket=self.bucket_name, Prefix=self._name
+        )
+        self._versions = (
+            sorted(
+                o["Key"]
+                for o in found_objects["Contents"]
+            )
+            if "Contents" in found_objects
+            else []
+        )
+
+    def _check_mode_and_set_version(self) -> None:
+        if "+" in self._mode:
+            raise ValueError("Read-write mode is not allowed.")
+
+        if "w" in self._mode:
+            if self._version is not None:
+                raise ValueError("Providing a version is not allowed in write mode.")
+
+            self._version = self.version_ids[-1] + 1 if self.version_ids else 0
+
+        elif "r" in self._mode:
+            if not self.version_ids:
+                raise FileNotFoundError(
+                    f"File {self._name} not found. No versions are available."
+                )
+
+            if self._version is None:
+                self._version = self.version_ids[-1]
+                logger.info(f"Lastest version of {self._local_name} is {self._version}")
+
+            elif self._version not in self.version_ids:
+                raise FileNotFoundError(
+                    f"File {self._name} not found with version {self._version}. Available versions: {self.version_ids}"
+                )
+        else:
+            raise ValueError("Unsupported file mode.")
+
+    @property
+    def _local_name(self) -> str:
+        return f"{self._name}-{self._version}"
+
+    @property
+    def _s3_name(self) -> str:
+        return f"{self._name}/{self._version}"
+        
+    @staticmethod
+    def _get_version_from_key(key: Union[str, Path]) -> int:
+        if isinstance(key, Path):
+            return int(key.name.split("-")[-1])
+        return int(key.split("/")[-1])
+
+
+    def _delete_old_versions_from_s3(self) -> None:
+        if self._keep_last_n is not None:
+            for key in (self._versions[: -self._keep_last_n] if self._keep_last_n > 0 else self._versions):
+                logger.info(
+                    f"Removing old version (keep_last_n={self._keep_last_n}): {key}"
+                )
+                self._client.delete_object(Bucket=self.bucket_name, Key=key)
+                
+    def _delete_old_versions_from_disk(self) -> None:
+        self.cache_path.mkdir(parents=True, exist_ok=True)
+
+        if self.max_cache_size is None:
+            return
+
+        allowed_size = human_readable_to_byte(self.max_cache_size)
+        assert allowed_size >= 0
+
+        least_recently_read = sorted(
+            [f for f in self.cache_path.glob("*")], key=lambda f: f.stat().st_atime
+        )
+
+        while sum(os.path.getsize(f) for f in least_recently_read) > allowed_size:
+            file = least_recently_read.pop(0)
+            logger.info(
+                f"Deleting file from cache to meet quota (max_cache_size={self.max_cache_size}): {file}"
+            )
+            os.unlink(file)

good_ai/src/open_s3/parse_arguments.py

@@ -0,0 +1,48 @@
+from argparse import ArgumentParser, Namespace
+from typing import Tuple
+
+
+def parse_arguments() -> Tuple[ArgumentParser, Namespace]:
+    parser = ArgumentParser(
+        description="Store and version large files in S3; open them like regular files. Caching included.",
+    )
+
+    parser.add_argument(
+        "-s",
+        "--secrets",
+        type=str,
+        help="path to an .ini configration file with your S3 credentials",
+        required=True,
+    )
+
+    parser.add_argument(
+        "-c",
+        "--cache",
+        nargs="+",
+        type=str,
+        help="download file into local cache, example: file_name:version",
+        required=False,
+    )
+
+    parser.add_argument(
+        "-p",
+        "--push",
+        nargs="+",
+        type=str,
+        help="push a local file into S3 and set it as the most recent version",
+        required=False,
+    )
+
+    parser.add_argument(
+        "-d",
+        "--delete",
+        nargs="+",
+        type=str,
+        help="delete every version of file from S3",
+        required=False,
+    )
+
+    parser.print_usage = parser.print_help  # type: ignore
+    args = parser.parse_args()
+
+    return parser, args

good_ai/src/sus.egg-info/PKG-INFO

@@ -0,0 +1,50 @@
+Metadata-Version: 2.1
+Name: sus
+Version: 0.0.3
+Summary: [S]coutinScience [u]tilitie[s]: reusable utilities for text processing
+Home-page: https://github.com/ScoutinScience/platform
+Author: ScoutinScience B.V.
+Author-email: andras@scoutinscience.com
+License: UNKNOWN
+Project-URL: Bug Tracker, https://github.com/ScoutinScience/platform/issues
+Platform: UNKNOWN
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+
+# **S**coutinScience **U**tilitie**S** for text processing [![Lint and test ScoutinScience utilities](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml/badge.svg)](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml)
+
+> amogus
+
+## Exports
+
+- [clean](src/sus/clean.py)
+- [unique](src/sus/unique.py)
+- [parallel_map](src/sus/parallel_map.py)
+- [match_names](src/sus/match_names/match_names.py)
+- [evaluate_ranking](src/sus/evaluate_ranking/evaluate_ranking.py)
+
+### Requires loading spacy model
+
+> This is automatic but will require some time.
+
+> Add this to the Dockerfile for caching the spaCy model:
+>
+> ```docker
+> RUN pip install --no-cache-dir en-core-web-lg@https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.2.0/en_core_web_lg-3.2.0-py3-none-any.whl
+> ```
+
+- [spacy model (nlp)](src/sus/nlp.py)
+- [get_sentences](src/sus/get_sentences.py)
+- [lemmatize_text](src/sus/lemmatize_text.py)
+- [lemmatize_token](src/sus/lemmatize_token.py)
+- [publication TEI](src/sus/publication_tei/publication_tei.py)
+
+## Development
+
+- Optional booleans must have a default value of `False`.
+- No imports in top-level `__init__.py`, in order to not load anything unnecessary automatically
+- Should only be updated through a PR
+
+

good_ai/src/sus.egg-info/SOURCES.txt

@@ -0,0 +1,78 @@
+.gitignore
+README.md
+pyproject.toml
+requirements.txt
+setup.cfg
+src/__init__.py
+src/sus/__init__.py
+src/sus/clean.py
+src/sus/get_sentences.py
+src/sus/lemmatize_text.py
+src/sus/lemmatize_token.py
+src/sus/nlp.py
+src/sus/parallel_map.py
+src/sus/unique.py
+src/sus.egg-info/PKG-INFO
+src/sus.egg-info/SOURCES.txt
+src/sus.egg-info/dependency_links.txt
+src/sus.egg-info/requires.txt
+src/sus.egg-info/top_level.txt
+src/sus/data/__init__.py
+src/sus/data/american_spellings.py
+src/sus/data/punctuations.py
+src/sus/evaluate_ranking/__init__.py
+src/sus/evaluate_ranking/draw_f1_iso_lines.py
+src/sus/evaluate_ranking/evaluate_ranking.py
+src/sus/external/__init__.py
+src/sus/external/negspacy/README.md
+src/sus/external/negspacy/__init__.py
+src/sus/external/negspacy/negation.py
+src/sus/external/negspacy/termsets.py
+src/sus/external/pylatexenc/README.md
+src/sus/external/pylatexenc/__init__.py
+src/sus/external/pylatexenc/_util.py
+src/sus/external/pylatexenc/version.py
+src/sus/external/pylatexenc/latex2text/__init__.py
+src/sus/external/pylatexenc/latex2text/__main__.py
+src/sus/external/pylatexenc/latex2text/_defaultspecs.py
+src/sus/external/pylatexenc/latexencode/__init__.py
+src/sus/external/pylatexenc/latexencode/__main__.py
+src/sus/external/pylatexenc/latexencode/_partial_latex_encoder.py
+src/sus/external/pylatexenc/latexencode/_uni2latexmap.py
+src/sus/external/pylatexenc/latexencode/_uni2latexmap_xml.py
+src/sus/external/pylatexenc/latexencode/_unicode_to_latex_encoder.py
+src/sus/external/pylatexenc/latexwalker/__init__.py
+src/sus/external/pylatexenc/latexwalker/__main__.py
+src/sus/external/pylatexenc/latexwalker/_defaultspecs.py
+src/sus/external/pylatexenc/macrospec/__init__.py
+src/sus/external/pylatexenc/macrospec/_argparsers.py
+src/sus/language/__init__.py
+src/sus/language/english_name_of_language.py
+src/sus/language/is_english.py
+src/sus/language/predict_language.py
+src/sus/match_names/__init__.py
+src/sus/match_names/config.py
+src/sus/match_names/match_names.py
+src/sus/match_names/name_parts.py
+src/sus/publication_tei/__init__.py
+src/sus/publication_tei/publication_tei.py
+src/sus/publication_tei/models/__init__.py
+src/sus/publication_tei/models/affiliation.py
+src/sus/publication_tei/models/author.py
+src/sus/publication_tei/models/element.py
+src/sus/publication_tei/models/publication_metadata.py
+src/sus/publication_tei/models/text.py
+tests/__init__.py
+tests/test_clean.py
+tests/test_evaluate_ranking.py
+tests/test_get_sentences.py
+tests/test_language.py
+tests/test_lemmatize_text.py
+tests/test_lemmatize_token.py
+tests/test_match_names.py
+tests/test_parallel_map.py
+tests/test_publication_tei.py
+tests/test_unique.py
+tests/data/10.1136_bmjspcare-2021-003026.pdf.tei.xml
+tests/data/bad.tei.xml
+tests/data/parsed.py

good_ai/src/sus.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

good_ai/src/sus.egg-info/requires.txt

@@ -0,0 +1,14 @@
+click<8.1.0
+unidecode>=1.3.0
+multiprocess>=0.70.0.0
+tqdm>=4.0.0
+psutil>=5.9.0
+beautifulsoup4>=4.10.0
+lxml>=4.6.0
+spacy>=3.2.0
+pydantic>=1.8.0
+scikit-learn>=1.0.0
+matplotlib>=3.5.0
+numpy>=1.22.0
+langcodes[data]>=3.3.0
+langdetect>=1.0.9

good_ai/src/sus.egg-info/top_level.txt

@@ -0,0 +1 @@
+sus

good_ai/src/sus/clean.py

@@ -0,0 +1,64 @@
+import html
+import logging
+import re
+import unicodedata
+
+import unidecode
+
+from .data import left_regular_punctuations, right_regular_punctuations
+from .external.pylatexenc.latex2text import LatexNodes2Text
+
+logger = logging.getLogger("clean")
+latex = LatexNodes2Text()
+
+
+joined_left_punctuations = "".join(left_regular_punctuations).replace("]", r"\]")
+joined_right_punctuations = "".join(right_regular_punctuations).replace("[", r"\[")
+
+
+def clean(
+    text: str,
+    ignore_xml: bool = False,
+    ignore_latex: bool = False,
+    remove_brackets: bool = False,
+    convert_to_ascii: bool = False,
+) -> str:
+    if not ignore_xml:
+        text = re.sub(r"<[^>]*>", " ", text)
+        text = html.unescape(text)
+
+    if not ignore_latex:
+        text = text.replace("%", "\\%")  # escape LaTeX comments before parsing as LaTeX
+
+        try:
+            text = latex.latex_to_text(text, tolerant_parsing=True, strict_braces=False)
+            text = text.replace("%s", " ")
+        except:
+            logger.exception("Latex parsing error")
+
+    if convert_to_ascii:
+        text = unicodedata.normalize("NFKD", text)
+
+        try:
+            text.encode("ASCII", errors="strict")
+        except UnicodeEncodeError:
+            text = "".join([c for c in text if not unicodedata.combining(c)])
+            text = unidecode.unidecode(text)
+
+    if remove_brackets:
+        text = re.sub(r"\[[^\]]*\]", " ", text)
+
+    # fix hypens: break- word => break-word
+    text = re.sub(r"(\S)-\s+", r"\1-", text)
+    text = re.sub(r"\s+-(\S)", r"-\1", text)
+
+    # collapse whitespace
+    text = re.sub(r"\s+", " ", text)
+
+    # fix punctuation
+    text = re.sub(rf" ([{joined_left_punctuations}])", r"\1", text)
+    text = re.sub(rf"([{joined_right_punctuations}]) ", r"\1", text)
+
+    text = text.strip()
+
+    return text

good_ai/src/sus/data/__init__.py

@@ -0,0 +1,6 @@
+from .american_spellings import american_spellings
+from .punctuations import (
+    left_regular_punctuations,
+    right_regular_punctuations,
+    sentence_ending_punctuations,
+)

good_ai/src/sus/data/punctuations.py

@@ -0,0 +1,22 @@
+sentence_ending_punctuations = [".", "?", "!", ":", ";"]
+
+# punctuations that usually have no space between them and the character to their left
+left_regular_punctuations = [
+    *sentence_ending_punctuations,
+    ",",
+    "'",
+    "%",
+    ")",
+    "}",
+    "]",
+]
+
+# punctuations that usually have no space between them and the character to their right
+right_regular_punctuations = [
+    "(",
+    "{",
+    "[",
+    "$",
+    "€",
+    "#",
+]

good_ai/src/sus/evaluate_ranking/__init__.py

@@ -0,0 +1,2 @@
+from .draw_f1_iso_lines import draw_f1_iso_lines
+from .evaluate_ranking import evaluate_ranking

good_ai/src/sus/evaluate_ranking/draw_f1_iso_lines.py

@@ -0,0 +1,28 @@
+from typing import Optional
+
+import matplotlib.pyplot as plt
+import numpy as np
+from matplotlib.axes import Axes
+
+
+def draw_f1_iso_lines(
+    resolution: int = 1000,
+    min: float = 0.2,
+    max: float = 0.8,
+    steps: int = 4,
+    axes: Optional[Axes] = None,
+) -> None:
+    if axes is None:
+        axes = plt.axes()
+
+    for f_score in np.linspace(min, max, num=steps):
+        x = np.linspace(f_score / (2 - f_score), 1, num=resolution)
+        y = f_score * x / (2 * x - f_score)
+
+        axes.plot(x[y >= 0], y[y >= 0], color="gray", alpha=0.2)
+
+        axes.annotate(
+            f"f1={f_score:0.1f}",
+            backgroundcolor="w",
+            xy=(0.9, y[int(resolution * 0.9)] + 0.02),
+        )

good_ai/src/sus/evaluate_ranking/evaluate_ranking.py

@@ -0,0 +1,90 @@
+from pathlib import Path
+from typing import Dict, List, Optional, Union
+
+import matplotlib
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.metrics import average_precision_score, precision_recall_curve
+
+from ..unique import unique
+from .draw_f1_iso_lines import draw_f1_iso_lines
+
+
+def evaluate_ranking(
+    expected: List[Union[str, float]],
+    actual_scores: List[float],
+    target_recall: float,
+    title: Optional[str] = "",
+    disable_interpolation: bool = False,
+    axes: Optional[plt.Axes] = None,
+    output_svg: Optional[Path] = None,
+    reverse_order: bool = False,
+    plot: bool = True,
+) -> Dict[Union[str, float], float]:
+    assert 0 <= target_recall <= 1
+
+    if plot and axes is None:
+        fig = plt.figure(figsize=(20, 10))
+        fig.patch.set_facecolor("white")
+        ax = plt.axes()
+    else:
+        ax = axes
+
+    classes = sorted(unique(expected), reverse=reverse_order)
+    str_classes = [str(c) for c in classes]
+
+    with matplotlib.rc_context({"font.size": 20}):
+        if plot:
+            ax.set_xmargin(0)
+
+            draw_f1_iso_lines(axes=ax)
+
+        results: Dict[Union[str, float], float] = {}
+        for i in range(len(classes) - 1):
+            binarized_expected = [
+                (v < classes[i]) if reverse_order else (v > classes[i])
+                for v in expected
+            ]
+            precision, recall, _ = precision_recall_curve(
+                binarized_expected, actual_scores
+            )
+
+            if not disable_interpolation:
+                for j in range(1, len(precision)):
+                    precision[j] = max(precision[j - 1], precision[j])
+
+            closest_recall_index = np.argmin(np.abs(recall - target_recall))
+            precision_at_closest_recall = precision[closest_recall_index]
+            average_precision = average_precision_score(
+                binarized_expected, actual_scores
+            )
+            results[classes[i]] = precision_at_closest_recall
+
+            if plot:
+                ax.plot(
+                    recall,
+                    precision,
+                    label=f"{'|'.join(str_classes[:i + 1])} ↔ {'|'.join(str_classes[i+1:])} (P@{target_recall:.2f}={precision_at_closest_recall:.2f}, AP={average_precision:.2f})",
+                )
+
+        if plot:
+            ax.legend(loc="upper right")
+            ax.axvline(x=target_recall, linestyle="--", color="#55c6bb", linewidth=2.0)
+
+            if title is None:
+                title = "Ranking evaluation"
+
+            ax.set_title(f'{title} ({" < ".join(str_classes)})', pad=20)
+
+            ax.set_xlabel("Recall")
+            ax.set_ylabel("Precision")
+
+            ax.set_xticks([target_recall] + sorted(ax.get_xticks()))
+
+        if plot and output_svg is None:
+            if axes is None:
+                plt.show()
+        elif output_svg:
+            plt.savefig(output_svg, format="svg")
+
+    return results

good_ai/src/sus/external/negspacy/README.md

@@ -0,0 +1 @@
+https://github.com/jenojp/negspacy

good_ai/src/sus/external/negspacy/negation.py

@@ -0,0 +1,222 @@
+import logging
+
+from spacy.language import Language
+from spacy.matcher import PhraseMatcher
+from spacy.tokens import Token
+
+from .termsets import termset
+
+default_ts = termset("en").get_patterns()
+
+
+@Language.factory(
+    "negex",
+    default_config={
+        "neg_termset": default_ts,
+        "extension_name": "negex",
+        "chunk_prefix": list(),
+    },
+)
+class Negex:
+    """
+        A spaCy pipeline component which identifies negated tokens in text.
+
+        Based on: NegEx - A Simple Algorithm for Identifying Negated Findings and Diseasesin Discharge Summaries
+    Chapman, Bridewell, Hanbury, Cooper, Buchanan
+
+    Parameters
+    ----------
+    nlp: object
+        spaCy language object
+    termset_lang: str
+        language code, if using default termsets (e.g. "en" for english)
+    extension_name: str
+        defaults to "negex"; whether entity is negated is then available as ent._.negex
+    pseudo_negations: list
+        list of phrases that cancel out a negation, if empty, defaults are used
+    preceding_negations: list
+        negations that appear before an entity, if empty, defaults are used
+    following_negations: list
+        negations that appear after an entity, if empty, defaults are used
+    termination: list
+        phrases that "terminate" a sentence for processing purposes such as "but". If empty, defaults are used
+
+    """
+
+    def __init__(
+        self,
+        nlp: Language,
+        name: str,
+        neg_termset: dict,
+        extension_name: str,
+        chunk_prefix: list,
+    ):
+        if not Token.has_extension(extension_name):
+            Token.set_extension(extension_name, default=False, force=True)
+
+        ts = neg_termset
+        expected_keys = [
+            "pseudo_negations",
+            "preceding_negations",
+            "following_negations",
+            "termination",
+        ]
+        if not set(ts.keys()) == set(expected_keys):
+            raise KeyError(
+                f"Unexpected or missing keys in 'neg_termset', expected: {expected_keys}, instead got: {list(ts.keys())}"
+            )
+
+        self.pseudo_negations = ts["pseudo_negations"]
+        self.preceding_negations = ts["preceding_negations"]
+        self.following_negations = ts["following_negations"]
+        self.termination = ts["termination"]
+
+        self.nlp = nlp
+        self.extension_name = extension_name
+        self.build_patterns()
+        self.chunk_prefix = list(nlp.tokenizer.pipe(chunk_prefix))
+
+    def build_patterns(self):
+        # efficiently build spaCy matcher patterns
+        self.matcher = PhraseMatcher(self.nlp.vocab, attr="LOWER")
+
+        self.pseudo_patterns = list(self.nlp.tokenizer.pipe(self.pseudo_negations))
+        self.matcher.add("pseudo", None, *self.pseudo_patterns)
+
+        self.preceding_patterns = list(
+            self.nlp.tokenizer.pipe(self.preceding_negations)
+        )
+        self.matcher.add("Preceding", None, *self.preceding_patterns)
+
+        self.following_patterns = list(
+            self.nlp.tokenizer.pipe(self.following_negations)
+        )
+        self.matcher.add("Following", None, *self.following_patterns)
+
+        self.termination_patterns = list(self.nlp.tokenizer.pipe(self.termination))
+        self.matcher.add("Termination", None, *self.termination_patterns)
+
+    def process_negations(self, doc):
+        """
+        Find negations in doc and clean candidate negations to remove pseudo negations
+
+        Parameters
+        ----------
+        doc: object
+            spaCy Doc object
+
+        Returns
+        -------
+        preceding: list
+            list of tuples for preceding negations
+        following: list
+            list of tuples for following negations
+        terminating: list
+            list of tuples of terminating phrases
+
+        """
+        ###
+        # does not work properly in spacy 2.1.8. Will incorporate after 2.2.
+        # Relying on user to use NER in meantime
+        # see https://github.com/jenojp/negspacy/issues/7
+        ###
+        # if not doc.is_nered:
+        #     raise ValueError(
+        #         "Negations are evaluated for Named Entities found in text. "
+        #         "Your SpaCy pipeline does not included Named Entity resolution. "
+        #         "Please ensure it is enabled or choose a different language model that includes it."
+        #     )
+        preceding = list()
+        following = list()
+        terminating = list()
+
+        matches = self.matcher(doc)
+        pseudo = [
+            (match_id, start, end)
+            for match_id, start, end in matches
+            if self.nlp.vocab.strings[match_id] == "pseudo"
+        ]
+
+        for match_id, start, end in matches:
+            if self.nlp.vocab.strings[match_id] == "pseudo":
+                continue
+            pseudo_flag = False
+            for p in pseudo:
+                if start >= p[1] and start <= p[2]:
+                    pseudo_flag = True
+                    continue
+            if not pseudo_flag:
+                if self.nlp.vocab.strings[match_id] == "Preceding":
+                    preceding.append((match_id, start, end))
+                elif self.nlp.vocab.strings[match_id] == "Following":
+                    following.append((match_id, start, end))
+                elif self.nlp.vocab.strings[match_id] == "Termination":
+                    terminating.append((match_id, start, end))
+                else:
+                    logging.warnings(
+                        f"phrase {doc[start:end].text} not in one of the expected matcher types."
+                    )
+        return preceding, following, terminating
+
+    def termination_boundaries(self, doc, terminating):
+        """
+        Create sub sentences based on terminations found in text.
+
+        Parameters
+        ----------
+        doc: object
+            spaCy Doc object
+        terminating: list
+            list of tuples with (match_id, start, end)
+
+        returns
+        -------
+        boundaries: list
+            list of tuples with (start, end) of spans
+
+        """
+        sent_starts = [sent.start for sent in doc.sents]
+        terminating_starts = [t[1] for t in terminating]
+        starts = sent_starts + terminating_starts + [len(doc)]
+        starts.sort()
+        boundaries = list()
+        index = 0
+        for i, start in enumerate(starts):
+            if not i == 0:
+                boundaries.append((index, start))
+            index = start
+        return boundaries
+
+    def negex(self, doc):
+        """
+        Negates entities of interest
+
+        Parameters
+        ----------
+        doc: object
+            spaCy Doc object
+
+        """
+        preceding, following, terminating = self.process_negations(doc)
+        boundaries = self.termination_boundaries(doc, terminating)
+        for b in boundaries:
+            sub_preceding = [i for i in preceding if b[0] <= i[1] < b[1]]
+            sub_following = [i for i in following if b[0] <= i[1] < b[1]]
+
+            for e in doc[b[0] : b[1]]:
+                if any(pre < e.i for pre in [i[1] for i in sub_preceding]):
+                    e._.set(self.extension_name, True)
+                    continue
+                if any(fol > e.i for fol in [i[2] for i in sub_following]):
+                    e._.set(self.extension_name, True)
+                    continue
+                if self.chunk_prefix:
+                    if any(
+                        e.text.lower().startswith(c.text.lower())
+                        for c in self.chunk_prefix
+                    ):
+                        e._.set(self.extension_name, True)
+        return doc
+
+    def __call__(self, doc):
+        return self.negex(doc)

good_ai/src/sus/external/negspacy/termsets.py

@@ -0,0 +1,229 @@
+"""
+Default termsets for various languages
+"""
+
+LANGUAGES = dict()
+
+# english termset dictionary
+en = dict()
+pseudo = [
+    "no further",
+    "not able to be",
+    "not certain if",
+    "not certain whether",
+    "not necessarily",
+    "without any further",
+    "without difficulty",
+    "without further",
+    "might not",
+    "not only",
+    "no increase",
+    "no significant change",
+    "no change",
+    "no definite change",
+    "not extend",
+    "not cause",
+]
+en["pseudo_negations"] = pseudo
+
+preceding = [
+    "absence of",
+    "declined",
+    "denied",
+    "denies",
+    "denying",
+    "no sign of",
+    "no signs of",
+    "not",
+    "not demonstrate",
+    "symptoms atypical",
+    "doubt",
+    "negative for",
+    "no",
+    "versus",
+    "without",
+    "doesn't",
+    "doesnt",
+    "don't",
+    "dont",
+    "didn't",
+    "didnt",
+    "wasn't",
+    "wasnt",
+    "weren't",
+    "werent",
+    "isn't",
+    "isnt",
+    "aren't",
+    "arent",
+    "cannot",
+    "can't",
+    "cant",
+    "couldn't",
+    "couldnt",
+    "never",
+]
+en["preceding_negations"] = preceding
+
+following = [
+    "declined",
+    "unlikely",
+    "was not",
+    "were not",
+    "wasn't",
+    "wasnt",
+    "weren't",
+    "werent",
+]
+en["following_negations"] = following
+
+termination = [
+    "although",
+    "apart from",
+    "as there are",
+    "aside from",
+    "but",
+    "except",
+    "however",
+    "involving",
+    "nevertheless",
+    "still",
+    "though",
+    "which",
+    "yet",
+]
+en["termination"] = termination
+
+LANGUAGES["en"] = en
+
+# en_clinical builds upon en
+en_clinical = dict()
+pseudo_clinical = pseudo + [
+    "gram negative",
+    "not rule out",
+    "not ruled out",
+    "not been ruled out",
+    "not drain",
+    "no suspicious change",
+    "no interval change",
+    "no significant interval change",
+]
+en_clinical["pseudo_negations"] = pseudo_clinical
+
+preceding_clinical = preceding + [
+    "patient was not",
+    "without indication of",
+    "without sign of",
+    "without signs of",
+    "without any reactions or signs of",
+    "no complaints of",
+    "no evidence of",
+    "no cause of",
+    "evaluate for",
+    "fails to reveal",
+    "free of",
+    "never developed",
+    "never had",
+    "did not exhibit",
+    "rules out",
+    "rule out",
+    "rule him out",
+    "rule her out",
+    "rule patient out",
+    "rule the patient out",
+    "ruled out",
+    "ruled him out",
+    "ruled her out",
+    "ruled patient out",
+    "ruled the patient out",
+    "r/o",
+    "ro",
+]
+en_clinical["preceding_negations"] = preceding_clinical
+
+following_clinical = following + ["was ruled out", "were ruled out", "free"]
+en_clinical["following_negations"] = following_clinical
+
+termination_clinical = termination + [
+    "cause for",
+    "cause of",
+    "causes for",
+    "causes of",
+    "etiology for",
+    "etiology of",
+    "origin for",
+    "origin of",
+    "origins for",
+    "origins of",
+    "other possibilities of",
+    "reason for",
+    "reason of",
+    "reasons for",
+    "reasons of",
+    "secondary to",
+    "source for",
+    "source of",
+    "sources for",
+    "sources of",
+    "trigger event for",
+]
+en_clinical["termination"] = termination_clinical
+LANGUAGES["en_clinical"] = en_clinical
+
+en_clinical_sensitive = dict()
+
+preceding_clinical_sensitive = preceding_clinical + [
+    "concern for",
+    "supposed",
+    "which causes",
+    "leads to",
+    "h/o",
+    "history of",
+    "instead of",
+    "if you experience",
+    "if you get",
+    "teaching the patient",
+    "taught the patient",
+    "teach the patient",
+    "educated the patient",
+    "educate the patient",
+    "educating the patient",
+    "monitored for",
+    "monitor for",
+    "test for",
+    "tested for",
+]
+en_clinical_sensitive["pseudo_negations"] = pseudo_clinical
+en_clinical_sensitive["preceding_negations"] = preceding_clinical_sensitive
+en_clinical_sensitive["following_negations"] = following_clinical
+en_clinical_sensitive["termination"] = termination_clinical
+
+LANGUAGES["en_clinical_sensitive"] = en_clinical_sensitive
+
+
+class termset:
+    def __init__(self, termset_lang):
+        self.pattern_types = [
+            "pseudo_negations",
+            "preceding_negations",
+            "following_negations",
+            "termination",
+        ]
+        self.terms = LANGUAGES[termset_lang]
+
+    def get_patterns(self):
+        return self.terms
+
+    def remove_patterns(self, pattern_dict):
+        for key, value in pattern_dict.items():
+            if key in self.pattern_types:
+                self.terms[key] = [i for i in self.terms[key] if i not in value]
+            else:
+                raise ValueError(f"Unexpected key: {key} not in {self.pattern_types}")
+
+    def add_patterns(self, pattern_dict):
+        for key, value in pattern_dict.items():
+            if key in self.pattern_types:
+                self.terms[key] = list(set(self.terms[key] + value))
+            else:
+                raise ValueError(f"Unexpected key: {key} not in {self.pattern_types}")

good_ai/src/sus/external/pylatexenc/README.md

@@ -0,0 +1 @@
+https://github.com/phfaist/pylatexenc

good_ai/src/sus/external/pylatexenc/__init__.py

@@ -0,0 +1,37 @@
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2015 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+
+"""
+Utilities for LaTeX to/from Unicode Text conversion.
+
+Main Site:
+
+    https://github.com/phfaist/pylatexenc/
+
+"""
+
+from .version import version_str as _version_str
+
+__version__ = _version_str

good_ai/src/sus/external/pylatexenc/_util.py

@@ -0,0 +1,161 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2019 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+
+# Internal module. Internal API may move, disappear or otherwise change at any
+# time and without notice.
+
+
+try:
+    # Python >= 3.3
+    from collections.abc import MutableMapping
+except ImportError:
+    from collections import MutableMapping
+
+import bisect
+import warnings
+
+# ------------------------------------------------------------------------------
+
+
+def pylatexenc_deprecated_ver(ver, msg, stacklevel=2):
+    warnings.warn(
+        "Deprecated (pylatexenc {}): {} ".format(ver, msg.strip()),
+        DeprecationWarning,
+        stacklevel=stacklevel + 1,
+    )
+
+
+def pylatexenc_deprecated_2(msg, stacklevel=2):
+    warnings.warn(
+        (
+            "Deprecated (pylatexenc 2.0): {} "
+            "[see https://pylatexenc.readthedocs.io/en/latest/new-in-pylatexenc-2/]"
+        ).format(msg.strip()),
+        DeprecationWarning,
+        stacklevel=stacklevel + 1,
+    )
+
+
+# ------------------------------------------------------------------------------
+
+
+class LazyDict(MutableMapping):
+    r"""
+    A lazy dictionary that loads its data when it is first queried.
+
+    This is used to store the legacy
+    :py:data:`pylatexenc.latexwalker.default_macro_dict` as well as
+    :py:data:`pylatexenc.latex2text.default_macro_dict` etc.  Such that these
+    "dictionaries" are still exposed at the module-level, but the data is loaded
+    only if they are actually queried.
+    """
+
+    def __init__(self, generate_dict_fn):
+        self._full_dict = None
+        self._generate_dict_fn = generate_dict_fn
+
+    def _ensure_instance(self):
+        if self._full_dict is not None:
+            return
+        self._full_dict = self._generate_dict_fn()
+
+    def __getitem__(self, key):
+        self._ensure_instance()
+        return self._full_dict.__getitem__(key)
+
+    def __setitem__(self, key, val):
+        self._ensure_instance()
+        return self._full_dict.__setitem__(key, val)
+
+    def __delitem__(self, key):
+        self._ensure_instance()
+        return self._full_dict.__delitem__(key)
+
+    def __iter__(self):
+        self._ensure_instance()
+        return iter(self._full_dict)
+
+    def __len__(self):
+        self._ensure_instance()
+        return len(self._full_dict)
+
+    def copy(self):
+        self._ensure_instance()
+        return self._full_dict.copy()
+
+    def clear(self):
+        self._ensure_instance()
+        return self._full_dict.clear()
+
+
+# ------------------------------------------------------------------------------
+
+
+class LineNumbersCalculator(object):
+    r"""
+    Utility to calculate line numbers.
+    """
+
+    def __init__(self, s):
+        super(LineNumbersCalculator, self).__init__()
+
+        def find_all_new_lines(x):
+            # first line starts at the beginning of the string
+            yield 0
+            k = 0
+            while k < len(x):
+                k = x.find("\n", k)
+                if k == -1:
+                    return
+                k += 1
+                # s[k] is the character after the newline, i.e., the 0-th column
+                # of the new line
+                yield k
+
+        self._pos_new_lines = list(find_all_new_lines(s))
+
+    def pos_to_lineno_colno(self, pos, as_dict=False):
+        r"""
+        Return the line and column number corresponding to the given `pos`.
+
+        Return a tuple `(lineno, colno)` giving line number and column number.
+        Line numbers start at 1 and column number start at zero, i.e., the
+        beginning of the document (`pos=0`) has line and column number `(1,0)`.
+        If `as_dict=True`, then a dictionary with keys 'lineno', 'colno' is
+        returned instead of a tuple.
+        """
+
+        # find line number in list
+
+        # line_no is the index of the last item in self._pos_new_lines that is <= pos.
+        line_no = bisect.bisect_right(self._pos_new_lines, pos) - 1
+        assert line_no >= 0 and line_no < len(self._pos_new_lines)
+
+        col_no = pos - self._pos_new_lines[line_no]
+        # 1+... so that line and column numbers start at 1
+        if as_dict:
+            return {"lineno": 1 + line_no, "colno": col_no}
+        return (1 + line_no, col_no)

good_ai/src/sus/external/pylatexenc/latex2text/__main__.py

@@ -0,0 +1,325 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2018 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+import argparse
+import fileinput
+import logging
+import sys
+
+from .. import latexwalker
+from ..latex2text import LatexNodes2Text, _strict_latex_spaces_predef
+from ..version import version_str
+
+
+def main(argv=None):
+
+    if argv is None:
+        argv = sys.argv[1:]
+
+    parser = argparse.ArgumentParser(prog="latex2text", add_help=False)
+
+    codegroup = parser.add_argument_group("Input options")
+
+    codegroup.add_argument(
+        "--code",
+        "-c",
+        action="store",
+        default=None,
+        metavar="LATEX_CODE",
+        help="Convert the given LATEX_CODE to unicode text instead of reading "
+        "from FILE or standard input.  You cannot specify FILEs if you use this "
+        "option, and any standard input is ignored.",
+    )
+
+    codegroup.add_argument(
+        "files",
+        metavar="FILE",
+        nargs="*",
+        help="Input files to read LaTeX code from. If no FILE(s) is/are specified, "
+        "LaTeX code is read from standard input unless --code is specified",
+    )
+
+    group = parser.add_argument_group("LatexWalker options")
+
+    group.add_argument(
+        "--parser-keep-inline-math",
+        action="store_const",
+        const=True,
+        dest="parser_keep_inline_math",
+        default=None,
+        help=argparse.SUPPRESS,
+    )
+    group.add_argument(
+        "--no-parser-keep-inline-math",
+        action="store_const",
+        const=False,
+        dest="parser_keep_inline_math",
+        help=argparse.SUPPRESS,
+    )
+
+    group.add_argument(
+        "--tolerant-parsing",
+        action="store_const",
+        const=True,
+        dest="tolerant_parsing",
+        default=True,
+    )
+    group.add_argument(
+        "--no-tolerant-parsing",
+        action="store_const",
+        const=False,
+        dest="tolerant_parsing",
+        help="Tolerate syntax errors when parsing, and attempt to continue (default yes)",
+    )
+
+    # I'm not sure this flag is useful and if it should be exposed at all.
+    # Accept it, but make it hidden.
+    parser.add_argument(
+        "--strict-braces",
+        action="store_const",
+        const=True,
+        dest="strict_braces",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
+    parser.add_argument(
+        "--no-strict-braces",
+        action="store_const",
+        const=False,
+        dest="strict_braces",
+        # help="Report errors for mismatching LaTeX braces (default no)"
+        help=argparse.SUPPRESS,
+    )
+
+    group = parser.add_argument_group("LatexNodes2Text options")
+
+    group.add_argument(
+        "--text-keep-inline-math",
+        action="store_const",
+        const=True,
+        dest="text_keep_inline_math",
+        default=None,
+        help=argparse.SUPPRESS,
+    )
+    group.add_argument(
+        "--no-text-keep-inline-math",
+        action="store_const",
+        const=False,
+        dest="text_keep_inline_math",
+        help=argparse.SUPPRESS,
+    )
+
+    group.add_argument(
+        "--math-mode",
+        action="store",
+        dest="math_mode",
+        choices=["text", "with-delimiters", "verbatim", "remove"],
+        default="text",
+        help="How to handle chunks of math mode LaTeX code. 'text' = convert "
+        "to text like the rest; 'with-delimiters' = same as 'text' but retain "
+        "the original math mode delimiters; 'verbatim' = keep verbatim LaTeX code; "
+        "'remove' = remove from input entirely",
+    )
+
+    group.add_argument(
+        "--fill-text",
+        dest="fill_text",
+        action="store",
+        nargs="?",
+        default=-1,
+        help="Attempt to wrap text to the given width, or 80 columns if option is "
+        "specified with no argument",
+    )
+
+    group.add_argument(
+        "--keep-comments",
+        action="store_const",
+        const=True,
+        dest="keep_comments",
+        default=False,
+    )
+    group.add_argument(
+        "--no-keep-comments",
+        action="store_const",
+        const=False,
+        dest="keep_comments",
+        help="Keep LaTeX comments in text output (default no)",
+    )
+
+    class ListWithHiddenItems(list):
+        def __init__(self, thelist, hiddenitems):
+            super(ListWithHiddenItems, self).__init__(thelist)
+            self.hiddenitems = hiddenitems
+
+        def __contains__(self, value):
+            return (
+                super(ListWithHiddenItems, self).__contains__(value)
+                or value in self.hiddenitems
+            )
+
+    strict_latex_spaces_choices = ListWithHiddenItems(
+        # the list
+        ["off", "on"]
+        + list(k for k in _strict_latex_spaces_predef.keys() if k != "default"),
+        # hidden items: Value is accepted, but not shown in list of choices
+        ["default"],
+    )
+    group.add_argument(
+        "--strict-latex-spaces",
+        choices=strict_latex_spaces_choices,
+        dest="strict_latex_spaces",
+        default="macros",
+        help="How to handle whitespace. See documentation for the class "
+        "LatexNodes2Text().",
+    )
+
+    group.add_argument(
+        "--keep-braced-groups",
+        action="store_const",
+        const=True,
+        dest="keep_braced_groups",
+        default=False,
+    )
+    group.add_argument(
+        "--no-keep-braced-groups",
+        action="store_const",
+        const=False,
+        dest="keep_braced_groups",
+        help="Keep LaTeX {braced groups} in text output (default no)",
+    )
+
+    group.add_argument(
+        "--keep-braced-groups-minlen",
+        type=int,
+        default=2,
+        dest="keep_braced_groups_minlen",
+        help="Only apply --keep-braced-groups to groups that contain at least "
+        "this many characters",
+    )
+
+    group = parser.add_argument_group("General options")
+
+    group.add_argument(
+        "-q",
+        "--quiet",
+        dest="logging_level",
+        action="store_const",
+        const=logging.ERROR,
+        default=logging.INFO,
+        help="Suppress warning messages",
+    )
+    group.add_argument(
+        "-v",
+        "--verbose",
+        dest="logging_level",
+        action="store_const",
+        const=logging.DEBUG,
+        help="Verbose output",
+    )
+    group.add_argument(
+        "--version",
+        action="version",
+        version="pylatexenc {}".format(version_str),
+        help="Show version information and exit",
+    )
+    group.add_argument(
+        "--help", action="help", help="Show this help information and exit"
+    )
+
+    args = parser.parse_args(argv)
+
+    logging.basicConfig()
+    logging.getLogger().setLevel(args.logging_level)
+    logger = logging.getLogger(__name__)
+
+    if (
+        args.parser_keep_inline_math is not None
+        or args.text_keep_inline_math is not None
+    ):
+        logger.warning(
+            "Options --parser-keep-inline-math and --text-keep-inline-math are "
+            "deprecated and no longer have any effect.  Please use "
+            "--math-mode=... instead."
+        )
+
+    latex = ""
+    if args.code:
+        if args.files:
+            logger.error(
+                "Cannot specify both FILEs and --code option. "
+                "Use --help option for more information."
+            )
+            sys.exit(1)
+        latex = args.code
+    else:
+        for line in fileinput.input(files=args.files):
+            latex += line
+
+    if args.fill_text != -1:
+        if args.fill_text is not None and len(args.fill_text):
+            fill_text = int(args.fill_text)
+        else:
+            fill_text = True
+    else:
+        fill_text = None
+
+    lw = latexwalker.LatexWalker(
+        latex, tolerant_parsing=args.tolerant_parsing, strict_braces=args.strict_braces
+    )
+
+    (nodelist, pos, len_) = lw.get_latex_nodes()
+
+    ln2t = LatexNodes2Text(
+        math_mode=args.math_mode,
+        keep_comments=args.keep_comments,
+        strict_latex_spaces=args.strict_latex_spaces,
+        keep_braced_groups=args.keep_braced_groups,
+        keep_braced_groups_minlen=args.keep_braced_groups_minlen,
+        fill_text=fill_text,
+    )
+
+    print(ln2t.nodelist_to_text(nodelist))
+
+
+def run_main():
+
+    try:
+
+        main()
+
+    except SystemExit:
+        raise
+    except:  # lgtm [py/catch-base-exception]
+        import pdb
+        import traceback
+
+        traceback.print_exc()
+        pdb.post_mortem()
+
+
+if __name__ == "__main__":
+
+    main()
+    # run_main() # debug

good_ai/src/sus/external/pylatexenc/latexencode/__init__.py

@@ -0,0 +1,331 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2018 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+r"""
+The `latexencode` module provides a set of routines that allows you to
+convert a unicode string to LaTeX escape sequences.
+
+For basic usage you can use the :py:func:`unicode_to_latex()` function
+directly::
+
+  >>> from pylatexenc.latexencode import unicode_to_latex
+  >>> print(unicode_to_latex('À votre santé'))
+  \`A votre sant\'e
+  >>> print(unicode_to_latex('The length of samples #3 & #4 is 3μm'))
+  The length of samples \#3 \& \#4 is 3\ensuremath{\mu}m
+
+The conversion is handled by the class :py:class:`UnicodeToLatexEncoder`.  If
+you are converting multiple strings, you may create an instance with the flags
+you like and invoke its method
+:py:meth:`~UnicodeToLatexEncoder.unicode_to_latex()` as many times as necessary::
+
+  >>> from pylatexenc.latexencode import UnicodeToLatexEncoder
+  >>> u = UnicodeToLatexEncoder(unknown_char_policy='replace')
+  >>> print(u.unicode_to_latex('À votre santé'))
+  \`A votre sant\'e
+  >>> print(u.unicode_to_latex('The length of samples #3 & #4 is 3μm'))
+  The length of samples \#3 \& \#4 is 3\ensuremath{\mu}m
+  >>> print(u.unicode_to_latex('À votre santé: 乾杯'))
+  No known latex representation for character: U+4E7E - ‘乾’
+  No known latex representation for character: U+676F - ‘杯’
+  \`A votre sant\'e: {\bfseries ?}{\bfseries ?}
+
+Example using custom conversion rules::
+
+  >>> from pylatexenc.latexencode import UnicodeToLatexEncoder, \
+  ...     UnicodeToLatexConversionRule, RULE_REGEX
+  >>> u = UnicodeToLatexEncoder(
+  ...     conversion_rules=[
+  ...         UnicodeToLatexConversionRule(rule_type=RULE_REGEX, rule=[
+  ...             (re.compile(r'-->'), r'\\textrightarrow'),
+  ...             (re.compile(r'<--'), r'\\textleftarrow'),
+  ...         ]),
+  ...         'defaults'
+  ...     ]
+  ... )
+  >>> print(u.unicode_to_latex("Cheers --> À votre santé"))
+  Cheers {\textrightarrow} \`A votre sant\'e
+
+See :py:class:`UnicodeToLatexEncoder` and
+:py:class:`UnicodeToLatexConversionRule`.  Note for regex rules, the replacement
+text is expanded like the second argument of `re.sub()` and backslashes need to
+be escaped even inside raw strings.
+
+.. versionadded:: 2.0
+
+   The class :py:class:`UnicodeToLatexEncoder` along with its helper functions
+   and classes were introduced in `pylatexenc 2.0`.
+
+   The earlier function :py:func:`utf8tolatex()` that was available in
+   `pylatexenc 1.x` is still provided unchanged, so code written for `pylatexenc
+   1.x` should work without changes.  New code is however strongly encouraged to
+   employ the new API.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import functools
+import itertools
+import logging
+import sys
+import unicodedata
+
+if sys.version_info.major > 2:
+    unicode = str  # need to support unicode() w/ no arguments
+    basestring = str
+    # use MappingProxyType for keeping
+    # inspect function argument names
+    from inspect import getfullargspec
+    from types import MappingProxyType as _MappingProxyType
+else:
+    _MappingProxyType = dict
+    # inspect function argument names -- simulate getfullargspec with getargspec (argh...)
+    from inspect import getargspec as getfullargspec
+
+logger = logging.getLogger(__name__)
+
+
+from .. import _util
+from ._partial_latex_encoder import PartialLatexToLatexEncoder
+from ._unicode_to_latex_encoder import (
+    RULE_CALLABLE,
+    RULE_DICT,
+    RULE_REGEX,
+    UnicodeToLatexConversionRule,
+    UnicodeToLatexEncoder,
+    get_builtin_conversion_rules,
+    get_builtin_uni2latex_dict,
+)
+
+# ------------------------------------------------
+
+
+# ------------------------------------------------
+
+
+# ------------------------------------------------
+
+
+_u2l_obj_cache = {}
+
+
+def unicode_to_latex(
+    s,
+    non_ascii_only=False,
+    replacement_latex_protection="braces",
+    unknown_char_policy="keep",
+    unknown_char_warning=True,
+):
+    r"""
+    Shorthand for constructing a :py:class:`UnicodeToLatexEncoder` instance and
+    calling its :py:meth:`~UnicodeToLatexEncoder.unicode_to_latex()` method.
+
+    The :py:class:`UnicodeToLatexEncoder` instances for given option settings
+    are cached, making repeated calls to :py:func:`unicode_to_latex()` possible
+    without creating a new instance upon each call.
+
+    The parameters `non_ascii_only`, `replacement_latex_protection`,
+    `unknown_char_policy`, and `unknown_char_warning` are directly passed on to
+    the :py:class:`UnicodeToLatexEncoder` constructor.  See the class doc for
+    :py:class:`UnicodeToLatexEncoder` for more information about what they do.
+
+    You may only use arguments to this function that are python hashable (like
+    `True`, `False`, or simple strings) to help us keep a cache of previously
+    constructed :py:class:`UnicodeToLatexEncoder` instances.  For instance, it
+    is not possible to provide a callable to `unknown_char_policy`.  It is also
+    not possible to specify custom conversion rules with this helper function.
+    If you need any of these features, simply create a
+    :py:class:`UnicodeToLatexEncoder` instance directly.
+    """
+
+    key = (
+        non_ascii_only,
+        replacement_latex_protection,
+        unknown_char_policy,
+        unknown_char_warning,
+    )
+
+    if key in _u2l_obj_cache:
+        u = _u2l_obj_cache[key]
+    else:
+        u = UnicodeToLatexEncoder(
+            non_ascii_only=non_ascii_only,
+            replacement_latex_protection=replacement_latex_protection,
+            unknown_char_policy=unknown_char_policy,
+            unknown_char_warning=unknown_char_warning,
+        )
+        _u2l_obj_cache[key] = u
+
+    return u.unicode_to_latex(s)
+
+
+# ------------------------------------------------------------------------------
+
+# Don't change pylatexenc 1.x function:
+
+
+def _get_deprecated_utf82latex():
+    #
+    # Don't issue a deprecation warning, because utf8tolatex() uses the
+    # `utf82latex` dict even if it isn't modified by the user.
+    #
+    #     _util.pylatexenc_deprecated_2(
+    #         "The module-level dictionary `pylatexenc.latexencode.utf82latex` is deprecated "
+    #         "and might be removed in a future version of `pylatexenc`.",
+    #     )
+
+    # return a copy of the dict so that the user can modify the module-level
+    # `utf82latex` dict without influencing the behavior of the new
+    # `unicode_to_latex()` routines. (E.g., if two python modules use
+    # pylatexenc.latexencode, we don't want one python module's use of
+    # `utf2tolatex()` to influence the behavior of another module's use of
+    # `unicode_to_latex()`.  If both modules use `utf8tolatex()`, we can't avoid
+    # this influence.)
+    from ._uni2latexmap import uni2latex as _uni2latex
+
+    return _uni2latex.copy()
+
+
+utf82latex = _util.LazyDict(generate_dict_fn=_get_deprecated_utf82latex)
+"""
+.. deprecated:: 2.0
+
+   Pylatexenc 1.x exposed the module-level dictionary `utf82latex` that could be
+   modified to alter the behavior of `utf8tolatex()`.
+
+   If you would like to obtain a copy of the built-in unicode to text
+   dictionary, see :py:func:`get_builtin_uni2latex_dict()`.  If you would like
+   to alter the behavior of :py:func:`utf8tolatex()`, you should use
+   :py:class:`UnicodeToLatexEncoder` which provides a rich interface for
+   specifying rules how to convert chars to LaTeX escapes.
+
+   For backwards compatibility, you can still modify the module-level dictionary
+   `utf82latex` (but you can't assign a new object to it) and this will directly
+   modify the global built-in dictionary of known latex escapes.  This is not
+   recommended however, and the `utf82latex` module-level dictionary might be
+   removed in the future.
+
+   .. warning::
+
+      Modifying the `utf82latex` module-level dictionary is not recommended.
+      Doing so will alter the behavior of the `utf8tolatex()` function also for
+      all other modules that also use `pylatexenc`!
+"""
+
+
+def utf8tolatex(
+    s,
+    non_ascii_only=False,
+    brackets=True,
+    substitute_bad_chars=False,
+    fail_bad_chars=False,
+):
+    """
+    .. note::
+
+       Since `pylatexenc 2.0`, it is recommended to use the the
+       :py:func:`unicode_to_latex()` function or the
+       :py:class:`UnicodeToLatexEncoder` class instead of the earlier function
+       `utf8tolatex()`.
+
+       The new routines provide much more flexibility and versatility.  For
+       instance, you can specify custom escape sequences for certain characters.
+       Some cheap benchmarks seem to indicate that the new routines are not
+       significantly slower than the `utf8tolatex()` function.  Also, the name
+       `utf8tolatex()` was poorly chosen, since the argument is in fact not
+       'utf-8'-encoded but rather a Python unicode string object.
+
+       The function `utf8tolatex()` is still provided unchanged from `pylatexenc
+       1.x`.  We do not plan to remove this function in the near future so it is
+       not (yet) considered as deprecated and we will continue to provide it in
+       near future versions of `pylatexenc`.  Bug reports, improvements, and new
+       features will however be directed to :py:func:`UnicodeToLatexEncoder()`.
+
+    Encode a UTF-8 string to a LaTeX snippet.
+
+    If `non_ascii_only` is set to `True`, then usual (ascii) characters such as ``#``,
+    ``{``, ``}`` etc. will not be escaped.  If set to `False` (the default), they are
+    escaped to their respective LaTeX escape sequences.
+
+    If `brackets` is set to `True` (the default), then LaTeX macros are enclosed in
+    brackets.  For example, ``sant\N{LATIN SMALL LETTER E WITH ACUTE}`` is replaced by
+    ``sant{\\'e}`` if `brackets=True` and by ``sant\\'e`` if `brackets=False`.
+
+    .. warning::
+        Using `brackets=False` might give you an invalid LaTeX string, so avoid
+        it! (for instance, ``ma\N{LATIN SMALL LETTER I WITH CIRCUMFLEX}tre`` will be
+        replaced incorrectly by ``ma\\^\\itre`` resulting in an unknown macro ``\\itre``).
+
+    If `substitute_bad_chars=True`, then any non-ascii character for which no LaTeX escape
+    sequence is known is replaced by a question mark in boldface. Otherwise (by default),
+    the character is left as it is.
+
+    If `fail_bad_chars=True`, then a `ValueError` is raised if we cannot find a
+    character substitution for any non-ascii character.
+
+    .. versionchanged:: 1.3
+
+        Added `fail_bad_chars` switch
+    """
+
+    s = unicode(s)  # make sure s is unicode
+    s = unicodedata.normalize("NFC", s)
+
+    if not s:
+        return ""
+
+    result = ""
+    for ch in s:
+        # logger.longdebug("Encoding char %r", ch)
+        if non_ascii_only and ord(ch) < 127:
+            result += ch
+        else:
+            # use the `utf82latex` dict -- not `_uni2latex` which should NOT be
+            # modified externally even for backwards-compatible code
+            lch = utf82latex.get(ord(ch), None)
+            if lch is not None:
+                # add brackets if needed, i.e. if we have a substituting macro.
+                # note: in condition, beware, that lch might be of zero length.
+                result += "{" + lch + "}" if brackets and lch[0:1] == "\\" else lch
+            elif (ord(ch) >= 32 and ord(ch) <= 127) or (ch in "\n\r\t"):
+                # ordinary printable ascii char, just add it
+                result += ch
+            else:
+                # non-ascii char
+                msg = "Character cannot be encoded into LaTeX: U+%04X - `%s'" % (
+                    ord(ch),
+                    ch,
+                )
+                if fail_bad_chars:
+                    raise ValueError(msg)
+
+                logger.warning(msg)
+                if substitute_bad_chars:
+                    result += r"{\bfseries ?}"
+                else:
+                    # keep unescaped char
+                    result += ch
+
+    return result

good_ai/src/sus/external/pylatexenc/latexencode/__main__.py

@@ -0,0 +1,148 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2019 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+
+import argparse
+import fileinput
+import logging
+import sys
+
+from ..latexencode import unicode_to_latex
+from ..version import version_str
+
+
+def main(argv=None):
+
+    if argv is None:
+        argv = sys.argv[1:]
+
+    parser = argparse.ArgumentParser(prog="latexencode", add_help=False)
+    parser.add_argument(
+        "files",
+        metavar="FILE",
+        nargs="*",
+        help="Input files (if none specified, read from stdandard input)",
+    )
+
+    parser.add_argument(
+        "--non-ascii-only",
+        action="store_const",
+        const=True,
+        dest="non_ascii_only",
+        default=False,
+    )
+    parser.add_argument(
+        "--no-non-ascii-only",
+        action="store_const",
+        const=False,
+        dest="non_ascii_only",
+        help="The option --non-ascii-only specifies that only non-ascii characters "
+        "are to be encoded into LaTeX sequences, and not characters like '$' "
+        "even though they might have a special LaTeX meaning.",
+    )
+
+    parser.add_argument(
+        "--replacement-latex-protection",
+        choices=(
+            "braces",
+            "braces-all",
+            "braces-almost-all",
+            "braces-after-macro",
+            "none",
+        ),
+        dest="replacement_latex_protection",
+        default="braces",
+        help=r"How to protect replacement latex code from producing invalid latex code "
+        r"when concatenated in a longer string.  One of 'braces', 'braces-all', "
+        r"'braces-almost-all', 'braces-after-macro', 'none'.  Example: using "
+        r"choice 'braces' we avoid the invalid replacement 'a→b' -> 'a\tob' "
+        r"with instead 'a{\to}b'.",
+    )
+
+    parser.add_argument(
+        "--unknown-char-policy",
+        choices=("keep", "replace", "ignore", "fail"),
+        dest="unknown_char_policy",
+        default="keep",
+        help="How to deal with nonascii characters with no known latex code equivalent.",
+    )
+
+    parser.add_argument(
+        "-q",
+        "--quiet",
+        dest="logging_level",
+        action="store_const",
+        const=logging.ERROR,
+        default=logging.INFO,
+        help="Suppress warning messages",
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version="pylatexenc {}".format(version_str),
+        help="Show version information and exit",
+    )
+    parser.add_argument(
+        "--help", action="help", help="Show this help information and exit"
+    )
+
+    args = parser.parse_args(argv)
+
+    logging.basicConfig()
+    logging.getLogger().setLevel(args.logging_level)
+
+    latex = ""
+    for line in fileinput.input(files=args.files):
+        latex += line
+
+    result = unicode_to_latex(
+        latex,
+        non_ascii_only=args.non_ascii_only,
+        replacement_latex_protection=args.replacement_latex_protection,
+        unknown_char_policy=args.unknown_char_policy,
+    )
+
+    sys.stdout.write(result)
+
+
+def run_main():
+    try:
+
+        main()
+
+    except SystemExit:
+        raise
+    except:  # lgtm [py/catch-base-exception]
+        import pdb
+        import traceback
+
+        traceback.print_exc()
+        pdb.post_mortem()
+
+
+if __name__ == "__main__":
+
+    # run_main()  ## DEBUG
+    main()

good_ai/src/sus/external/pylatexenc/latexencode/_partial_latex_encoder.py

@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2021 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+# import sys
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+from ._unicode_to_latex_encoder import (
+    RULE_CALLABLE,
+    UnicodeToLatexConversionRule,
+    UnicodeToLatexEncoder,
+)
+
+# if sys.version_info.major == 2:
+#     bytes = str
+#     str = unicode
+
+
+class PartialLatexToLatexEncoder(UnicodeToLatexEncoder):
+    r"""
+    Encode a string while preserving some (fuzzily detected) LaTeX constructs
+    that the input string already has (e.g. accent macros or inline math modes).
+
+    Sometimes you need to fully LaTeX-encode a string that already has some
+    LaTeX constructs.  For instance, titles of bibliographic entries might
+    include some inline math or accents, but they might also include unicode
+    characters that need to be encoded.  Using a
+    :py:class:`UnicodeToLatexEncoder` on such strings would result in ugly
+    doubly-escaped strings such as ``\textbackslash{}'\{e\}``.  Instead,
+    constructs such as ``\'{e}`` should be preserved while other characters
+    and/or constructs (say '&' or '%') as well as unicode characters should be
+    encoded.
+
+    This class offers a simple partial solution: Characters are encoded as per
+    the given `conversion_rules` (or the default conversion rules of
+    :py:class:`UnicodeToLatexEncoder` objects), except that the characters in
+    `keep_latex_chars` are to be interpreted as LaTeX and are not to be further
+    encoded.
+
+    .. versionadded: 2.10
+    """
+
+    def __init__(
+        self,
+        # keyword arguments:
+        keep_latex_chars=r"\${}^_",
+        conversion_rules=None,
+        **kwargs
+    ):
+
+        base_conversion_rules = conversion_rules
+        if base_conversion_rules is None:
+            base_conversion_rules = ["defaults"]
+
+        super(PartialLatexToLatexEncoder, self).__init__(
+            # only a single rule, our own special method that tries to parse
+            # partial latex.
+            conversion_rules=[
+                UnicodeToLatexConversionRule(
+                    rule_type=RULE_CALLABLE,
+                    rule=self._do_partial_latex_encode_step,
+                    replacement_latex_protection="none",
+                )
+            ]
+            + base_conversion_rules,
+            **kwargs
+        )
+
+        self.keep_latex_chars = keep_latex_chars
+
+    def _do_partial_latex_encode_step(self, s, pos):
+        r"""
+        This method is used as a "callable rule" for the
+        :py:class:`UnicodeToLatexEncoder` object.
+
+        The strategy is to see if we have something that looks like a LaTeX char
+        we want to keep.  If so, keep it as is; if not, return `None` so that
+        further rules can be considered by the base unicode encoder.
+        """
+
+        from ..latexwalker import LatexWalker
+
+        if s[pos] in self.keep_latex_chars:
+            # Read a token and if it is a macro, keep the full macro!
+            lw = LatexWalker(s, tolerant_parsing=False)
+
+            tok = lw.get_token(pos, environments=False)
+
+            tok_as_latex = tok.pre_space + s[tok.pos : tok.pos + tok.len]
+
+            # keep the LaTeX token as-is
+            return (tok.pos + tok.len - pos, tok_as_latex)
+
+        return None

good_ai/src/sus/external/pylatexenc/latexencode/_unicode_to_latex_encoder.py

@@ -0,0 +1,686 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2021 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import functools
+import itertools
+import logging
+import sys
+import unicodedata
+
+if sys.version_info.major > 2:
+    unicode = str  # need to support unicode() w/ no arguments
+    basestring = str
+    # use MappingProxyType for keeping
+    # inspect function argument names
+    from inspect import getfullargspec
+    from types import MappingProxyType as _MappingProxyType
+else:
+    _MappingProxyType = dict
+    # inspect function argument names -- simulate getfullargspec with getargspec (argh...)
+    from inspect import getargspec as getfullargspec
+
+logger = logging.getLogger(__name__)
+
+
+def get_builtin_uni2latex_dict():
+    r"""
+    Return a dictionary that contains the default collection of known LaTeX
+    escape sequences for unicode characters.
+
+    The keys of the dictionary are integers that correspond to unicode code
+    points (i.e., `ord(char)`).  The values are the corresponding LaTeX
+    replacement strings.
+
+    The returned dictionary may not be modified.  To alter the behavior of
+    :py:func:`unicode_to_latex()`, you should specify custom rules to a new
+    instance of :py:class:`UnicodeToLatexEncoder`.
+
+    .. versionadded:: 2.0
+
+       This function was introduced in `pylatexenc 2.0`.
+    """
+    from ._uni2latexmap import uni2latex as _uni2latex
+
+    return _MappingProxyType(_uni2latex)
+
+
+RULE_DICT = 0
+r"""
+Indicates a rule type that is a dictionary of unicode point values to
+replacement strings. See :py:class:`UnicodeToLatexConversionRule`.
+
+.. versionadded:: 2.0
+
+   This member was introduced in pylatexenc version 2.0.
+"""
+
+RULE_REGEX = 1
+r"""
+Indicates a rule type that is a list (or iterable) of pairs
+`(compiled_regular_expression, replacement_string)`.  See
+:py:class:`UnicodeToLatexConversionRule`.
+
+.. versionadded:: 2.0
+
+   This member was introduced in pylatexenc version 2.0.
+"""
+
+RULE_CALLABLE = 2
+r"""
+Indicates a rule type that is a custom callable.  See
+:py:class:`UnicodeToLatexConversionRule`.
+
+.. versionadded:: 2.0
+
+   This member was introduced in pylatexenc version 2.0.
+"""
+
+
+class UnicodeToLatexConversionRule:
+    r"""
+    Specify a rule how to convert unicode characters into LaTeX escapes.
+
+    .. py:attribute:: rule_type
+    
+       One of :py:data:`RULE_DICT`, :py:data:`RULE_REGEX`, or
+       :py:data:`RULE_CALLABLE`.
+
+    .. py:attribute:: rule
+
+       A specification of the rule itself.  The `rule` attribute is an object
+       that depends on what `rule_type` is set to.  See below.
+
+    .. py:attribute:: replacement_latex_protection
+
+       If non-`None`, then the setting here will override any
+       `replacement_latex_protection` set on
+       :py:class:`UnicodeToLatexConversionRule` objects.  By default the value
+       is `None`, and you can set a replacement_latex_protection globally for
+       all rules on the :py:class:`UnicodeToLatexEncoder` object.
+
+       The use of this attribute is mainly in case you have a fancy rule in
+       which you already guarantee that whatever you output is valid LaTeX even
+       if concatenated with the remainder of the string; in this case you can
+       set `replacement_latex_protection='none'` to avoid unnecessary or
+       unwanted braces around the generated code.
+
+       .. versionadded:: 2.10
+
+          The `replacement_latex_protection` attribute was introduced in
+          `pylatexenc 2.10`.
+
+
+    Constructor syntax::
+    
+        UnicodeToLatexConversionRule(RULE_XXX, <...>)
+        UnicodeToLatexConversionRule(rule_type=RULE_XXX, rule=<...>)
+
+        UnicodeToLatexConversionRule(..., replacement_latex_protection='none')
+
+    Note that you can get some built-in rules via the
+    :py:func:`get_builtin_conversion_rules()` function::
+
+        conversion_rules = get_builtin_conversion_rules('defaults') # all defaults
+
+
+    Rules types:
+    
+      - `RULE_DICT`: If `rule_type` is `RULE_DICT`, then `rule` should be a
+        dictionary whose keys are integers representing unicode code points
+        (e.g., `0x210F`), and whose values are corresponding replacement strings
+        (e.g., ``r'\hbar'``).  See :py:func:`get_builtin_uni2latex_dict()` for
+        an example.
+
+      - `RULE_REGEX`: If `rule_type` is `RULE_REGEX`, then `rule` should be an
+        iterable of tuple pairs `(compiled_regular_expression,
+        replacement_string)` where `compiled_regular_expression` was obtained
+        with `re.compile(...)` and `replacement_string` is anything that can be
+        specified as the second (`repl`) argument of `re.sub(...)`.  This can be
+        a replacement string that includes escapes (like ``\1, \2, \g<name>``)
+        for captured sub-expressions or a callable that takes a match object as
+        argument.
+
+        .. note::
+    
+           The replacement string is parsed like the second argument to
+           `re.sub()` and backslashes have a special meaning because they can
+           refer to captured sub-expressions.  For a literal backslash, use two
+           backslashes ``\\`` in raw strings, four backslashes in normal
+           strings.
+
+        Example::
+
+          regex_conversion_rule = UnicodeToLatexConversionRule(
+              rule_type=RULE_REGEX,
+              rule=[
+                  # protect acronyms of capital letters with braces,
+                  # e.g.: ABC -> {ABC}
+                  (re.compile(r'[A-Z]{2,}'), r'{\1}'),
+                  # Additional rules, e.g., "..." -> "\ldots"
+                  (re.compile(r'...'), r'\\ldots'), # note double \\
+              ]
+          )
+
+      - `RULE_CALLABLE`: If `rule_type` is `RULE_CALLABLE`, then `rule` should
+        be a callable that accepts two arguments, the unicode string and the
+        position in the string (an integer).  The callable will be called with
+        the original unicode string as argument and the position of the
+        character that needs to be encoded.  If this rule can encode the given
+        character at the given position, it should return a tuple
+        `(consumed_length, replacement_string)` where `consumed_length` is the
+        number of characters in the unicode string that `replacement_string`
+        represents.  If the character(s) at the given position can't be encoded
+        by this rule, the callable should return `None` to indicate that further
+        rules should be attempted.
+
+        If the callable accepts an additional argument called `u2lobj`, then the
+        :py:class:`UnicodeToLatexEncoder` instance is provided to that argument.
+
+        For example, the following callable should achieve the same effect as
+        the previous example with regexes::
+
+          def convert_stuff(s, pos):
+              m = re.match(r'[A-Z]{2,}', s, pos)
+              if m is not None:
+                  return (m.end()-m.start(), '{'+m.group()+'}')
+              if s.startswith('...', pos): # or  s[pos:pos+3] == '...'
+                  return (3, r'\ldots')
+              return None
+
+
+    .. versionadded:: 2.0
+
+       This class was introduced in `pylatexenc 2.0`.
+    """
+
+    def __init__(
+        self,
+        rule_type,
+        rule=None,
+        # keyword-only, please:
+        replacement_latex_protection=None,
+    ):
+        self.rule_type = rule_type
+        self.rule = rule
+        self.replacement_latex_protection = replacement_latex_protection
+
+    def __repr__(self):
+        return "{}(rule_type={!r}, rule=<{}>, replacement_latex_protection={})".format(
+            self.__class__.__name__,
+            self.rule_type,
+            type(self.rule).__name__,
+            repr(self.replacement_latex_protection),
+        )
+
+
+def get_builtin_conversion_rules(builtin_name):
+    r"""
+    Return a built-in set of conversion rules specified by a given name
+    `builtin_name`.
+
+    There are two builtin conversion rules, with the following names:
+
+      - `'defaults'`: the default conversion rules, a custom-curated list of
+        unicode chars to LaTeX escapes.
+
+      - `'unicode-xml'`: the conversion rules derived from the `unicode.xml` file
+        maintained at https://www.w3.org/TR/xml-entity-names/#source by David
+        Carlisle.
+
+    The return value is a list of :py:class:`UnicodeToLatexConversionRule`
+    objects that can be either directly specified to the `conversion_rules=`
+    argument of :py:class:`UnicodeToLatexEncoder`, or included in a larger list
+    that can be provided to that argument.
+
+    .. versionadded:: 2.0
+
+       This function was introduced in `pylatexenc 2.0`.
+    """
+    if builtin_name == "defaults":
+        return [
+            UnicodeToLatexConversionRule(
+                rule_type=RULE_DICT, rule=get_builtin_uni2latex_dict()
+            )
+        ]
+    if builtin_name == "unicode-xml":
+        from . import _uni2latexmap_xml
+
+        return [
+            UnicodeToLatexConversionRule(
+                rule_type=RULE_DICT, rule=_uni2latexmap_xml.uni2latex
+            )
+        ]
+    raise ValueError("Unknown builtin rule set: {}".format(builtin_name))
+
+
+class UnicodeToLatexEncoder(object):
+    r"""
+    Encode a string with unicode characters into a LaTeX snippet.
+
+    The following general attributes can be specified as keyword arguments to
+    the constructor.  Note: These attributes must be specified to the
+    constructor and may NOT be subsequently modified.  This is because in the
+    constructor we pre-compile some rules and flags to optimize calls to
+    :py:meth:`unicode_to_text()`.
+
+    .. py:attribute:: non_ascii_only
+
+       Whether we should convert only non-ascii characters into LaTeX sequences,
+       or also all known ascii characters with special LaTeX meaning such as
+       '\\\\', '$', '&', etc.
+
+       If `non_ascii_only` is set to `True` (the default is `False`), then
+       conversion rules are not applied at positions in the string where an
+       ASCII character is encountered.
+
+    .. py:attribute:: conversion_rules
+
+       The conversion rules, specified as a list of
+       :py:class:`UnicodeToLatexConversionRule` objects.  For each position in
+       the string, the rules will be applied in the given sequence until a
+       replacement string is found.
+
+       Instead of a :py:class:`UnicodeToLatexConversionRule` object you may also
+       specify a string specifying a built-in rule (e.g., 'defaults'), which
+       will be expanded to the corresponding rules according to
+       :py:func:`get_builtin_conversion_rules()`.
+
+       If you specify your own list of rules using this argument, you will
+       probably want to include presumably at the end of your list the element
+       'defaults' to include all built-in default conversion rules.  To override
+       built-in rules, simply add your custom rules earlier in the list.
+       Example::
+
+         conversion_rules = [
+             # our custom rules
+             UnicodeToLatexConversionRule(RULE_REGEX, [
+                 # double \\ needed, see UnicodeToLatexConversionRule
+                 ( re.compile(r'...'), r'\\ldots' ),
+                 ( re.compile(r'î'), r'\\^i' ),
+             ]),
+             # plus all the default rules
+             'defaults'
+         ]
+         u = UnicodeToLatexEncoder(conversion_rules=conversion_rules)
+
+    .. py:attribute:: replacement_latex_protection
+
+       How to "protect" LaTeX replacement text that looks like it could be
+       interpreted differently if concatenated to arbitrary strings before and
+       after.
+
+       Currently in the default scheme only one situation is recognized: if the
+       replacement string ends with a latex macro invocation with a non-symbol
+       macro name, e.g. ``\textemdash`` or ``\^\i``.  Indeed, if we naively
+       replace these texts in an arbitrary string (like ``maître``), we might
+       get an invalid macro invocation (like ``ma\^\itre`` which causes un known
+       macro name ``\itre``).
+
+       Possible protection schemes are:
+
+         - 'braces' (the default):  Any suspicious replacement text (that
+           might look fragile) is placed in curly braces ``{...}``.
+
+         - 'braces-all':  All replacement latex escapes are surrounded in
+           protective curly braces ``{...}``, regardless of whether or not they
+           might be deemed "fragile" or "unsafe".
+
+         - 'braces-almost-all':  Almost all replacement latex escapes are
+           surrounded in protective curly braces ``{...}``.  This option
+           emulates closely the behavior of `brackets=True` of the function
+           `utf8tolatex()` in `pylatexenc 1.x`, though I'm not sure it is really
+           useful.  [Specifically, all those replacement strings that start with
+           a backslash are surrounded by curly braces].
+
+         - 'braces-after-macro':  In the situation where the replacement latex
+           code ends with a string-named macro, then a pair of empty braces is
+           added at the end of the replacement text to protect the macro.
+
+         - 'none': No protection is applied, even in "unsafe" cases.  This is
+           not recommended, as this will likely result in invalid LaTeX
+           code. (Note this is the string 'none', not Python's built-in `None`.)
+
+         - any callable object: The callable should take a single argument, the
+           replacement latex string associated with a piece of the input (maybe
+           a special character) that has been encoded; it should return the
+           actual string to append to the output string.
+
+         .. versionadded:: 2.10
+
+            You can specify a callable object to `replacement_latex_protection`
+            since `pylatexenc 2.10`.
+
+    .. py:attribute:: unknown_char_policy
+
+       What to do when a non-ascii character is encountered without any known
+       substitution macro.  The attribute `unknown_char_policy` can be set to one of:
+
+         - 'keep': keep the character as is;
+
+         - 'replace': replace the character by a boldface question mark;
+
+         - 'ignore': ignore the character from the input entirely and don't
+           output anything for it;
+
+         - 'fail': raise a `ValueError` exception;
+
+         - 'unihex': output the unicode hexadecimal code (U+XXXX) of the
+           character in typewriter font;
+
+         - a Python callable --- will be called with argument the character that
+           could not be encoded.  (If the callable accepts a second argument
+           called 'u2lobj', then the `UnicodeToLatexEncoder` instance is
+           provided to that argument.)  The return value of the callable is used
+           as LaTeX replacement code.
+
+    .. py:attribute:: unknown_char_warning
+
+       In addition to the `unknown_char_policy`, this attribute indicates
+       whether or not (`True` or `False`) one should generate a warning when a
+       nonascii character without any known latex representation is
+       encountered. (Default: True)
+
+    .. py:attribute:: latex_string_class
+
+       The return type of :py:meth:`unicode_to_latex()`.  Normally this is a
+       simple unicode string (`str` on `Python 3` or `unicode` on `Python 2`).
+
+       But you can specify your custom string type via the `latex_string_class`
+       argument.  The `latex_string_class` will be invoked with no arguments to
+       construct an empty object (so `latex_string_class` can be either an
+       object that can be constructed with no arguments or it can be a function
+       with no arguments that return a fresh object instance).  The object must
+       support the operation "+=", i.e., you should overload the ``__iadd__()``
+       method.
+
+       For instance, you can record the chunks that would have been appended
+       into a single string as follows::
+
+           class LatexChunkList:
+               def __init__(self):
+                   self.chunks = []
+
+               def __iadd__(self, s):
+                   self.chunks.append(s)
+                   return self
+
+           u = UnicodeToLatexEncoder(latex_string_class=LatexChunkList,
+                                     replacement_latex_protection='none')
+           result = u.unicode_to_latex("é → α")
+           # result.chunks == [ r"\'e", ' ', r'\textrightarrow', ' ',
+           #                    r'\ensuremath{\alpha}' ]
+
+    .. warning::
+
+       None of the above attributes should be modified after constructing the
+       object.  The values specified to the class constructor are final and
+       cannot be changed.  [Indeed, the class constructor "compiles" these
+       attribute values into a data structure that makes
+       :py:meth:`unicode_to_text()` slightly more efficient.]
+
+    .. versionadded:: 2.0
+
+       This class was introduced in `pylatexenc 2.0`.
+    """
+
+    def __init__(self, **kwargs):
+        self.non_ascii_only = kwargs.pop("non_ascii_only", False)
+        self.conversion_rules = kwargs.pop("conversion_rules", ["defaults"])
+        self.replacement_latex_protection = kwargs.pop(
+            "replacement_latex_protection", "braces"
+        )
+        self.unknown_char_policy = kwargs.pop("unknown_char_policy", "keep")
+        self.unknown_char_warning = kwargs.pop("unknown_char_warning", True)
+        self.latex_string_class = kwargs.pop("latex_string_class", unicode)
+
+        if kwargs:
+            logger.warning(
+                "Ignoring unknown keyword arguments: %s", ",".join(kwargs.keys())
+            )
+
+        super(UnicodeToLatexEncoder, self).__init__(**kwargs)
+
+        # build generator that expands built-in conversion rules
+        expanded_conversion_rules = itertools.chain.from_iterable(
+            (get_builtin_conversion_rules(r) if isinstance(r, basestring) else [r])
+            for r in self.conversion_rules
+        )
+
+        #
+        # now "pre-compile" some stuff so that calls to unicode_to_latex() can
+        # hopefully execute faster
+        #
+
+        # "pre-compile" rules and check rule types:
+        self._compiled_rules = []
+        for rule in expanded_conversion_rules:
+            if rule.rule_type == RULE_DICT:
+                self._compiled_rules.append(
+                    functools.partial(self._apply_rule_dict, rule.rule, rule)
+                )
+            elif rule.rule_type == RULE_REGEX:
+                self._compiled_rules.append(
+                    functools.partial(self._apply_rule_regex, rule.rule, rule)
+                )
+            elif rule.rule_type == RULE_CALLABLE:
+                thecallable = rule.rule
+                if "u2lobj" in getfullargspec(thecallable)[0]:
+                    thecallable = functools.partial(rule.rule, u2lobj=self)
+                self._compiled_rules.append(
+                    functools.partial(self._apply_rule_callable, thecallable, rule)
+                )
+            else:
+                raise TypeError("Invalid rule type: {}".format(rule.rule_type))
+
+        # bad char policy:
+        if isinstance(self.unknown_char_policy, basestring):
+            self._do_unknown_char = self._get_method_fn(
+                "do_unknown_char", self.unknown_char_policy, what="unknown_char_policy"
+            )
+        elif callable(self.unknown_char_policy):
+            fn = self.unknown_char_policy
+            if "u2lobj" in getfullargspec(fn)[0]:
+                self._do_unknown_char = functools.partial(
+                    self.unknown_char_policy, u2lobj=self
+                )
+            else:
+                self._do_unknown_char = self.unknown_char_policy
+        else:
+            raise TypeError(
+                "Invalid argument for unknown_char_policy: {!r}".format(
+                    self.unknown_char_policy
+                )
+            )
+
+        # bad char warning:
+        if not self.unknown_char_warning:
+            self._do_warn_unknown_char = lambda ch: None  # replace method by no-op
+
+        # set a method that will skip ascii characters if required:
+        if self.non_ascii_only:
+            self._maybe_skip_ascii = self._check_do_skip_ascii
+        else:
+            self._maybe_skip_ascii = lambda s, p: False
+
+        # set a method to protect replacement latex code, if necessary:
+        self._apply_protection = self._get_replacement_latex_fn(
+            self.replacement_latex_protection
+        )
+
+    def _get_method_fn(self, base, name, what):
+        selfmethname = "_" + base + "_" + name.replace("-", "_")
+        if not hasattr(self, selfmethname):
+            raise ValueError("Invalid {}: {}".format(what, name))
+        return getattr(self, selfmethname)
+
+    def _get_replacement_latex_fn(self, replacement_latex_protection):
+        if callable(replacement_latex_protection):
+            return replacement_latex_protection
+        return self._get_method_fn(
+            "apply_protection",
+            replacement_latex_protection,
+            what="replacement_latex_protection",
+        )
+
+    def unicode_to_latex(self, s):
+        """
+        Convert unicode characters in the string `s` into latex escape sequences,
+        according to the rules and options given to the constructor.
+        """
+
+        s = unicode(s)  # make sure s is unicode
+        s = unicodedata.normalize("NFC", s)
+
+        class _NS:
+            pass
+
+        p = _NS()
+        p.latex = self.latex_string_class()
+        p.pos = 0
+
+        while p.pos < len(s):
+
+            if self._maybe_skip_ascii(s, p):
+                continue
+
+            for compiledrule in self._compiled_rules:
+                if compiledrule(s, p):
+                    break
+            else:
+                # for-else, see
+                # https://docs.python.org/2/tutorial/controlflow.html\
+                # #break-and-continue-statements-and-else-clauses-on-loops
+                ch = s[p.pos]
+                o = ord(ch)
+                if (o >= 32 and o <= 127) or (ch in "\n\r\t"):
+                    p.latex += ch
+                    p.pos += 1
+                else:
+                    self._do_warn_unknown_char(ch)
+                    p.latex += self._do_unknown_char(ch)
+                    p.pos += 1
+
+        return p.latex
+
+    def _check_do_skip_ascii(self, s, p):
+        if ord(s[p.pos]) < 127:
+            # skip, we only want to convert non-ascii chars
+            p.latex += s[p.pos]
+            p.pos += 1
+            return True
+        return False
+
+    def _apply_rule_dict(self, ruledict, rule, s, p):
+        o = ord(s[p.pos])
+        if o in ruledict:
+            self._apply_replacement(p, ruledict[o], 1, rule)
+            return True
+        return None
+
+    def _apply_rule_regex(self, ruleregexes, rule, s, p):
+        for regex, repl in ruleregexes:
+            m = regex.match(s, p.pos)
+            if m is not None:
+                if callable(repl):
+                    replstr = repl(m)
+                else:
+                    replstr = m.expand(repl)
+                self._apply_replacement(p, replstr, m.end() - m.start(), rule)
+                return True
+        return None
+
+    def _apply_rule_callable(self, rulecallable, rule, s, p):
+        res = rulecallable(s, p.pos)
+        if res is None:
+            return None
+        (consumed, repl) = res
+        self._apply_replacement(p, repl, consumed, rule)
+        return True
+
+    def _apply_replacement(self, p, repl, numchars, ruleobj):
+        # check for possible replacement latex protection, like braces.
+
+        protect_fn = self._apply_protection
+
+        # maybe the rule object has overridden the replacement_latex_protection to use.
+        if ruleobj.replacement_latex_protection is not None:
+            protect_fn = self._get_replacement_latex_fn(
+                ruleobj.replacement_latex_protection
+            )
+
+        repl = protect_fn(repl)
+        p.latex += repl
+        p.pos += numchars
+
+    def _apply_protection_none(self, repl):
+        # no protection
+        return repl
+
+    def _apply_protection_braces(self, repl):
+        k = repl.rfind("\\")
+        if k >= 0 and repl[k + 1 :].isalpha():
+            # has dangling named macro, apply protection.
+            return "{" + repl + "}"
+        return repl
+
+    def _apply_protection_braces_almost_all(self, repl):
+        if repl[0:1] == "\\":
+            return "{" + repl + "}"
+        return repl
+
+    def _apply_protection_braces_all(self, repl):
+        return "{" + repl + "}"
+
+    def _apply_protection_braces_after_macro(self, repl):
+        k = repl.rfind("\\")
+        if k >= 0 and repl[k + 1 :].isalpha():
+            # has dangling named macro, apply protection.
+            return repl + "{}"
+        return repl
+
+    # policies for "bad chars":
+    def _do_unknown_char_keep(self, ch):
+        return ch
+
+    def _do_unknown_char_replace(self, ch):
+        return r"{\bfseries ?}"
+
+    def _do_unknown_char_ignore(self, ch):
+        return ""
+
+    def _do_unknown_char_fail(self, ch):
+        raise ValueError(
+            "No known latex representation for character: U+%04X - ‘%s’" % (ord(ch), ch)
+        )
+
+    def _do_unknown_char_unihex(self, ch):
+        return r"\ensuremath{\langle}\texttt{U+%04X}\ensuremath{\rangle}" % (ord(ch))
+
+    def _do_warn_unknown_char(self, ch):
+        logger.warning(
+            "No known latex representation for character: U+%04X - ‘%s’", ord(ch), ch
+        )

good_ai/src/sus/external/pylatexenc/latexwalker/__main__.py

@@ -0,0 +1,230 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2018 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+import argparse
+import fileinput
+import json
+import logging
+import sys
+
+from ..latexwalker import LatexWalker, disp_node, make_json_encoder
+from ..version import version_str
+
+
+def main(argv=None):
+
+    if argv is None:
+        argv = sys.argv[1:]
+
+    parser = argparse.ArgumentParser(prog="latexwalker", add_help=False)
+
+    parser.add_argument(
+        "--output-format",
+        metavar="FORMAT",
+        dest="output_format",
+        choices=["human", "json"],
+        default="human",
+        help='Requested output format for the node tree ("human" or "json")',
+    )
+    parser.add_argument(
+        "--json-indent",
+        metavar="NUMSPACES",
+        dest="json_indent",
+        type=int,
+        default=2,
+        help="Indentation in JSON output (specify number of spaces "
+        "per indentation level)",
+    )
+    parser.add_argument(
+        "--json-compact",
+        dest="json_indent",
+        action="store_const",
+        const=None,
+        help="Output compact JSON",
+    )
+
+    parser.add_argument(
+        "--keep-inline-math",
+        action="store_const",
+        const=True,
+        dest="keep_inline_math",
+        default=True,
+        help=argparse.SUPPRESS,
+    )
+    parser.add_argument(
+        "--no-keep-inline-math",
+        action="store_const",
+        const=False,
+        dest="keep_inline_math",
+        help=argparse.SUPPRESS,
+    )
+
+    parser.add_argument(
+        "--tolerant-parsing",
+        action="store_const",
+        const=True,
+        dest="tolerant_parsing",
+        default=True,
+    )
+    parser.add_argument(
+        "--no-tolerant-parsing",
+        action="store_const",
+        const=False,
+        dest="tolerant_parsing",
+        help="Tolerate syntax errors when parsing, and attempt "
+        "to continue (default yes)",
+    )
+
+    # I'm not sure this flag is useful and if it should be exposed at all.
+    # Accept it, but make it hidden.
+    parser.add_argument(
+        "--strict-braces",
+        action="store_const",
+        const=True,
+        dest="strict_braces",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
+    parser.add_argument(
+        "--no-strict-braces",
+        action="store_const",
+        const=False,
+        dest="strict_braces",
+        # help="Report errors for mismatching LaTeX braces (default no)"
+        help=argparse.SUPPRESS,
+    )
+
+    parser.add_argument(
+        "-q",
+        "--quiet",
+        dest="logging_level",
+        action="store_const",
+        const=logging.ERROR,
+        default=logging.INFO,
+        help="Suppress warning messages",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        dest="logging_level",
+        action="store_const",
+        const=logging.DEBUG,
+        help="Verbose output",
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version="pylatexenc {}".format(version_str),
+        help="Show version information and exit",
+    )
+    parser.add_argument(
+        "--help", action="help", help="Show this help information and exit"
+    )
+
+    parser.add_argument(
+        "--code",
+        "-c",
+        action="store",
+        default=None,
+        metavar="LATEX_CODE",
+        help="Convert the given LATEX_CODE to unicode text instead of reading "
+        "from FILE or standard input.  You cannot specify FILEs if you use this "
+        "option, and any standard input is ignored.",
+    )
+
+    parser.add_argument(
+        "files",
+        metavar="FILE",
+        nargs="*",
+        help="Input files (if none specified, read from stdandard input)",
+    )
+
+    args = parser.parse_args(argv)
+
+    logging.basicConfig()
+    logging.getLogger().setLevel(args.logging_level)
+    logger = logging.getLogger(__name__)
+
+    latex = ""
+    if args.code:
+        if args.files:
+            logger.error(
+                "Cannot specify both FILEs and --code option. "
+                "Use --help option for more information."
+            )
+            sys.exit(1)
+        latex = args.code
+    else:
+        for line in fileinput.input(files=args.files):
+            latex += line
+
+    latexwalker = LatexWalker(
+        latex, tolerant_parsing=args.tolerant_parsing, strict_braces=args.strict_braces
+    )
+
+    (nodelist, pos, len_) = latexwalker.get_latex_nodes()
+
+    if args.output_format == "human":
+        print("\n--- NODES ---\n")
+        for n in nodelist:
+            disp_node(n)
+        print("\n-------------\n")
+        return
+
+    if args.output_format == "json":
+        json.dump(
+            {
+                "nodelist": nodelist,
+            },
+            sys.stdout,
+            cls=make_json_encoder(latexwalker),
+            indent=args.json_indent,
+        )
+        sys.stdout.write("\n")
+        return
+
+    raise ValueError("Invalid output format: " + args.output_format)
+
+
+def run_main():
+
+    try:
+
+        main()
+
+    except SystemExit:
+        raise
+    except:  # lgtm [py/catch-base-exception]
+        import pdb
+        import traceback
+
+        traceback.print_exc()
+        pdb.post_mortem()
+
+
+if __name__ == "__main__":
+
+    # run_main() # debug
+    main()

good_ai/src/sus/external/pylatexenc/latexwalker/_defaultspecs.py

@@ -0,0 +1,459 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2019 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+
+# Internal module. May change without notice.
+
+
+from ..macrospec import (
+    EnvironmentSpec,
+    MacroSpec,
+    MacroStandardArgsParser,
+    VerbatimArgsParser,
+    std_environment,
+    std_macro,
+    std_specials,
+)
+
+specs = [
+    #
+    # CATEGORY: latex-base
+    #
+    (
+        "latex-base",
+        {
+            "macros": [
+                std_macro("documentclass", True, 1),
+                std_macro("usepackage", True, 1),
+                std_macro("RequirePackage", True, 1),
+                std_macro("selectlanguage", True, 1),
+                std_macro("setlength", True, 2),
+                std_macro("addlength", True, 2),
+                std_macro("setcounter", True, 2),
+                std_macro("addcounter", True, 2),
+                std_macro("newcommand", "*{[[{"),
+                std_macro("renewcommand", "*{[[{"),
+                std_macro("providecommand", "*{[[{"),
+                std_macro("newenvironment", "*{[[{{"),
+                std_macro("renewenvironment", "*{[[{{"),
+                std_macro("provideenvironment", "*{[[{{"),
+                std_macro("DeclareMathOperator", "*{{"),
+                std_macro("hspace", "*{"),
+                std_macro("vspace", "*{"),
+                MacroSpec(
+                    "mbox",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                # \title, \author, \date
+                MacroSpec("title", "{"),
+                MacroSpec("author", "{"),
+                MacroSpec("date", "{"),
+                # (Note: single backslash) end of line with optional no-break ('*') and
+                # additional vertical spacing, e.g. \\*[2mm]
+                #
+                # Special for this command: don't allow an optional spacing argument
+                # [2mm] to be separated by spaces from the rest of the macro.  This
+                # emulates the behavior in AMS environments, and avoids some errors;
+                # e.g. in "\begin{align} A=0 \\ [C,D]=0 \end{align}" the "[C,D]"
+                # does not get captured as an optional macro argument.
+                MacroSpec(
+                    "\\",
+                    args_parser=MacroStandardArgsParser(
+                        "*[", optional_arg_no_space=True
+                    ),
+                ),
+                std_macro("item", True, 0),
+                # \input{someotherfile}
+                std_macro("input", False, 1),
+                std_macro("include", False, 1),
+                std_macro("includegraphics", True, 1),
+                std_macro("chapter", "*[{"),
+                std_macro("section", "*[{"),
+                std_macro("subsection", "*[{"),
+                std_macro("subsubsection", "*[{"),
+                std_macro("pagagraph", "*[{"),
+                std_macro("subparagraph", "*[{"),
+                std_macro("bibliography", "{"),
+                std_macro("emph", False, 1),
+                MacroSpec(
+                    "textrm",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "textit",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "textbf",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "textmd",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "textsc",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "textsf",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "textsl",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "texttt",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "textup",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "text",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                std_macro("mathrm", False, 1),  # only allowed in math mode anyway
+                std_macro("mathbb", False, 1),  # only allowed in math mode anyway
+                std_macro("mathbf", False, 1),
+                std_macro("mathit", False, 1),
+                std_macro("mathsf", False, 1),
+                std_macro("mathtt", False, 1),
+                std_macro("mathcal", False, 1),
+                std_macro("mathscr", False, 1),
+                std_macro("mathfrak", False, 1),
+                std_macro("label", False, 1),
+                std_macro("ref", False, 1),
+                std_macro("autoref", False, 1),
+                std_macro("cref", False, 1),
+                std_macro("Cref", False, 1),
+                std_macro("eqref", False, 1),
+                std_macro("url", False, 1),
+                std_macro("hypersetup", False, 1),
+                std_macro("footnote", True, 1),
+                std_macro("keywords", False, 1),
+                std_macro("hphantom", True, 1),
+                std_macro("vphantom", True, 1),
+                std_macro("'", False, 1),
+                std_macro("`", False, 1),
+                std_macro('"', False, 1),
+                std_macro("c", False, 1),
+                std_macro("^", False, 1),
+                std_macro("~", False, 1),
+                std_macro("H", False, 1),
+                std_macro("k", False, 1),
+                std_macro("=", False, 1),
+                std_macro("b", False, 1),
+                std_macro(".", False, 1),
+                std_macro("d", False, 1),
+                std_macro("r", False, 1),
+                std_macro("u", False, 1),
+                std_macro("v", False, 1),
+                MacroSpec(
+                    "ensuremath",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[True]),
+                ),
+                std_macro("not", False, 1),
+                std_macro("vec", False, 1),
+                std_macro("dot", False, 1),
+                std_macro("hat", False, 1),
+                std_macro("check", False, 1),
+                std_macro("breve", False, 1),
+                std_macro("acute", False, 1),
+                std_macro("grave", False, 1),
+                std_macro("tilde", False, 1),
+                std_macro("bar", False, 1),
+                std_macro("ddot", False, 1),
+                std_macro("frac", False, 2),
+                std_macro("nicefrac", False, 2),
+                std_macro("sqrt", True, 1),
+                MacroSpec("overline", "{"),
+                MacroSpec("underline", "{"),
+                MacroSpec("widehat", "{"),
+                MacroSpec("widetilde", "{"),
+                MacroSpec("wideparen", "{"),
+                MacroSpec("overleftarrow", "{"),
+                MacroSpec("overrightarrow", "{"),
+                MacroSpec("overleftrightarrow", "{"),
+                MacroSpec("underleftarrow", "{"),
+                MacroSpec("underrightarrow", "{"),
+                MacroSpec("underleftrightarrow", "{"),
+                MacroSpec("overbrace", "{"),
+                MacroSpec("underbrace", "{"),
+                MacroSpec("overgroup", "{"),
+                MacroSpec("undergroup", "{"),
+                MacroSpec("overbracket", "{"),
+                MacroSpec("underbracket", "{"),
+                MacroSpec("overlinesegment", "{"),
+                MacroSpec("underlinesegment", "{"),
+                MacroSpec("overleftharpoon", "{"),
+                MacroSpec("overrightharpoon", "{"),
+                MacroSpec("xleftarrow", "[{"),
+                MacroSpec("xrightarrow", "[{"),
+                std_macro("ket", False, 1),
+                std_macro("bra", False, 1),
+                std_macro("braket", False, 2),
+                std_macro("ketbra", False, 2),
+                std_macro("texorpdfstring", False, 2),
+                # xcolor commands
+                MacroSpec("definecolor", "[{{{"),
+                MacroSpec("providecolor", "[{{{"),
+                MacroSpec("colorlet", "[{[{"),
+                MacroSpec("color", "[{"),
+                MacroSpec("textcolor", "[{{"),
+                MacroSpec("pagecolor", "[{"),
+                MacroSpec("nopagecolor", ""),
+                MacroSpec("colorbox", "[{{"),
+                MacroSpec("fcolorbox", "[{[{{"),
+                MacroSpec("boxframe", "{{{"),
+                MacroSpec("rowcolors", "*[{{{"),
+            ],
+            "environments": [
+                # NOTE: Starred variants (as in \begin{equation*}) are not specified as
+                # for macros with an argspec='*'.  Rather, we need to define a separate
+                # spec for the starred variant as the star really is part of the
+                # environment name.  If you specify argspec='*', the parser will try to
+                # look for an expression of the form '\begin{equation}*'
+                std_environment("figure", "["),
+                std_environment("figure*", "["),
+                std_environment("table", "["),
+                std_environment("table*", "["),
+                std_environment("abstract", None),
+                std_environment("tabular", "{"),
+                std_environment("tabular*", "{{"),
+                std_environment("tabularx", "{[{"),
+                std_environment("array", "[{"),
+                std_environment("equation", None, is_math_mode=True),
+                std_environment("equation*", None, is_math_mode=True),
+                std_environment("eqnarray", None, is_math_mode=True),
+                std_environment("eqnarray*", None, is_math_mode=True),
+                # AMS environments
+                std_environment("align", None, is_math_mode=True),
+                std_environment("align*", None, is_math_mode=True),
+                std_environment("gather", None, is_math_mode=True),
+                std_environment("gather*", None, is_math_mode=True),
+                std_environment("flalign", None, is_math_mode=True),
+                std_environment("flalign*", None, is_math_mode=True),
+                std_environment("multline", None, is_math_mode=True),
+                std_environment("multline*", None, is_math_mode=True),
+                std_environment("alignat", "{", is_math_mode=True),
+                std_environment("alignat*", "{", is_math_mode=True),
+                std_environment("split", None, is_math_mode=True),
+            ],
+            "specials": [
+                std_specials("&"),
+                # TODO --- for this, we need to parse their argument but don't use
+                #          the standard args parser because we need to be able to
+                #          accept arguments like "x_\mathrm{initial}"
+                #
+                # std_specials('^'),
+                # std_specials('_'),
+            ],
+        },
+    ),
+    #
+    # CATEGORY: nonascii-specials
+    #
+    (
+        "nonascii-specials",
+        {
+            "macros": [],
+            "environments": [],
+            "specials": [
+                std_specials("~"),
+                # cf. https://tex.stackexchange.com/a/439652/32188 "fake ligatures":
+                std_specials("``"),
+                std_specials("''"),
+                std_specials("--"),
+                std_specials("---"),
+                std_specials("!`"),
+                std_specials("?`"),
+            ],
+        },
+    ),
+    #
+    # CATEGORY: verbatim
+    #
+    (
+        "verbatim",
+        {
+            "macros": [
+                MacroSpec(
+                    "verb",
+                    args_parser=VerbatimArgsParser(verbatim_arg_type="verb-macro"),
+                ),
+            ],
+            "environments": [
+                EnvironmentSpec(
+                    "verbatim",
+                    args_parser=VerbatimArgsParser(
+                        verbatim_arg_type="verbatim-environment"
+                    ),
+                ),
+            ],
+            "specials": [
+                # optionally users could include the specials "|" like in latex-doc
+                # for verbatim |\like \this|...
+            ],
+        },
+    ),
+    #
+    # CATEGORY: theorems
+    #
+    (
+        "theorems",
+        {
+            "macros": [],
+            "environments": [
+                std_environment("theorem", "["),
+                std_environment("proposition", "["),
+                std_environment("lemma", "["),
+                std_environment("corollary", "["),
+                std_environment("definition", "["),
+                std_environment("conjecture", "["),
+                std_environment("remark", "["),
+                #
+                std_environment("proof", "["),
+                # short names
+                std_environment("thm", "["),
+                std_environment("prop", "["),
+                std_environment("lem", "["),
+                std_environment("cor", "["),
+                std_environment("conj", "["),
+                std_environment("rem", "["),
+                std_environment("defn", "["),
+            ],
+            "specials": [],
+        },
+    ),
+    #
+    # CATEGORY: enumitem
+    #
+    (
+        "enumitem",
+        {
+            "macros": [],
+            "environments": [
+                std_environment("enumerate", "["),
+                std_environment("itemize", "["),
+                std_environment("description", "["),
+            ],
+            "specials": [],
+        },
+    ),
+    #
+    # CATEGORY: natbib
+    #
+    (
+        "natbib",
+        {
+            "macros": [
+                std_macro("cite", "*[[{"),
+                std_macro("citet", "*[[{"),
+                std_macro("citep", "*[[{"),
+                std_macro("citealt", "*[[{"),
+                std_macro("citealp", "*[[{"),
+                std_macro("citeauthor", "*[[{"),
+                std_macro("citefullauthor", "[[{"),
+                std_macro("citeyear", "[[{"),
+                std_macro("citeyearpar", "[[{"),
+                std_macro("Citet", "*[[{"),
+                std_macro("Citep", "*[[{"),
+                std_macro("Citealt", "*[[{"),
+                std_macro("Citealp", "*[[{"),
+                std_macro("Citeauthor", "*[[{"),
+                std_macro("citetext", "{"),
+                std_macro("citenum", "{"),
+                std_macro("defcitealias", "{{"),
+                std_macro("citetalias", "[[{"),
+                std_macro("citepalias", "[[{"),
+            ],
+            "environments": [],
+            "specials": [],
+        },
+    ),
+    #
+    # CATEGORY: latex-ethuebung
+    #
+    (
+        "latex-ethuebung",
+        {
+            "macros": [
+                # ethuebung
+                std_macro("UebungLoesungFont", False, 1),
+                std_macro("UebungHinweisFont", False, 1),
+                std_macro("UebungExTitleFont", False, 1),
+                std_macro("UebungSubExTitleFont", False, 1),
+                std_macro("UebungTipsFont", False, 1),
+                std_macro("UebungLabel", False, 1),
+                std_macro("UebungSubLabel", False, 1),
+                std_macro("UebungLabelEnum", False, 1),
+                std_macro("UebungLabelEnumSub", False, 1),
+                std_macro("UebungSolLabel", False, 1),
+                std_macro("UebungHinweisLabel", False, 1),
+                std_macro("UebungHinweiseLabel", False, 1),
+                std_macro("UebungSolEquationLabel", False, 1),
+                std_macro("UebungTipsLabel", False, 1),
+                std_macro("UebungTipsEquationLabel", False, 1),
+                std_macro("UebungsblattTitleSeries", False, 1),
+                std_macro("UebungsblattTitleSolutions", False, 1),
+                std_macro("UebungsblattTitleTips", False, 1),
+                std_macro("UebungsblattNumber", False, 1),
+                std_macro("UebungsblattTitleFont", False, 1),
+                std_macro("UebungTitleCenterVSpacing", False, 1),
+                std_macro("UebungAttachedSolutionTitleTop", False, 1),
+                std_macro("UebungAttachedSolutionTitleFont", False, 1),
+                std_macro("UebungAttachedSolutionTitle", False, 1),
+                std_macro("UebungTextAttachedSolution", False, 1),
+                std_macro("UebungDueByLabel", False, 1),
+                std_macro("UebungDueBy", False, 1),
+                std_macro("UebungLecture", False, 1),
+                std_macro("UebungProf", False, 1),
+                std_macro("UebungLecturer", False, 1),
+                std_macro("UebungSemester", False, 1),
+                std_macro("UebungLogoFile", False, 1),
+                std_macro("UebungLanguage", False, 1),
+                std_macro("UebungStyle", False, 1),
+                #
+                std_macro("uebung", "{["),
+                std_macro("exercise", "{["),
+                std_macro("keywords", False, 1),
+                std_macro("subuebung", False, 1),
+                std_macro("subexercise", False, 1),
+                std_macro("pdfloesung", True, 1),
+                std_macro("pdfsolution", True, 1),
+                std_macro("exenumfulllabel", False, 1),
+                std_macro("hint", False, 1),
+                std_macro("hints", False, 1),
+                std_macro("hinweis", False, 1),
+                std_macro("hinweise", False, 1),
+            ],
+            "environments": [],
+            "specials": [],
+        },
+    ),
+]

good_ai/src/sus/external/pylatexenc/macrospec/__init__.py

@@ -0,0 +1,785 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2019 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+r"""
+Provides classes and helper functions to describe a LaTeX context of known
+macros and environments, specifying how they should be parsed by
+:py:mod:`pylatexenc.latexwalker`.
+
+.. versionadded:: 2.0
+
+   The entire module :py:mod:`pylatexenc.macrospec` was introduced in
+   `pylatexenc 2.0`.
+"""
+
+
+import sys
+
+if sys.version_info.major > 2:
+    # Py3
+    def unicode(s):
+        return s
+
+    _basestring = str
+    _str_from_unicode = lambda x: x
+    _unicode_from_str = lambda x: x
+else:
+    # Py2
+    _basestring = basestring
+    _str_from_unicode = lambda x: unicode(x).encode("utf-8")
+    _unicode_from_str = lambda x: x.decode("utf-8")
+
+
+# ------------------------------------------------------------------------------
+
+from ._argparsers import (
+    MacroStandardArgsParser,
+    ParsedMacroArgs,
+    ParsedVerbatimArgs,
+    VerbatimArgsParser,
+)
+
+# ------------------------------------------------------------------------------
+
+
+class MacroSpec(object):
+    r"""
+    Stores the specification of a macro.
+
+    This stores the macro name and instructions on how to parse the macro
+    arguments.
+
+    .. py:attribute:: macroname
+
+       The name of the macro, without the leading backslash.
+
+    .. py:attribute:: args_parser
+
+       The parser instance that can understand this macro's arguments.  For
+       standard LaTeX macros this is usually a
+       :py:class:`MacroStandardArgsParser` instance.
+
+       If you specify a string, then for convenience this is interpreted as an
+       argspec argument for :py:class:`MacroStandardArgsParser` and such an
+       instance is automatically created.
+    """
+
+    def __init__(self, macroname, args_parser=MacroStandardArgsParser(), **kwargs):
+        super(MacroSpec, self).__init__(**kwargs)
+        self.macroname = macroname
+        if isinstance(args_parser, _basestring):
+            self.args_parser = MacroStandardArgsParser(args_parser)
+        else:
+            self.args_parser = args_parser
+
+    def parse_args(self, *args, **kwargs):
+        r"""
+        Shorthand for calling the :py:attr:`args_parser`\ 's `parse_args()` method.
+        See :py:class:`MacroStandardArgsParser`.
+        """
+        return self.args_parser.parse_args(*args, **kwargs)
+
+    def __repr__(self):
+        return "MacroSpec(macroname=%r, args_parser=%r)" % (
+            self.macroname,
+            self.args_parser,
+        )
+
+
+class EnvironmentSpec(object):
+    r"""
+    Stores the specification of a LaTeX environment.
+
+    This stores the environment name and instructions on how to parse any
+    arguments provided after ``\begin{environment}<args>``.
+
+    .. py:attribute:: environmentname
+
+       The name of the environment, i.e., the argument of ``\begin{...}`` and
+       ``\end{...}``.
+
+    .. py:attribute:: args_parser
+
+       The parser instance that can understand this environment's arguments.
+       For standard LaTeX environment this is usually a
+       :py:class:`MacroStandardArgsParser` instance.
+
+       If you specify a string, then for convenience this is interpreted as an
+       argspec argument for :py:class:`MacroStandardArgsParser` and such an
+       instance is automatically created.
+
+    .. py:attribute:: is_math_mode
+
+       A boolean that indicates whether or not the contents is to be interpreted
+       in Math Mode.  This would be True for environments like
+       ``\begin{equation}``, ``\begin{align}``, etc., but False for
+       ``\begin{figure}``, etc.
+
+    .. note::
+
+       Starred variants of environments (as in ``\begin{equation*}``) must not
+       be specified using an argspec as for macros (e.g., `argspec='*'`).
+       Rather, we need to define a separate environment spec for the starred
+       variant with the star in the name itself (``EnvironmentSpec('equation*',
+       None)``) because the star really is part of the environment name.  If you
+       happened to use ``EnvironmentSpec('equation', '*')``, then the parser
+       would recognize the expression ``\begin{equation}*`` but not
+       ``\begin{equation*}``.
+    """
+
+    def __init__(
+        self,
+        environmentname,
+        args_parser=MacroStandardArgsParser(),
+        is_math_mode=False,
+        **kwargs
+    ):
+        super(EnvironmentSpec, self).__init__(**kwargs)
+        self.environmentname = environmentname
+        if isinstance(args_parser, _basestring):
+            self.args_parser = MacroStandardArgsParser(args_parser)
+        else:
+            self.args_parser = args_parser
+        self.is_math_mode = is_math_mode
+
+    def parse_args(self, *args, **kwargs):
+        r"""
+        Shorthand for calling the :py:attr:`args_parser`\ 's `parse_args()` method.
+        See :py:class:`MacroStandardArgsParser`.
+        """
+        return self.args_parser.parse_args(*args, **kwargs)
+
+    def __repr__(self):
+        return (
+            "EnvironmentSpec(environmentname=%r, args_parser=%r, is_math_mode=%r)"
+            % (self.environmentname, self.args_parser, self.is_math_mode)
+        )
+
+
+class SpecialsSpec(object):
+    r"""
+    Specification of a LaTeX "special char sequence": an active char, a
+    ligature, or some other non-macro char sequence that has a special meaning.
+
+    For instance, '&', '~', and '``' are considered as "specials".
+
+    .. py:attribute:: specials_chars
+
+       The string (one or several characters) that has a special meaning. E.g.,
+       '&', '~', '``', etc.
+
+    .. py:attribute:: args_parser
+
+       A parser (e.g. :py:class:`MacroStandardArgsParser`) that is invoked when
+       the specials is encountered.  Can/should be set to `None` if the specials
+       should not parse any arguments (e.g. '~').
+    """
+
+    def __init__(self, specials_chars, args_parser=None, **kwargs):
+        super(SpecialsSpec, self).__init__(**kwargs)
+        self.specials_chars = specials_chars
+        self.args_parser = args_parser
+
+    def parse_args(self, *args, **kwargs):
+        r"""
+        Basically a shorthand for calling the :py:attr:`args_parser`\ 's
+        `parse_args()` method.  See :py:class:`MacroStandardArgsParser`.
+
+        If however the py:attr:`args_parser` attribute is `None`, then this
+        method returns `None`.
+        """
+        if self.args_parser is None:
+            return None
+        return self.args_parser.parse_args(*args, **kwargs)
+
+    def __repr__(self):
+        return "SpecialsSpec(specials_chars=%r, args_parser=%r)" % (
+            self.specials_chars,
+            self.args_parser,
+        )
+
+
+# ------------------------------------------------------------------------------
+
+
+def std_macro(macname, *args, **kwargs):
+    r"""
+    Return a macro specification for the given macro.  Syntax::
+
+      spec = std_macro(macname, argspec)
+      #  or
+      spec = std_macro(macname, optarg, numargs)
+      #  or
+      spec = std_macro( (macname, argspec), )
+      #  or
+      spec = std_macro( (macname, optarg, numargs), )
+      #  or
+      spec = std_macro( spec ) # spec is already a `MacroSpec` -- no-op
+
+    - `macname` is the name of the macro, without the leading backslash.
+
+    - `argspec` is a string either characters "\*", "{" or "[", in which star
+      indicates an optional asterisk character (e.g. starred macro variants),
+      each curly brace specifies a mandatory argument and each square bracket
+      specifies an optional argument in square brackets.  For example, "{{\*[{"
+      expects two mandatory arguments, then an optional star, an optional
+      argument in square brackets, and then another mandatory argument.
+
+      `argspec` may also be `None`, which is the same as ``argspec=''``.
+
+    - `optarg` may be one of `True`, `False`, or `None`, corresponding to these
+      possibilities:
+
+      + if `True`, the macro expects as first argument an optional argument in
+        square brackets. Then, `numargs` specifies the number of additional
+        mandatory arguments to the command, given in usual curly braces (or
+        simply as one TeX token like a single macro)
+
+      + if `False`, the macro only expects a number of mandatory arguments given
+        by `numargs`. The mandatory arguments are given in usual curly braces
+        (or simply as one TeX token like a single macro)
+
+      + if `None`, then `numargs` is a string like `argspec` above.  I.e.,
+        ``std_macro(macname, None, argspec)`` is the same as
+        ``std_macro(macname, argspec)``.
+
+    - `numargs`: depends on `optarg`, see above.
+
+    To make environment specifications (:py:class:`EnvironmentSpec`) instead of
+    a macro specification, use the function :py:func:`std_environment()`
+    instead.
+
+    The helper function :py:func:`std_environment()` is a shorthand for calling
+    this function with additional keyword arguments.  An optional keyword
+    argument `make_environment_spec=True` to the present function may be
+    specified to return an `EnvironmentSpec` instead of a `MacroSpec`.  In this
+    case, you can further specify the `environment_is_math_mode=True|False` to
+    specify whether of not the environment represents a math mode.
+    """
+
+    if isinstance(macname, tuple):
+        if len(args) != 0:
+            raise TypeError(
+                "No positional arguments expected if first argument is a tuple"
+            )
+        args = tuple(macname[1:])
+        macname = macname[0]
+
+    if isinstance(macname, MacroSpec):
+        if len(args) != 0:
+            raise TypeError(
+                "No positional arguments expected if first argument is a MacroSpec"
+            )
+        return macname
+
+    if isinstance(macname, EnvironmentSpec):
+        if len(args) != 0:
+            raise TypeError(
+                "No positional arguments expected if first argument is a EnvironmentSpec"
+            )
+        return macname
+
+    if len(args) == 1:
+        # std_macro(macname, argspec)
+        argspec = args[0]
+    elif len(args) != 2:
+        raise TypeError(
+            "Wrong number of arguments for std_macro, macname={!r}, args={!r}".format(
+                macname, args
+            )
+        )
+    elif not args[0] and isinstance(args[1], _basestring):
+        # argspec given in numargs
+        argspec = args[1]
+    else:
+        argspec = ""
+        if args[0]:
+            argspec = "["
+        argspec += "{" * args[1]
+
+    if kwargs.get("make_environment_spec", False):
+        return EnvironmentSpec(
+            macname,
+            args_parser=MacroStandardArgsParser(argspec),
+            is_math_mode=kwargs.get("environment_is_math_mode", False),
+        )
+    return MacroSpec(macname, args_parser=MacroStandardArgsParser(argspec))
+
+
+def std_environment(envname, *args, **kwargs):
+    r"""
+    Return an environment specification for the given environment.  Syntax::
+
+      spec = std_environment(envname, argspec, is_math_mode=True|False)
+      #  or
+      spec = std_environment(envname, optarg, numargs, is_math_mode=True|False)
+      #  or
+      spec = std_environment( (envname, argspec), is_math_mode=True|False)
+      #  or
+      spec = std_environment( (envname, optarg, numargs), is_math_mode=True|False)
+      #  or
+      spec = std_environment( spec ) # spec is already a `EnvironmentSpec` -- no-op
+
+    - `envname` is the name of the environment, i.e., the argument to
+      ``\begin{...}``.
+
+    - `argspec` is a string either characters "\*", "{" or "[", in which star
+      indicates an optional asterisk character (e.g. starred environment
+      variants), each curly brace specifies a mandatory argument and each square
+      bracket specifies an optional argument in square brackets.  For example,
+      "{{\*[{" expects two mandatory arguments, then an optional star, an
+      optional argument in square brackets, and then another mandatory argument.
+
+      `argspec` may also be `None`, which is the same as ``argspec=''``.
+
+    .. note::
+
+       See :py:class:`EnvironmentSpec` for an important remark about starred
+       variants for environments.  TL;DR: a starred verison of an environment is
+       defined as a separate `EnvironmentSpec` with the star in the name and
+       *not* using an ``argspec='*'``.
+
+    - `optarg` may be one of `True`, `False`, or `None`, corresponding to these
+      possibilities:
+
+      + if `True`, the environment expects as first argument an optional argument in
+        square brackets. Then, `numargs` specifies the number of additional
+        mandatory arguments to the command, given in usual curly braces (or
+        simply as one TeX token like a single environment)
+
+      + if `False`, the environment only expects a number of mandatory arguments given
+        by `numargs`. The mandatory arguments are given in usual curly braces
+        (or simply as one TeX token like a single environment)
+
+      + if `None`, then `numargs` is a string like `argspec` above.  I.e.,
+        ``std_environment(envname, None, argspec)`` is the same as
+        ``std_environment(envname, argspec)``.
+
+    - `numargs`: depends on `optarg`, see above.
+
+    - `is_math_mode`: if set to True, then the environment represents a math
+      mode environment (e.g., 'equation', 'align', 'gather', etc.), i.e., whose
+      contents should be parsed in an appropriate math mode.  Note that
+      `is_math_mode` *must* be given as a keyword argument, in contrast to all
+      other arguments which must be positional (non-keyword) arguments.
+    """
+    is_math_mode = kwargs.pop("is_math_mode", False)
+    kwargs2 = dict(kwargs)
+    kwargs2.update(make_environment_spec=True, environment_is_math_mode=is_math_mode)
+    return std_macro(envname, *args, **kwargs2)
+
+
+def std_specials(specials_chars):
+    r"""
+    Return a latex specials specification for the given character sequence.  Syntax::
+
+      spec = std_specials(specials_chars)
+
+    where `specials_chars` is the sequence of characters that has a special
+    LaTeX meaning, e.g. ``&`` or ``''``.
+
+    This helper function only allows to create specs for simple specials without
+    any argument parsing.  For more complicated specials, you can instantiate a
+    :py:class:`SpecialsSpec` directly.
+    """
+    return SpecialsSpec(specials_chars, args_parser=None)
+
+
+# ------------------------------------------------------------------------------
+
+
+class LatexContextDb(object):
+    r"""
+    Store a database of specifications of known macros, environments, and other
+    latex specials.  This might be, e.g., how many arguments a macro accepts, or
+    how to determine the text representation of a macro or environment.
+
+    When used with :py:class:`pylatexenc.latexwalker.LatexWalker`, the
+    specifications describe mostly rules for parsing arguments of macros and
+    environments, and which sequences of characters to consider as "latex
+    specials".  Specifications for macros, environments, and other specials are
+    stored as :py:class:`MacroSpec`, :py:class:`EnvironmentSpec`, and
+    :py:class:`SpecialsSpec` instances, respectively.
+    When used with :py:class:`pylatexenc.latex2text.LatexNodes2Text`, the
+    specifications for macros, environments, and other specials are stored as
+    :py:class:`pylatexenc.latex2text.MacroTextSpec` ,
+    :py:class:`pylatexenc.latex2text.EnvironmentTextSpec`, and
+    :py:class:`pylatexenc.latex2text.SpecialsTextSpec` instances, respectively.
+
+    In fact, the objects stored in this database may be of any type, except that
+    macro specifications must have an attribute `macroname`, environment
+    specifications must have an attribute `environmentname`, and specials
+    specification must have an attribute `specials_chars`.
+
+    The `LatexContextDb` instance is meant to be (pseudo-)immutable.  Once
+    constructed and all the definitions added with
+    :py:meth:`add_context_category()`, one should refrain from modifying it
+    directly after providing it to, e.g., a
+    :py:class:`~pylatexenc.latexwalker.LatexWalker` object.  The reason is that
+    the latex walker keeps track of what the latex context was when parsing
+    nodes, and modifying the context will modify that stored information, too.
+    Instead of being tempted to modify the object, create a new one with
+    :py:meth:`filter_context()`.
+
+    See :py:func:`pylatexenc.latexwalker.get_default_latex_context_db()` for the
+    default latex context for `latexwalker` with a default collection of known
+    latex macros and environments.
+    See :py:func:`pylatexenc.latex2text.get_default_latex_context_db()` for the
+    default latex context for `latex2text` with a set of text replacements for a
+    collection of known macros and environments.
+    """
+
+    def __init__(self, **kwargs):
+        super(LatexContextDb, self).__init__(**kwargs)
+
+        self.category_list = []
+        self.d = {}
+
+        self.unknown_macro_spec = None
+        self.unknown_environment_spec = None
+        self.unknown_specials_spec = None
+
+    def add_context_category(
+        self,
+        category,
+        macros=[],
+        environments=[],
+        specials=[],
+        prepend=False,
+        insert_before=None,
+        insert_after=None,
+    ):
+        r"""
+        Register a category of macro and environment specifications in the context
+        database.
+
+        The category name `category` must not already exist in the database.
+
+        The argument `macros` is an iterable (e.g., a list) of macro
+        specification objects.  The argument `environments` is an iterable
+        (e.g., a list) of environment spec objects.  Similarly, the `specials`
+        argument is an iterable of latex specials spec instances.
+
+        If you specify `prepend=True`, then macro and environment lookups will
+        prioritize this category over other categories.  Categories are normally
+        searched for in the order they are registered to the database; if you
+        specify `prepend=True`, then the new category is prepended to the
+        existing list so that it is searched first.
+
+        If `insert_before` is not `None`, then it must be a string; the
+        definitions are inserted in the category list immediately before the
+        given category name, or at the beginning of the list if the given
+        category doesn't exist.  If `insert_after` is not `None`, then it must
+        be a string; the definitions are inserted in the category list
+        immediately after the given category name, or at the end of the list if
+        the given category doesn't exist.
+
+        You may only specify one of `prepend=True`, `insert_before='...'` or
+        `insert_after='...'`.
+        """
+
+        if category in self.category_list:
+            raise ValueError(
+                "Category {} is already registered in the context database".format(
+                    category
+                )
+            )
+
+        # ensure only one of these options is set
+        if len([x for x in (prepend, insert_before, insert_after) if x]) > 1:
+            raise TypeError(
+                "add_context_category(): You may only specify one of "
+                "prepend=True, insert_before=... or insert_after=..."
+            )
+
+        if prepend:
+            self.category_list.insert(0, category)
+        elif insert_before:
+            if insert_before in self.category_list:
+                i = self.category_list.index(insert_before)
+            else:
+                i = 0
+            self.category_list.insert(i, category)
+        elif insert_after:
+            if insert_after in self.category_list:
+                i = (
+                    self.category_list.index(insert_after) + 1
+                )  # insert after found category
+            else:
+                i = len(self.category_list)
+            self.category_list.insert(i, category)
+        else:
+            self.category_list.append(category)
+
+        self.d[category] = {
+            "macros": dict((m.macroname, m) for m in macros),
+            "environments": dict((e.environmentname, e) for e in environments),
+            "specials": dict((s.specials_chars, s) for s in specials),
+        }
+
+    def set_unknown_macro_spec(self, macrospec):
+        r"""
+        Set the macro spec to use when encountering a macro that is not in the
+        database.
+        """
+        self.unknown_macro_spec = macrospec
+
+    def set_unknown_environment_spec(self, environmentspec):
+        r"""
+        Set the environment spec to use when encountering a LaTeX environment that
+        is not in the database.
+        """
+        self.unknown_environment_spec = environmentspec
+
+    def set_unknown_specials_spec(self, specialsspec):
+        r"""
+        Set the latex specials spec to use when encountering a LaTeX environment
+        that is not in the database.
+        """
+        self.unknown_specials_spec = specialsspec
+
+    def categories(self):
+        r"""
+        Return a list of valid category names that are registered in the current
+        database context.
+        """
+        return list(self.category_list)
+
+    def get_macro_spec(self, macroname):
+        r"""
+        Look up a macro specification by macro name.  The macro name is searched for
+        in all categories one by one and the first match is returned.
+
+        Returns a macro spec instance that matches the given `macroname`.  If
+        the macro name was not found, we return the default macro specification
+        set by :py:meth:`set_unknown_macro_spec()` or `None` if no such spec was
+        set.
+        """
+        for cat in self.category_list:
+            # search categories in the given order
+            if macroname in self.d[cat]["macros"]:
+                return self.d[cat]["macros"][macroname]
+        return self.unknown_macro_spec
+
+    def get_environment_spec(self, environmentname):
+        r"""
+        Look up an environment specification by environment name.  The environment
+        name is searched for in all categories one by one and the first match is
+        returned.
+
+        Returns the environment spec.  If the environment name was not found, we
+        return the default environment specification set by
+        :py:meth:`set_unknown_environment_spec()` or `None` if no such spec was
+        set.
+        """
+        for cat in self.category_list:
+            # search categories in the given order
+            if environmentname in self.d[cat]["environments"]:
+                return self.d[cat]["environments"][environmentname]
+        return self.unknown_environment_spec
+
+    def get_specials_spec(self, specials_chars):
+        r"""
+        Look up a "latex specials" specification by character sequence.  The
+        sequence name is searched for in all categories one by one and the first
+        match is returned.
+
+        If you are parsing a chunk of LaTeX code, you should use
+        :py:meth:`test_for_specials()` instead.  Unlike
+        :py:meth:`test_for_specials()`, :py:meth:`get_specials_spec()` returns
+        the first match regardless of matched length.  [Rationale: we only need
+        to worry about matching the longest specials sequence when parsing LaTeX
+        code.  Calling `get_specials_spec()` means one has already parsed the
+        sequence and one is looking up additional specs on it.]
+
+        Returns the specials spec.  If the latex specials was not found, we
+        return the default latex specials specification set by
+        :py:meth:`set_unknown_specials_spec()` or `None` if no such spec was
+        set.
+        """
+        for cat in self.category_list:
+            # search categories in the given order
+            if specials_chars in self.d[cat]["specials"]:
+                return self.d[cat]["specials"][specials_chars]
+        return self.unknown_specials_spec
+
+    def test_for_specials(self, s, pos, parsing_state=None):
+        r"""
+        Test the given position in the string for any LaTeX specials.  The lookup
+        proceeds by searching for in all categories one by one and the first
+        match is returned, except that the longest match accross all categories
+        is returned.  For instance, a match of '``' in a later category will
+        take precedence over a match of '`' in a earlier-searched category.
+
+        Returns a specials spec instance, or `None` if no specials are detected
+        at the position `pos`.
+        """
+        best_match_len = 0
+        best_match_s = None
+        for cat in self.category_list:
+            # search categories in the given order
+            for specials_chars in self.d[cat]["specials"].keys():
+                if len(specials_chars) > best_match_len and s.startswith(
+                    specials_chars, pos
+                ):
+                    best_match_s = self.d[cat]["specials"][specials_chars]
+                    best_match_len = len(specials_chars)
+
+        return best_match_s  # this is None if no match
+
+    def iter_macro_specs(self, categories=None):
+        r"""
+        Yield the macro specs corresponding to all macros in the given categories.
+
+        If `categories` is `None`, then the known macro specs from all
+        categories are provided in one long iterable sequence.  Otherwise,
+        `categories` should be a list or iterable of category names (e.g.,
+        'latex-base') of macro specs to return.
+
+        The macro specs from the different categories specified are concatenated
+        into one long sequence which is yielded spec by spec.
+        """
+
+        if categories is None:
+            categories = self.category_list
+
+        for c in categories:
+            if c not in self.category_list:
+                raise ValueError(
+                    "Invalid latex macro spec db category: {!r} (Expected one of {!r})".format(
+                        c, self.category_list
+                    )
+                )
+            for spec in self.d[c]["macros"].values():
+                yield spec
+
+    def iter_environment_specs(self, categories=None):
+        r"""
+        Yield the environment specs corresponding to all environments in the given
+        categories.
+
+        If `categories` is `None`, then the known environment specs from all
+        categories are provided in one long iterable sequence.  Otherwise,
+        `categories` should be a list or iterable of category names (e.g.,
+        'latex-base') of environment specs to return.
+
+        The environment specs from the different categories specified are
+        concatenated into one long sequence which is yielded spec by spec.
+        """
+
+        if categories is None:
+            categories = self.category_list
+
+        for c in categories:
+            if c not in self.category_list:
+                raise ValueError(
+                    "Invalid latex environment spec db category: {!r} (Expected one of {!r})".format(
+                        c, self.category_list
+                    )
+                )
+            for spec in self.d[c]["environments"].values():
+                yield spec
+
+    def iter_specials_specs(self, categories=None):
+        r"""
+        Yield the specials specs corresponding to all environments in the given
+        categories.
+
+        If `categories` is `None`, then the known specials specs from all
+        categories are provided in one long iterable sequence.  Otherwise,
+        `categories` should be a list or iterable of category names (e.g.,
+        'latex-base') of specials specs to return.
+
+        The specials specs from the different categories specified are
+        concatenated into one long sequence which is yielded spec by spec.
+        """
+
+        if categories is None:
+            categories = self.category_list
+
+        for c in categories:
+            if c not in self.category_list:
+                raise ValueError(
+                    "Invalid latex environment spec db category: {!r} (Expected one of {!r})".format(
+                        c, self.category_list
+                    )
+                )
+            for spec in self.d[c]["specials"].values():
+                yield spec
+
+    def filter_context(self, keep_categories=[], exclude_categories=[], keep_which=[]):
+        r"""
+        Return a new :py:class:`LatexContextDb` instance where we only keep
+        certain categories of macro and environment specifications.
+
+        If `keep_categories` is set to a nonempty list, then the returned
+        context will not contain any definitions that do not correspond to the
+        specified categories.
+
+        If `exclude_categories` is set to a nonempty list, then the returned
+        context will not contain any definitions that correspond to the
+        specified categories.
+
+        It is explicitly fine to have category names in `keep_categories` and
+        `exclude_categories` that don't exist in the present object
+        (cf. :py:meth:`categories()`).
+
+        The argument `keep_which`, if non-empty, specifies which definitions to
+        keep.  It should be a subset of the list ['macros', 'environments',
+        'specials'].
+
+        The returned context will make a copy of the dictionaries that store the
+        macro and environment specifications, but the specification classes (and
+        corresponding argument parsers) might correspond to the same instances.
+        I.e., the returned context is not a full deep copy.
+        """
+
+        new_context = LatexContextDb()
+
+        new_context.unknown_macro_spec = self.unknown_macro_spec
+        new_context.unknown_environment_spec = self.unknown_environment_spec
+        new_context.unknown_specials_spec = self.unknown_specials_spec
+
+        keep_macros = not keep_which or "macros" in keep_which
+        keep_environments = not keep_which or "environments" in keep_which
+        keep_specials = not keep_which or "specials" in keep_which
+
+        for cat in self.category_list:
+            if keep_categories and cat not in keep_categories:
+                continue
+            if exclude_categories and cat in exclude_categories:
+                continue
+
+            # include this category
+            new_context.add_context_category(
+                cat,
+                macros=self.d[cat]["macros"].values() if keep_macros else [],
+                environments=self.d[cat]["environments"].values()
+                if keep_environments
+                else [],
+                specials=self.d[cat]["specials"].values() if keep_specials else [],
+            )
+
+        return new_context

good_ai/src/sus/external/pylatexenc/macrospec/_argparsers.py

@@ -0,0 +1,497 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2019 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+
+# Internal module. Internal API may move, disappear or otherwise change at any
+# time and without notice.
+
+
+import sys
+
+if sys.version_info.major > 2:
+    # Py3
+    def unicode(s):
+        return s
+
+    _basestring = str
+    _str_from_unicode = lambda x: x
+    _unicode_from_str = lambda x: x
+else:
+    # Py2
+    _basestring = basestring
+    _str_from_unicode = lambda x: unicode(x).encode("utf-8")
+    _unicode_from_str = lambda x: x.decode("utf-8")
+
+
+class ParsedMacroArgs(object):
+    r"""
+    Parsed representation of macro arguments.
+
+    The base class provides a simple way of storing the arguments as a list of
+    parsed nodes.
+
+    This base class can be subclassed to store additional information and
+    provide more advanced APIs to access macro arguments for certain categories
+    of macros.
+
+    Arguments:
+
+      - `argnlist` is a list of latexwalker nodes that represent macro
+        arguments.  If the macro arguments are too complicated to store in a
+        list, leave this as `None`.  (But then code that uses the latexwalker
+        must be aware of your own API to access the macro arguments.)
+
+        The difference between `argnlist` and the legacy `nodeargs` is that all
+        options, regardless of optional or mandatory, are stored in the list
+        `argnlist` with possible `None`\ 's at places where optional arguments
+        were not provided.  Previously, whether a first optional argument was
+        included in `nodeoptarg` or `nodeargs` depended on how the macro
+        specification was given.
+
+      - `argspec` is a string or a list that describes how each corresponding
+        argument in `argnlist` represents.  If the macro arguments are too
+        complicated to store in a list, leave this as `None`.  For standard
+        macros and parsed arguments this is a string with characters '*', '[',
+        '{' describing an optional star argument, an optional
+        square-bracket-delimited argument, and a mandatory argument.
+
+    Attributes:
+
+    .. py:attribute:: argnlist
+
+       The list of latexwalker nodes that was provided to the constructor
+
+    .. py:attribute:: argspec
+
+       Argument type specification provided to the constructor
+
+    .. py:attribute:: legacy_nodeoptarg_nodeargs
+
+       A tuple `(nodeoptarg, nodeargs)` that should be exposed as properties in
+       :py:class:`~pylatexenc.latexwalker.LatexMacroNode` to provide (as best as
+       possible) compatibility with pylatexenc < 2.
+
+       This is either `(<1st optional arg node>, <list of remaining args>)` if
+       the first argument is optional and all remaining args are mandatory; or
+       it is `(None, <list of args>)` for any other argument structure.
+    """
+
+    def __init__(self, argnlist=[], argspec="", **kwargs):
+        super(ParsedMacroArgs, self).__init__(**kwargs)
+
+        self.argnlist = argnlist
+        self.argspec = argspec
+
+        # for LatexMacroNode to provide some kind of compatibility with pylatexenc < 2
+        self.legacy_nodeoptarg_nodeargs = self._get_legacy_attribs(
+            self.argspec, self.argnlist
+        )
+
+    def _get_legacy_attribs(self, argspec, argnlist):
+        nskip = 0
+        while argspec.startswith("*"):
+            argspec = argspec[1:]
+            nskip += 1
+        if argspec[0:1] == "[" and all(x == "{" for x in argspec[1:]):
+            return (argnlist[nskip], argnlist[nskip + 1 :])
+        else:
+            return (None, argnlist)
+
+    def to_json_object(self):
+        r"""
+        Called when we export the node structure to JSON when running latexwalker in
+        command-line.
+
+        Return a representation of the current parsed arguments in an object,
+        typically a dictionary, that can easily be exported to JSON.  The object
+        may contain latex nodes and other parsed-argument objects, as we use a
+        custom JSON encoder that understands these types.
+
+        Subclasses may
+        """
+
+        return dict(
+            argspec=self.argspec,
+            argnlist=self.argnlist,
+        )
+
+    def __repr__(self):
+        return "{}(argspec={!r}, argnlist={!r})".format(
+            self.__class__.__name__, self.argspec, self.argnlist
+        )
+
+
+class MacroStandardArgsParser(object):
+    r"""
+    Parses the arguments to a LaTeX macro.
+
+    This class parses a simple macro argument specification with a specified
+    arrangement of optional and mandatory arguments.
+
+    This class also serves as base class for more advanced argument parsers
+    (e.g. for a ``\verb+...+`` macro argument parser).  In such cases,
+    subclasses should attempt to provide the most suitable `argspec` (and
+    `argnlist` for the corresponding :py:class:`ParsedMacroArgs`) for their use,
+    if appropriate, or set them to `None`.
+
+    Arguments:
+
+      - `argspec`: must be a string in which each character corresponds to an
+        argument.  The character '{' represents a mandatory argument (single
+        token or LaTeX group) and the character '[' denotes an optional argument
+        delimited by braces.  The character '\*' denotes a possible star char at
+        that position in the argument list, a corresponding
+        ``latexwalker.LatexCharsNode('*')`` (or `None` if no star) will be
+        inserted in the argument node list.  For instance, the string '\*{[[{'
+        would be suitable to specify the signature of the '\\newcommand' macro.
+
+        Currently, the argspec string may only contain the characters '\*', '{'
+        and '['.
+
+        The `argspec` may also be `None`, which is the same as specifying an
+        empty string.
+
+      - `optional_arg_no_space`: If set to `True`, then an optional argument
+        cannot have any whitespace between the preceeding tokens and the '['
+        character.  Set this to `True` in cases such as for ``\\`` in AMS-math
+        environments, where AMS apparently introduced a patch to prevent a
+        bracket on a new line after ``\\`` from being interpreted as the
+        optional argument to ``\\``.
+
+      - `args_math_mode`: Either `None`, or a list of the same length as
+        `argspec`.  If a list is given, then each item must be `True`, `False`,
+        or `None`.  The corresponding argument (cf. `argspec`) is then
+        respectively parsed in math mode (`True`), in text mode (`False`), or
+        with the mode unchanged (`None`).  If `args_math_mode` is `None`, then
+        all arguments are parsed in the same mode as the current mode.
+
+      - additional unrecognized keyword arguments are passed on to superclasses
+        in case of multiple inheritance
+
+    Attributes:
+
+    .. py:attribute:: argspec
+
+       Argument type specification provided to the constructor.
+
+    .. py:attribute:: optional_arg_no_space
+
+       See the corresponding constructor argument.
+
+    .. py:attribute:: args_math_mode
+
+       See the corresponding constructor argument.
+    """
+
+    def __init__(
+        self, argspec=None, optional_arg_no_space=False, args_math_mode=None, **kwargs
+    ):
+        super(MacroStandardArgsParser, self).__init__(**kwargs)
+        self.argspec = argspec if argspec else ""
+        self.optional_arg_no_space = optional_arg_no_space
+        self.args_math_mode = args_math_mode
+        # catch bugs, make sure that argspec is a string with only accepted chars
+        if not isinstance(self.argspec, _basestring) or not all(
+            x in "*[{" for x in self.argspec
+        ):
+            raise TypeError(
+                "argspec must be a string containing chars '*', '[', '{{' only: {!r}".format(
+                    self.argspec
+                )
+            )
+        # non-documented attribute that makes us ignore any leading '*'.  We use
+        # this to emulate pylatexenc 1.x behavior when using the MacrosDef()
+        # function explicitly
+        self._like_pylatexenc1x_ignore_leading_star = False
+
+    def parse_args(self, w, pos, parsing_state=None):
+        r"""
+        Parse the arguments encountered at position `pos` in the
+        :py:class:`~pylatexenc.latexwalker.LatexWalker` instance `w`.
+
+        You may override this function to provide custom parsing of complicated
+        macro arguments (say, ``\verb+...+``).  The method will be called by
+        keyword arguments, so the argument names should not be altered.
+
+        The argument `w` is the :py:class:`pylatexenc.latexwalker.LatexWalker`
+        object that is currently parsing LaTeX code.  You can call methods like
+        `w.get_goken()`, `w.get_latex_expression()` etc., to parse and read
+        arguments.
+
+        The argument `parsing_state` is the current parsing state in the
+        :py:class:`~pylatexenc.latexwalker.LatexWalker` (e.g., are we currently
+        in math mode?).  See doc for
+        :py:class:`~pylatexenc.latexwalker.ParsingState`.
+
+        This function should return a tuple `(argd, pos, len)` where:
+
+        - `argd` is a :py:class:`ParsedMacroArgs` instance, or an instance of a
+          subclass of :py:class:`ParsedMacroArgs`.  The base `parse_args()`
+          provided here returns a :py:class:`ParsedMacroArgs` instance.
+
+        - `pos` is the position of the first parsed content.  It should be the
+          same as the `pos` argument, except if there is whitespace at that
+          position in which case the returned `pos` would have to be the
+          position where the argument contents start.
+
+        - `len` is the length of the parsed expression.  You will probably want
+          to continue parsing stuff at the index `pos+len` in the string.
+        """
+
+        from .. import latexwalker
+
+        if parsing_state is None:
+            parsing_state = w.make_parsing_state()
+
+        argnlist = []
+
+        if self.args_math_mode is not None and len(self.args_math_mode) != len(
+            self.argspec
+        ):
+            raise ValueError(
+                "Invalid args_math_mode={!r} for argspec={!r}!".format(
+                    self.args_math_mode, self.argspec
+                )
+            )
+
+        def get_inner_parsing_state(j):
+            if self.args_math_mode is None:
+                return parsing_state
+            amm = self.args_math_mode[j]
+            if amm is None or amm == parsing_state.in_math_mode:
+                return parsing_state
+            if amm == True:
+                return parsing_state.sub_context(in_math_mode=True)
+            return parsing_state.sub_context(in_math_mode=False)
+
+        p = pos
+
+        if self._like_pylatexenc1x_ignore_leading_star:
+            # ignore any leading '*' character
+            tok = w.get_token(p)
+            if tok.tok == "char" and tok.arg == "*":
+                p = tok.pos + tok.len
+
+        for j, argt in enumerate(self.argspec):
+            if argt == "{":
+                (node, np, nl) = w.get_latex_expression(
+                    p, strict_braces=False, parsing_state=get_inner_parsing_state(j)
+                )
+                p = np + nl
+                argnlist.append(node)
+
+            elif argt == "[":
+
+                if self.optional_arg_no_space and p < len(w.s) and w.s[p].isspace():
+                    # don't try to read optional arg, we don't allow space
+                    argnlist.append(None)
+                    continue
+
+                optarginfotuple = w.get_latex_maybe_optional_arg(
+                    p, parsing_state=get_inner_parsing_state(j)
+                )
+                if optarginfotuple is None:
+                    argnlist.append(None)
+                    continue
+                (node, np, nl) = optarginfotuple
+                p = np + nl
+                argnlist.append(node)
+
+            elif argt == "*":
+                # possible star.
+                tok = w.get_token(p)
+                if tok.tok == "char" and tok.arg.startswith("*"):
+                    # has star
+                    argnlist.append(
+                        w.make_node(
+                            latexwalker.LatexCharsNode,
+                            parsing_state=get_inner_parsing_state(j),
+                            chars="*",
+                            pos=tok.pos,
+                            len=1,
+                        )
+                    )
+                    p = tok.pos + 1
+                else:
+                    argnlist.append(None)
+
+            else:
+                raise LatexWalkerError(
+                    "Unknown macro argument kind for macro: {!r}".format(argt)
+                )
+
+        parsed = ParsedMacroArgs(
+            argspec=self.argspec,
+            argnlist=argnlist,
+        )
+
+        return (parsed, pos, p - pos)
+
+    def __repr__(self):
+        return (
+            "{}(argspec={!r}, optional_arg_no_space={!r}, args_math_mode={!r})".format(
+                self.__class__.__name__,
+                self.argspec,
+                self.optional_arg_no_space,
+                self.args_math_mode,
+            )
+        )
+
+
+class ParsedVerbatimArgs(ParsedMacroArgs):
+    r"""
+    Parsed representation of arguments to LaTeX verbatim constructs, such as
+    ``\begin{verbatim}...\end{verbatim}`` or ``\verb|...|``.
+
+    Instances of `ParsedVerbatimArgs` are returned by the args parser
+    :py:class:`VerbatimArgsParser`.
+
+    Arguments:
+
+      - `verbatim_chars_node` --- a properly initialized
+        :py:class:`pylatexenc.latexwalker.LatexCharsNode` that stores the
+        verbatim text provided.  It is used to initialize the base class
+        :py:class:`ParsedMacroArgs` to expose a single mandatory argument with
+        the given verbatim text.  The `verbatim_text` attribute is initialized
+        from this node, too.
+
+      - `verbatim_delimiters` --- a 2-item tuple of characters used to delimit
+        the verbatim arguemnt (in case of a ``\verb+...+`` macro) or `None`.
+
+    Attributes:
+
+    .. py:attribute:: verbatim_text
+
+       The verbatim text that was provided
+
+    .. py:attribute:: verbatim_delimiters
+
+       If the verbatim text was specified as an argument to ``\verb$...$``, then
+       this is set to a 2-item tuple that specifies the begin and end
+       delimiters.  Otherwise, the attribute is `None`.
+    """
+
+    def __init__(self, verbatim_chars_node, verbatim_delimiters=None, **kwargs):
+
+        # provide argspec/argnlist to the parent class so that any code that is
+        # not "verbatim environment-aware" sees this simply as the argument to
+        # an empty verbatim environment
+        super(ParsedVerbatimArgs, self).__init__(
+            argspec="{", argnlist=[verbatim_chars_node], **kwargs
+        )
+
+        self.verbatim_text = verbatim_chars_node.chars
+        self.verbatim_delimiters = verbatim_delimiters
+
+    def __repr__(self):
+        return "{}(verbatim_text={!r}, verbatim_delimiters={!r})".format(
+            self.__class__.__name__, self.verbatim_text, self.verbatim_delimiters
+        )
+
+
+class VerbatimArgsParser(MacroStandardArgsParser):
+    r"""
+    Parses the arguments to various LaTeX "verbatim" constructs such as
+    ``\begin{verbatim}...\end{verbatim}`` environment or ``\verb+...+``.
+
+    This class also serves to illustrate how to write custom parsers for
+    complicated macro arguments.  See also :py:class:`MacroStandardArgsParser`.
+
+    Arguments:
+
+    .. py:attribute:: verbatim_arg_type
+
+      One of 'verbatim-environment' or 'verb-macro'.
+    """
+
+    def __init__(self, verbatim_arg_type, **kwargs):
+        super(VerbatimArgsParser, self).__init__(argspec="{", **kwargs)
+        self.verbatim_arg_type = verbatim_arg_type
+
+    def parse_args(self, w, pos, parsing_state=None):
+
+        from .. import latexwalker
+
+        if self.verbatim_arg_type == "verbatim-environment":
+            # simply scan the string until we find '\end{verbatim}'.  That's
+            # exactly how LaTeX processes it.
+            endverbpos = w.s.find(r"\end{verbatim}", pos)
+            if endverbpos == -1:
+                raise latexwalker.LatexWalkerParseError(
+                    s=w.s, pos=pos, msg=r"Cannot find matching \end{verbatim}"
+                )
+            # do NOT include the "\end{verbatim}", latexwalker will expect to
+            # see it:
+            len_ = endverbpos - pos
+
+            argd = ParsedVerbatimArgs(
+                verbatim_chars_node=w.make_node(
+                    latexwalker.LatexCharsNode,
+                    parsing_state=parsing_state,
+                    chars=w.s[pos : pos + len_],
+                    pos=pos,
+                    len=len_,
+                )
+            )
+            return (argd, pos, len_)
+
+        if self.verbatim_arg_type == "verb-macro":
+            # read the next nonwhitespace char. This is the delimiter of the
+            # argument
+            while w.s[pos].isspace():
+                pos += 1
+                if pos >= len(w.s):
+                    raise latexwalker.LatexWalkerParseError(
+                        s=w.s, pos=pos, msg=r"Missing argument to \verb command"
+                    )
+            verbdelimchar = w.s[pos]
+            beginpos = pos + 1
+            endpos = w.s.find(verbdelimchar, beginpos)
+            if endpos == -1:
+                raise latexwalker.LatexWalkerParseError(
+                    s=w.s,
+                    pos=pos,
+                    msg=r"End of stream reached while reading argument to \verb command",
+                )
+
+            verbarg = w.s[beginpos:endpos]
+
+            argd = ParsedVerbatimArgs(
+                verbatim_chars_node=w.make_node(
+                    latexwalker.LatexCharsNode,
+                    parsing_state=parsing_state,
+                    chars=verbarg,
+                    pos=beginpos,
+                    len=endpos - beginpos,
+                ),
+                verbatim_delimiters=(verbdelimchar, verbdelimchar),
+            )
+
+            return (argd, pos, endpos + 1 - pos)  # include delimiters in pos/len
+
+    def __repr__(self):
+        return "{}(verbatim_arg_type={!r})".format(
+            self.__class__.__name__, self.verbatim_arg_type
+        )

good_ai/src/sus/external/pylatexenc/version.py

@@ -0,0 +1,58 @@
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2021 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+
+#
+# Self-note: Checklist
+#
+#   1) First some checks:
+#
+#       - Set below in this file ' version_str = "X.Xb" ' (beta version for next
+#         release) for the following tests.
+#
+#       - tests pass: https://travis-ci.org/github/phfaist/pylatexenc
+#
+#       - LGTM looks good: https://lgtm.com/projects/g/phfaist/pylatexenc/
+#
+#       - python package creation works: (python setup.py sdist, pip install
+#         dist/pylatexenc-xxx.tar.gz)
+#
+#   2) update change log (doc/changes.rst)
+#
+#   3) bump version number here
+#
+#   4) git commit any remaining changes
+#
+#   5) " git tag vX.X -am '<message>' "
+#
+#   6) " git push && git push --tags "
+#
+#   7) on github.com, fill in release details with a summary of changes etc.
+#
+#   8) create the source package for PyPI (" python3 setup.py sdist ")
+#
+#   8) upload package to PyPI (twine upload dist/pylatexenc-X.X.tar.gz -r realpypi)
+#
+
+version_str = "2.10"

good_ai/src/sus/get_sentences.py

@@ -0,0 +1,18 @@
+from typing import List
+
+from .data import sentence_ending_punctuations
+from .nlp import nlp
+
+
+def get_sentences(text: str, ignore_partial: bool = False) -> List[str]:
+    doc = nlp(text)
+    possible_sentences = [s.text for s in doc.sents]
+
+    if ignore_partial:
+        possible_sentences = [
+            s
+            for s in possible_sentences
+            if s[0].isupper() and s[-1] in sentence_ending_punctuations
+        ]
+
+    return possible_sentences

good_ai/src/sus/language/__init__.py

@@ -0,0 +1,3 @@
+from .english_name_of_language import english_name_of_language
+from .is_english import is_english
+from .predict_language import predict_language

good_ai/src/sus/language/english_name_of_language.py

@@ -0,0 +1,10 @@
+from typing import Optional
+
+from langcodes import Language
+
+
+def english_name_of_language(language_code: Optional[str]) -> str:
+    if not language_code:
+        language_code = "und"
+
+    return Language.get(language_code).display_name()

good_ai/src/sus/language/is_english.py

@@ -0,0 +1,11 @@
+from typing import Optional
+
+from langcodes import standardize_tag, tag_distance
+
+
+def is_english(language_code: Optional[str]) -> bool:
+    if not language_code:
+        language_code = "und"
+
+    language_code = standardize_tag(language_code)
+    return tag_distance(language_code, "en") < 15

good_ai/src/sus/language/predict_language.py

@@ -0,0 +1,16 @@
+from typing import Optional
+
+from langcodes import Language
+from langdetect import LangDetectException, detect
+
+
+def predict_language(text: Optional[str]) -> str:
+    if not text:
+        return Language.make().to_tag()
+
+    try:
+        language_code = detect(text)
+    except LangDetectException:
+        return Language.make().to_tag()
+
+    return Language.get(language_code).to_tag()

good_ai/src/sus/lemmatize_text.py

@@ -0,0 +1,21 @@
+from typing import List
+
+from .lemmatize_token import lemmatize_token
+from .nlp import nlp
+
+
+def lemmatize_text(
+    text: str,
+    add_negation: bool = False,
+    add_part_of_speech: bool = False,
+) -> List[str]:
+    doc = nlp(text)
+
+    return [
+        lemmatize_token(
+            t,
+            add_negation=add_negation,
+            add_part_of_speech=add_part_of_speech,
+        )
+        for t in doc
+    ]

good_ai/src/sus/lemmatize_token.py

@@ -0,0 +1,20 @@
+from spacy.tokens import Token
+
+from .data import american_spellings
+
+
+def lemmatize_token(
+    token: Token,
+    add_negation: bool = False,
+    add_part_of_speech: bool = False,
+) -> str:
+    lemma = token.lemma_.lower()
+
+    lemma = american_spellings.get(lemma, lemma)
+
+    if add_part_of_speech:
+        lemma = f"{lemma}_{token.pos_}"
+    if add_negation and token._.negex:
+        lemma = f"NOT_{lemma}"
+
+    return lemma

good_ai/src/sus/match_names/__init__.py

@@ -0,0 +1 @@
+from .match_names import get_name_match_score, get_name_parts, match_names

good_ai/src/sus/match_names/config.py

@@ -0,0 +1,5 @@
+last_name_weight = 1.5
+first_name_weight = 0.9
+infixes_weight = 0.5
+initials_weight = 0.6
+match_threshold = 1.51

good_ai/src/sus/match_names/match_names.py

@@ -0,0 +1,162 @@
+import functools
+import re
+from typing import List, Optional, Tuple
+
+from ..clean import clean
+from .config import (
+    first_name_weight,
+    infixes_weight,
+    initials_weight,
+    last_name_weight,
+    match_threshold,
+)
+from .name_parts import NameParts
+
+
+def match_names(n1: Optional[str], n2: Optional[str]) -> bool:
+    score = get_name_match_score(n1, n2)
+    return score > match_threshold
+
+
+def get_name_match_score(n1: Optional[str], n2: Optional[str]) -> float:
+    if not n1 or not n2:
+        return 0
+
+    p1 = get_name_parts(n1)
+    p2 = get_name_parts(n2)
+
+    for f1 in p1.first_names:
+        for f2 in p2.first_names:
+            if f1[0] == f2[0]:
+                p1.initials = [i for i in p1.initials if i != f1[0]]
+                p2.initials = [i for i in p2.initials if i != f1[0]]
+
+    last_name_score = last_name_weight * _match_percentage(p1.last_names, p2.last_names)
+    first_name_score = first_name_weight * _match_percentage(
+        p1.first_names, p2.first_names
+    )
+    initials_score = initials_weight * _match_percentage(p1.initials, p2.initials)
+    infixes_score = min(0, infixes_weight * _match_percentage(p1.infixes, p2.infixes))
+
+    return last_name_score + first_name_score + initials_score + infixes_score
+
+
+def get_name_parts(name: str) -> NameParts:
+    result = _get_name_parts(name)
+    return result.copy()
+
+
+@functools.lru_cache(maxsize=None)
+def _get_name_parts(name: str) -> NameParts:
+    name = _clean_name(name)
+
+    first_names, name = _get_parenthesized_first_names(name)
+    initials, name = _get_initials(name)
+
+    last_names = []
+    infixes, name = _get_infixes(name)
+    if "," in name:
+        [last_name, *rest] = name.split(",")
+        rest_combined = " ".join(rest)
+        last_names.extend(last_name.split(" "))
+        first_names.extend(rest_combined.split(" "))
+    else:
+        parts = name.strip().split(" ")
+        if parts:
+            last_names.append(parts[-1])
+            parts.pop(-1)
+            first_names.extend(parts)
+        else:
+            last_names.append(name)
+
+    first_names = [f for f in first_names if f]
+    for f in first_names:
+        if all(f[0] != i for i in initials):
+            initials.append(f[0])
+
+    return NameParts(
+        first_names=first_names,
+        initials=initials,
+        infixes=infixes,
+        last_names=[
+            name
+            for last_name in last_names
+            for name in (last_name.split("-") if "-" in last_name else [last_name])
+            if name
+        ],
+    )
+
+
+def _clean_name(name: str) -> str:
+    name = clean(name, convert_to_ascii=True)
+
+    name = re.sub(
+        r"""
+            (MD|PhD|Ir|dr|Dr|DR|MSc|MA|BSc|BA|Prof)
+            [,.]?
+            [ ]*
+        """,
+        "",
+        name,
+        flags=re.VERBOSE,
+    )
+
+    subs_made = 1
+    while subs_made:
+        name, subs_made = re.subn(r"([A-Z]\.)\s+([A-Z])\b", r"\g<1>\g<2>", name)
+
+    name = name.replace(r"\s+-\s+", "-")
+    name = " ".join(p for p in name.split(" ") if p)
+    name = re.sub(r"^([^,.]+) ([A-Z])$", r"\g<1>, \g<2>.", name)
+    return name
+
+
+def _get_parenthesized_first_names(name: str) -> Tuple[List[str], str]:
+    groups = re.search(r"\(([a-zA-Z ]+)\)", name)
+    if groups:
+        return (
+            groups.group(1).split(" "),
+            _remove_between_indices(name, groups.start(), groups.end()),
+        )
+
+    return [], name
+
+
+def _get_initials(n: str) -> Tuple[List[str], str]:
+    initials = []
+
+    groups = re.search(r"\b(?P<abr1>([A-Z]\.)*)(?P<abr2>[A-Z])(\.|$)", n)
+    if groups:
+        first_name_abbreviations = [
+            *groups.group("abr1").split("."),
+            groups.group("abr2"),
+        ]
+        first_name_abbreviations = [f for f in first_name_abbreviations if f != ""]
+        initials.extend(first_name_abbreviations)
+        n = _remove_between_indices(n, groups.start(), groups.end())
+
+    return initials, n
+
+
+def _get_infixes(n: str) -> Tuple[List[str], str]:
+    infixes = ["de", "van", "der", "den", "te", "ter", "ten"]
+
+    parts = n.split(" ")
+    return [p for p in parts if p.lower() in infixes], " ".join(
+        p for p in parts if p.lower() not in infixes
+    )
+
+
+def _remove_between_indices(s: str, start: int, end: int) -> str:
+    return s[:start] + s[end:]
+
+
+def _match_percentage(l1: List[str], l2: List[str]) -> float:
+    if not l1 or not l2:
+        return 0
+
+    s1 = set(l1)
+    s2 = set(l2)
+    common_q = 2 * len(s1 & s2) / (len(s1) + len(s2))
+    different_q = min(len(s1 - s2) / len(s1), len(s2 - s1) / len(s2))
+    return common_q - different_q

good_ai/src/sus/match_names/name_parts.py

@@ -0,0 +1,10 @@
+from typing import List
+
+from pydantic import BaseModel
+
+
+class NameParts(BaseModel):
+    first_names: List[str]
+    initials: List[str]
+    infixes: List[str]
+    last_names: List[str]

good_ai/src/sus/nlp.py

@@ -0,0 +1,21 @@
+try:
+    import en_core_web_lg
+except ImportError:
+    import subprocess
+
+    print("Spacy model en_core_web_lg not found locally, downloading...")
+
+    subprocess.call(
+        [
+            "pip",
+            "install",
+            "https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.2.0/en_core_web_lg-3.2.0-py3-none-any.whl",
+        ]
+    )
+    import en_core_web_lg
+
+from .external.negspacy import negation  # noqa: F401 it's important to import this
+
+nlp = en_core_web_lg.load()
+
+nlp.add_pipe("negex")

good_ai/src/sus/parallel_map.py

@@ -0,0 +1,34 @@
+from math import ceil
+from typing import Any, Callable, Iterable, List, Optional
+
+import multiprocess as mp
+import psutil
+from tqdm.auto import tqdm
+
+
+def parallel_map(
+    function: Callable[[Any], Any],
+    values: Iterable[Any],
+    chunk_size: Optional[int] = None,
+    concurrency: int = psutil.cpu_count(),
+    disable_progress: bool = False,
+) -> List[Any]:
+    assert concurrency > 0
+    assert chunk_size is None or chunk_size > 0
+
+    values = list(values)
+
+    if not chunk_size:
+        chunk_size = max(1, ceil(len(values) / concurrency / 10))
+
+    if concurrency == 1:
+        iterable = values if disable_progress else tqdm(values)
+        return [function(v) for v in iterable]
+
+    with mp.Pool(processes=concurrency) as pool:
+        if disable_progress:
+            return pool.map(function, values, chunksize=chunk_size)
+
+        return list(
+            tqdm(pool.imap(function, values, chunksize=chunk_size), total=len(values))
+        )

good_ai/src/sus/publication_tei/__init__.py

@@ -0,0 +1,2 @@
+from .models import *
+from .publication_tei import PublicationTEI

good_ai/src/sus/publication_tei/models/__init__.py

@@ -0,0 +1,5 @@
+from .affiliation import Affiliation
+from .author import Author
+from .element import Element, Meta, MetaType, Paragraph, Title
+from .publication_metadata import PublicationMetadata
+from .text import Text

good_ai/src/sus/publication_tei/models/affiliation.py

@@ -0,0 +1,11 @@
+from typing import Optional, Tuple
+
+from pydantic import BaseModel
+
+
+class Affiliation(BaseModel):
+    institutions: Tuple[str, ...]
+    departments: Tuple[str, ...]
+    laboratories: Tuple[str, ...]
+    country: Optional[str]
+    settlement: Optional[str]

good_ai/src/sus/publication_tei/models/author.py

@@ -0,0 +1,14 @@
+from typing import List, Optional
+
+from pydantic import BaseModel
+
+from .affiliation import Affiliation
+
+
+class Author(BaseModel):
+    name: Optional[str]
+    orcid: Optional[str]
+    email: Optional[str]
+    corresponding: bool
+    affiliations: List[Affiliation]
+    coordinates: Optional[str]

good_ai/src/sus/publication_tei/models/element.py

@@ -0,0 +1,30 @@
+from typing import List, Literal, Union
+
+from pydantic import BaseModel
+
+from .text import Text
+
+
+class Title(BaseModel):
+    text: Text
+
+
+class Paragraph(BaseModel):
+    sentences: List[Text]
+
+
+MetaType = Literal[
+    "abstract_start",
+    "abstract_end",
+    "acknowledgements_start",
+    "acknowledgements_end",
+    "annex_start",
+    "annex_end",
+]
+
+
+class Meta(BaseModel):
+    meta_type: MetaType
+
+
+Element = Union[Title, Paragraph, Meta]

good_ai/src/sus/publication_tei/models/publication_metadata.py

@@ -0,0 +1,14 @@
+from typing import List, Optional
+
+from pydantic import BaseModel
+
+
+class PublicationMetadata(BaseModel):
+    language: Optional[str]
+    title: Optional[str]
+    publisher: Optional[str]
+    doi: Optional[str]
+    md5: Optional[str]
+    publication_date: Optional[str]
+    keywords: List[str]
+    reference_count: int

good_ai/src/sus/publication_tei/models/text.py

@@ -0,0 +1,7 @@
+from pydantic import BaseModel
+
+
+class Text(BaseModel):
+    content: str
+    document_order: int
+    coordinates: str

good_ai/src/sus/publication_tei/publication_tei.py

@@ -0,0 +1,176 @@
+from functools import cached_property
+from typing import Any, List, Optional, Union
+
+from bs4 import BeautifulSoup
+from bs4.element import NavigableString, Tag
+from sus.publication_tei.models.element import Paragraph
+
+from ..clean import clean
+from .models import Affiliation, Author, Element, Meta, PublicationMetadata, Text, Title
+
+
+class PublicationTEI:
+    def __init__(self, tei: str):
+        self._document_order_counter = 0
+
+        if tei:
+            cleaned_xml = clean(tei, ignore_xml=True)
+            self.soup = BeautifulSoup(cleaned_xml, "xml")
+        else:
+            self.soup = BeautifulSoup()
+
+    @cached_property
+    def publication_metadata(self) -> PublicationMetadata:
+        publication_date = (
+            self.soup.publicationStmt.date.get("when")
+            if self.soup.publicationStmt and self.soup.publicationStmt.date
+            else None
+        )
+
+        keywords = (
+            [self._element_to_text(k) for k in self.soup.keywords.find_all("term")]
+            if self.soup.keywords
+            else []
+        )
+
+        return PublicationMetadata(
+            language=self.soup.teiHeader.get("xml:lang")
+            if self.soup.teiHeader
+            else None,
+            title=self._element_to_text(self.soup.title),
+            publisher=self._element_to_text(self.soup.publisher),
+            doi=self._element_to_text(self.soup.find("idno", type="DOI")),
+            md5=self._element_to_text(self.soup.find("idno", type="MD5")),
+            publication_date=publication_date,
+            keywords=keywords,
+            reference_count=self.get_reference_count(),
+        )
+
+    def get_reference_count(self) -> int:
+        references = self.soup.find("div", {"type": "references"})
+
+        if not references:
+            return 0
+
+        return len(references.findAll("biblStruct"))
+
+    @cached_property
+    def authors(self) -> List[Author]:
+        if not self.soup.analytic:
+            return []
+
+        return [
+            self._parse_author(author)
+            for author in self.soup.analytic.find_all("author")
+        ]
+
+    @cached_property
+    def content(self) -> List[Element]:
+        self._document_order_counter = 0
+        return self._get_elements(self.soup)
+
+    @cached_property
+    def sentences(self) -> List[Text]:
+        return [
+            sentence
+            for element in self.content
+            if isinstance(element, Paragraph)
+            for sentence in element.sentences
+        ]
+
+    def _parse_author(self, raw: Tag) -> Author:
+        return Author(
+            name=(
+                clean(" ".join(name.get_text() for name in raw.persName))
+                if raw.persName
+                else None
+            ),
+            orcid=self._element_to_text(raw.find(attrs={"type": "ORCID"})),
+            email=self._element_to_text(raw.email),
+            corresponding=raw.get("role") == "corresp",
+            affiliations=[
+                self._parse_affiliation(aff) for aff in raw.find_all("affiliation")
+            ],
+            coordinates=raw.persName.get("coords") if raw.persName else None,
+        )
+
+    def _parse_affiliation(self, raw: Tag) -> Affiliation:
+        return Affiliation(
+            institutions=[
+                self._element_to_text(v)
+                for v in raw.find_all("orgName", attrs={"type": "institution"})
+            ],
+            departments=[
+                self._element_to_text(v)
+                for v in raw.find_all("orgName", attrs={"type": "department"})
+            ],
+            laboratories=[
+                self._element_to_text(v)
+                for v in raw.find_all("orgName", attrs={"type": "laboratory"})
+            ],
+            country=self._element_to_text(raw.address.country)
+            if raw.address and raw.address.country
+            else None,
+            settlement=self._element_to_text(raw.address.settlement)
+            if raw.address and raw.address.settlement
+            else None,
+        )
+
+    def _get_elements(self, raw: Tag) -> List[Element]:
+        results: List[Element] = []
+
+        for r in raw.find_all(["abstract", "div", "head", "p"]):
+            if r.name == "abstract":
+                results.append(Meta(meta_type="abstract_start"))
+                results.extend(self._get_primitives(r))
+                results.append(Meta(meta_type="abstract_end"))
+            elif r.name == "div" and r.get("type") == "acknowledgement":
+                results.append(Meta(meta_type="acknowledgements_start"))
+                results.extend(self._get_primitives(r))
+                results.append(Meta(meta_type="acknowledgements_end"))
+            elif r.name == "div" and r.get("type") == "annex":
+                results.append(Meta(meta_type="annex_start"))
+                results.extend(self._get_primitives(r))
+                results.append(Meta(meta_type="annex_end"))
+            elif not r.find_parents(["abstract", "div"]):
+                results.extend(self._get_primitives(r))
+
+        return results
+
+    def _get_primitives(self, raw: Tag) -> List[Element]:
+        results: List[Element] = []
+
+        for r in raw.find_all(["head", "p"]):
+            if r.name == "head" and r.get("coords") and r.get_text():
+                results.append(Title(text=self._parse_text(r)))
+            elif r.name == "p" and r.find_all("s"):
+                results.append(
+                    Paragraph(
+                        sentences=[
+                            self._parse_text(sentence)
+                            for sentence in r.find_all("s")
+                            if sentence.get_text()
+                        ]
+                    )
+                )
+
+        return results
+
+    def _element_to_text(
+        self, element: Optional[NavigableString], default: Any = None
+    ) -> Union[str, Any]:
+        return (
+            clean(element.get_text(separator=" ", strip=True)) if element else default
+        )
+
+    def _parse_text(self, raw: Tag) -> Text:
+        return Text(
+            content=clean(raw.get_text()),
+            document_order=self._generate_document_order_id(),
+            coordinates=raw.get("coords"),
+        )
+
+    def _generate_document_order_id(self) -> int:
+        value = self._document_order_counter
+        self._document_order_counter += 1
+        return value

good_ai/src/sus/unique.py

@@ -0,0 +1,16 @@
+from typing import Any, Callable, Iterable, List
+
+
+def unique(
+    values: Iterable[Any], *, key: Callable[[Any], Any] = lambda v: v
+) -> List[Any]:
+    """Only keep first occurrences and maintian order"""
+
+    key_values = {}
+    for v in values:
+        k = key(v)
+        if k not in key_values:
+            # dicts maintin insertion order: https://mail.python.org/pipermail/python-dev/2017-December/151283.html
+            key_values[k] = v
+
+    return list(key_values.values())