Move files

great_ai/utilities/__init__.py

@@ -0,0 +1,11 @@
+from .chunk import chunk
+from .clean import clean
+from .config_file import ConfigFile, ParseError
+from .evaluate_ranking import evaluate_ranking
+from .get_sentences import get_sentences
+from .language import english_name_of_language, is_english, predict_language
+from .logger import get_logger
+from .match_names import match_names
+from .parallel_map import WorkerException, parallel_map, threaded_parallel_map
+from .unchunk import unchunk
+from .unique import unique

great_ai/utilities/chunk.py

@@ -0,0 +1,17 @@
+from typing import Iterable, List, TypeVar
+
+T = TypeVar("T")
+
+
+def chunk(values: Iterable[T], chunk_size: int) -> Iterable[T]:
+    assert chunk_size >= 1
+
+    result: List[T] = []
+    for v in values:
+        result.append(v)
+        if len(result) == chunk_size:
+            yield result
+            result = []
+
+    if len(result) > 0:
+        yield result

great_ai/utilities/clean.py

@@ -0,0 +1,68 @@
+import html
+import re
+import unicodedata
+
+import unidecode
+
+from .data import left_regular_punctuations, right_regular_punctuations
+from .external.pylatexenc.latex2text import LatexNodes2Text
+from .logger import get_logger
+
+logger = get_logger("clean")
+latex = LatexNodes2Text()
+
+
+joined_left_punctuations = "".join(left_regular_punctuations).replace("]", r"\]")
+joined_right_punctuations = "".join(right_regular_punctuations).replace("[", r"\[")
+
+
+def clean(
+    text: str,
+    ignore_xml: bool = False,
+    ignore_latex: bool = False,
+    remove_brackets: bool = False,
+    convert_to_ascii: bool = False,
+) -> str:
+    if not ignore_xml:
+        text = re.sub(r"<[^>]*>", " ", text)
+        text = html.unescape(text)
+
+    if not ignore_latex:
+        text = text.replace("%", "\\%")  # escape LaTeX comments before parsing as LaTeX
+
+        try:
+            text = latex.latex_to_text(text, tolerant_parsing=True, strict_braces=False)
+            text = text.replace("%s", " ")
+        except:
+            logger.exception("Latex parsing error")
+
+    if convert_to_ascii:
+        text = unicodedata.normalize("NFKD", text)
+
+        try:
+            text.encode("ASCII", errors="strict")
+        except UnicodeEncodeError:
+            text = "".join([c for c in text if not unicodedata.combining(c)])
+            text = unidecode.unidecode(text)
+
+    text = re.sub(
+        r"\b[a-zA-Z](?:[\t ]+[a-zA-Z]\b)+", lambda m: re.sub(r"[\t ]", "", m[0]), text
+    )  # A R T I C L E => ARTICLE
+
+    if remove_brackets:
+        text = re.sub(r"\[[^\]]*\]", " ", text)
+
+    # fix hypens: break- word => break-word
+    text = re.sub(r"(\S)-\s+", r"\1-", text)
+    text = re.sub(r"\s+-(\S)", r"-\1", text)
+
+    # collapse whitespace
+    text = re.sub(r"\s+", " ", text)
+
+    # fix punctuation
+    text = re.sub(rf" ([{joined_left_punctuations}])", r"\1", text)
+    text = re.sub(rf"([{joined_right_punctuations}]) ", r"\1", text)
+
+    text = text.strip()
+
+    return text

great_ai/utilities/config_file/__init__.py

@@ -0,0 +1,2 @@
+from .config_file import ConfigFile
+from .parse_error import ParseError

great_ai/utilities/config_file/config_file.py

@@ -0,0 +1,91 @@
+import os
+from pathlib import Path
+from typing import Dict, Iterable, Tuple, Union
+
+from ..logger import get_logger
+from .parse_error import ParseError
+from .pattern import pattern
+
+ENVIRONMENT_VARIABLE_KEY_PREFIX = "ENV"
+
+logger = get_logger("ConfigFile")
+
+
+class ConfigFile:
+    def __init__(self, path: Union[Path, str]) -> None:
+        if not isinstance(path, Path):
+            path = Path(path)
+
+        if not path.exists():
+            raise FileNotFoundError(path.absolute())
+
+        self._path = path
+        self._key_values: Dict[str, str] = {}
+
+        self._parse()
+
+    def _parse(self):
+        with open(self._path, encoding="utf-8") as f:
+            lines: str = f.read()
+
+        matches = pattern.findall(lines)
+        for key, *values in matches:
+            try:
+                value = next(v for v in values if v)
+            except StopIteration:
+                raise ParseError(
+                    f"Cannot parse config file ({self._path.absolute()}), error at key `{key}`"
+                )
+
+            already_exists = key in self._key_values
+            if already_exists and not value.startswith(
+                f"{ENVIRONMENT_VARIABLE_KEY_PREFIX}:"
+            ):
+                raise KeyError(
+                    f"Key `{key}` has been already defined and its value is `{self._key_values[key]}`"
+                )
+
+            if value.startswith(f"{ENVIRONMENT_VARIABLE_KEY_PREFIX}:"):
+                _, value = value.split(":")
+                if value not in os.environ:
+                    issue = f'The value of `{key}` contains the "{ENVIRONMENT_VARIABLE_KEY_PREFIX}` prefix but `{value}` is not defined as an environment variable'
+                    if already_exists:
+                        logger.warning(
+                            f"{issue}, using the default value defined above (`{self._key_values[key]}`)"
+                        )
+                        continue
+                    else:
+                        raise KeyError(
+                            f"{issue} and no default value has been provided"
+                        )
+                else:
+                    value = os.environ[value]
+
+            self._key_values[key] = value
+
+    def __getattr__(self, key: str) -> str:
+        if key in self._key_values:
+            return self._key_values[key]
+        raise KeyError(
+            f"Key `{key}` is not found in configuration file ({self._path.absolute()})"
+        )
+
+    __getitem__ = __getattr__
+
+    def __iter__(self) -> Iterable[Tuple[str, str]]:
+        return iter(self._key_values)
+
+    def __len__(self) -> int:
+        return len(self._key_values)
+
+    def keys(self):
+        return self._key_values.keys()
+
+    def values(self):
+        return self._key_values.values()
+
+    def items(self):
+        return self._key_values.items()
+
+    def __repr__(self):
+        return f"{type(self).__name__}(path={self._path}) {self._key_values}"

great_ai/utilities/config_file/parse_error.py

@@ -0,0 +1,2 @@
+class ParseError(Exception):
+    pass

great_ai/utilities/config_file/pattern.py

@@ -0,0 +1,20 @@
+import re
+
+pattern = re.compile(
+    r"""
+        (?:^|\n)        # new key-value pairs must start on a new line
+        \s*             # leading whitespace is allowed
+        (?!\#)          # the key cannot start with a `#` symbol
+        (\w+?)          # then comes the key
+        \s*=\s*         # the key and value are separated by an equal sign
+        (?:             # then comes the value
+              "([^"]*)"   # the value can be surrounded by quotes: "value"
+            | '([^']*)'   # the value can be surrounded by quotes: 'value'
+            | `([^`]*)`   # the value can be surrounded by quotes: `value`
+            | ([^#\n]*?)  # or it is bare, in that case, the trailing whitespace is ignored
+        )
+        \s*(?:\#.*)?    # comments can be added with the `#` symbol
+        (?=\n|$)        # a key-value pairs are separated by new lines
+    """,
+    flags=re.UNICODE | re.VERBOSE,
+)

great_ai/utilities/data/__init__.py

@@ -0,0 +1,6 @@
+from .american_spellings import american_spellings
+from .punctuations import (
+    left_regular_punctuations,
+    right_regular_punctuations,
+    sentence_ending_punctuations,
+)

great_ai/utilities/data/punctuations.py

@@ -0,0 +1,22 @@
+sentence_ending_punctuations = [".", "?", "!", ":", ";"]
+
+# punctuations that usually have no space between them and the character to their left
+left_regular_punctuations = [
+    *sentence_ending_punctuations,
+    ",",
+    "'",
+    "%",
+    ")",
+    "}",
+    "]",
+]
+
+# punctuations that usually have no space between them and the character to their right
+right_regular_punctuations = [
+    "(",
+    "{",
+    "[",
+    "$",
+    "€",
+    "#",
+]

great_ai/utilities/evaluate_ranking/__init__.py

@@ -0,0 +1,2 @@
+from .draw_f1_iso_lines import draw_f1_iso_lines
+from .evaluate_ranking import evaluate_ranking

great_ai/utilities/evaluate_ranking/draw_f1_iso_lines.py

@@ -0,0 +1,28 @@
+from typing import Optional
+
+import matplotlib.pyplot as plt
+import numpy as np
+from matplotlib.axes import Axes
+
+
+def draw_f1_iso_lines(
+    resolution: int = 1000,
+    min: float = 0.2,
+    max: float = 0.8,
+    steps: int = 4,
+    axes: Optional[Axes] = None,
+) -> None:
+    if axes is None:
+        axes = plt.axes()
+
+    for f_score in np.linspace(min, max, num=steps):
+        x = np.linspace(f_score / (2 - f_score), 1, num=resolution)
+        y = f_score * x / (2 * x - f_score)
+
+        axes.plot(x[y >= 0], y[y >= 0], color="gray", alpha=0.2)
+
+        axes.annotate(
+            f"f1={f_score:0.1f}",
+            backgroundcolor="w",
+            xy=(0.9, y[int(resolution * 0.9)] + 0.02),
+        )

great_ai/utilities/evaluate_ranking/evaluate_ranking.py

@@ -0,0 +1,90 @@
+from pathlib import Path
+from typing import Dict, List, Optional, Union
+
+import matplotlib
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.metrics import average_precision_score, precision_recall_curve
+
+from ..unique import unique
+from .draw_f1_iso_lines import draw_f1_iso_lines
+
+
+def evaluate_ranking(
+    expected: List[Union[str, float]],
+    actual_scores: List[float],
+    target_recall: float,
+    title: Optional[str] = "",
+    disable_interpolation: bool = False,
+    axes: Optional[plt.Axes] = None,
+    output_svg: Optional[Path] = None,
+    reverse_order: bool = False,
+    plot: bool = True,
+) -> Dict[Union[str, float], float]:
+    assert 0 <= target_recall <= 1
+
+    if plot and axes is None:
+        fig = plt.figure(figsize=(20, 10))
+        fig.patch.set_facecolor("white")
+        ax = plt.axes()
+    else:
+        ax = axes
+
+    classes = sorted(unique(expected), reverse=reverse_order)
+    str_classes = [str(c) for c in classes]
+
+    with matplotlib.rc_context({"font.size": 20}):
+        if plot:
+            ax.set_xmargin(0)
+
+            draw_f1_iso_lines(axes=ax)
+
+        results: Dict[Union[str, float], float] = {}
+        for i in range(len(classes) - 1):
+            binarized_expected = [
+                (v < classes[i]) if reverse_order else (v > classes[i])
+                for v in expected
+            ]
+            precision, recall, _ = precision_recall_curve(
+                binarized_expected, actual_scores
+            )
+
+            if not disable_interpolation:
+                for j in range(1, len(precision)):
+                    precision[j] = max(precision[j - 1], precision[j])
+
+            closest_recall_index = np.argmin(np.abs(recall - target_recall))
+            precision_at_closest_recall = precision[closest_recall_index]
+            average_precision = average_precision_score(
+                binarized_expected, actual_scores
+            )
+            results[classes[i]] = precision_at_closest_recall
+
+            if plot:
+                ax.plot(
+                    recall,
+                    precision,
+                    label=f"{'|'.join(str_classes[:i + 1])} ↔ {'|'.join(str_classes[i+1:])} (P@{target_recall:.2f}={precision_at_closest_recall:.2f}, AP={average_precision:.2f})",
+                )
+
+        if plot:
+            ax.legend(loc="upper right")
+            ax.axvline(x=target_recall, linestyle="--", color="#55c6bb", linewidth=2.0)
+
+            if title is None:
+                title = "Ranking evaluation"
+
+            ax.set_title(f'{title} ({" < ".join(str_classes)})', pad=20)
+
+            ax.set_xlabel("Recall")
+            ax.set_ylabel("Precision")
+
+            ax.set_xticks([target_recall] + sorted(ax.get_xticks()))
+
+        if plot and output_svg is None:
+            if axes is None:
+                plt.show()
+        elif output_svg:
+            plt.savefig(output_svg, format="svg")
+
+    return results

great_ai/utilities/external/pylatexenc/README.md

@@ -0,0 +1 @@
+https://github.com/phfaist/pylatexenc

great_ai/utilities/external/pylatexenc/__init__.py

@@ -0,0 +1,37 @@
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2015 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+
+"""
+Utilities for LaTeX to/from Unicode Text conversion.
+
+Main Site:
+
+    https://github.com/phfaist/pylatexenc/
+
+"""
+
+from .version import version_str as _version_str
+
+__version__ = _version_str

great_ai/utilities/external/pylatexenc/_util.py

@@ -0,0 +1,161 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2019 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+
+# Internal module. Internal API may move, disappear or otherwise change at any
+# time and without notice.
+
+
+try:
+    # Python >= 3.3
+    from collections.abc import MutableMapping
+except ImportError:
+    from collections import MutableMapping
+
+import bisect
+import warnings
+
+# ------------------------------------------------------------------------------
+
+
+def pylatexenc_deprecated_ver(ver, msg, stacklevel=2):
+    warnings.warn(
+        "Deprecated (pylatexenc {}): {} ".format(ver, msg.strip()),
+        DeprecationWarning,
+        stacklevel=stacklevel + 1,
+    )
+
+
+def pylatexenc_deprecated_2(msg, stacklevel=2):
+    warnings.warn(
+        (
+            "Deprecated (pylatexenc 2.0): {} "
+            "[see https://pylatexenc.readthedocs.io/en/latest/new-in-pylatexenc-2/]"
+        ).format(msg.strip()),
+        DeprecationWarning,
+        stacklevel=stacklevel + 1,
+    )
+
+
+# ------------------------------------------------------------------------------
+
+
+class LazyDict(MutableMapping):
+    r"""
+    A lazy dictionary that loads its data when it is first queried.
+
+    This is used to store the legacy
+    :py:data:`pylatexenc.latexwalker.default_macro_dict` as well as
+    :py:data:`pylatexenc.latex2text.default_macro_dict` etc.  Such that these
+    "dictionaries" are still exposed at the module-level, but the data is loaded
+    only if they are actually queried.
+    """
+
+    def __init__(self, generate_dict_fn):
+        self._full_dict = None
+        self._generate_dict_fn = generate_dict_fn
+
+    def _ensure_instance(self):
+        if self._full_dict is not None:
+            return
+        self._full_dict = self._generate_dict_fn()
+
+    def __getitem__(self, key):
+        self._ensure_instance()
+        return self._full_dict.__getitem__(key)
+
+    def __setitem__(self, key, val):
+        self._ensure_instance()
+        return self._full_dict.__setitem__(key, val)
+
+    def __delitem__(self, key):
+        self._ensure_instance()
+        return self._full_dict.__delitem__(key)
+
+    def __iter__(self):
+        self._ensure_instance()
+        return iter(self._full_dict)
+
+    def __len__(self):
+        self._ensure_instance()
+        return len(self._full_dict)
+
+    def copy(self):
+        self._ensure_instance()
+        return self._full_dict.copy()
+
+    def clear(self):
+        self._ensure_instance()
+        return self._full_dict.clear()
+
+
+# ------------------------------------------------------------------------------
+
+
+class LineNumbersCalculator(object):
+    r"""
+    Utility to calculate line numbers.
+    """
+
+    def __init__(self, s):
+        super(LineNumbersCalculator, self).__init__()
+
+        def find_all_new_lines(x):
+            # first line starts at the beginning of the string
+            yield 0
+            k = 0
+            while k < len(x):
+                k = x.find("\n", k)
+                if k == -1:
+                    return
+                k += 1
+                # s[k] is the character after the newline, i.e., the 0-th column
+                # of the new line
+                yield k
+
+        self._pos_new_lines = list(find_all_new_lines(s))
+
+    def pos_to_lineno_colno(self, pos, as_dict=False):
+        r"""
+        Return the line and column number corresponding to the given `pos`.
+
+        Return a tuple `(lineno, colno)` giving line number and column number.
+        Line numbers start at 1 and column number start at zero, i.e., the
+        beginning of the document (`pos=0`) has line and column number `(1,0)`.
+        If `as_dict=True`, then a dictionary with keys 'lineno', 'colno' is
+        returned instead of a tuple.
+        """
+
+        # find line number in list
+
+        # line_no is the index of the last item in self._pos_new_lines that is <= pos.
+        line_no = bisect.bisect_right(self._pos_new_lines, pos) - 1
+        assert line_no >= 0 and line_no < len(self._pos_new_lines)
+
+        col_no = pos - self._pos_new_lines[line_no]
+        # 1+... so that line and column numbers start at 1
+        if as_dict:
+            return {"lineno": 1 + line_no, "colno": col_no}
+        return (1 + line_no, col_no)

great_ai/utilities/external/pylatexenc/latex2text/__main__.py

@@ -0,0 +1,325 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2018 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+import argparse
+import fileinput
+import logging
+import sys
+
+from .. import latexwalker
+from ..latex2text import LatexNodes2Text, _strict_latex_spaces_predef
+from ..version import version_str
+
+
+def main(argv=None):
+
+    if argv is None:
+        argv = sys.argv[1:]
+
+    parser = argparse.ArgumentParser(prog="latex2text", add_help=False)
+
+    codegroup = parser.add_argument_group("Input options")
+
+    codegroup.add_argument(
+        "--code",
+        "-c",
+        action="store",
+        default=None,
+        metavar="LATEX_CODE",
+        help="Convert the given LATEX_CODE to unicode text instead of reading "
+        "from FILE or standard input.  You cannot specify FILEs if you use this "
+        "option, and any standard input is ignored.",
+    )
+
+    codegroup.add_argument(
+        "files",
+        metavar="FILE",
+        nargs="*",
+        help="Input files to read LaTeX code from. If no FILE(s) is/are specified, "
+        "LaTeX code is read from standard input unless --code is specified",
+    )
+
+    group = parser.add_argument_group("LatexWalker options")
+
+    group.add_argument(
+        "--parser-keep-inline-math",
+        action="store_const",
+        const=True,
+        dest="parser_keep_inline_math",
+        default=None,
+        help=argparse.SUPPRESS,
+    )
+    group.add_argument(
+        "--no-parser-keep-inline-math",
+        action="store_const",
+        const=False,
+        dest="parser_keep_inline_math",
+        help=argparse.SUPPRESS,
+    )
+
+    group.add_argument(
+        "--tolerant-parsing",
+        action="store_const",
+        const=True,
+        dest="tolerant_parsing",
+        default=True,
+    )
+    group.add_argument(
+        "--no-tolerant-parsing",
+        action="store_const",
+        const=False,
+        dest="tolerant_parsing",
+        help="Tolerate syntax errors when parsing, and attempt to continue (default yes)",
+    )
+
+    # I'm not sure this flag is useful and if it should be exposed at all.
+    # Accept it, but make it hidden.
+    parser.add_argument(
+        "--strict-braces",
+        action="store_const",
+        const=True,
+        dest="strict_braces",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
+    parser.add_argument(
+        "--no-strict-braces",
+        action="store_const",
+        const=False,
+        dest="strict_braces",
+        # help="Report errors for mismatching LaTeX braces (default no)"
+        help=argparse.SUPPRESS,
+    )
+
+    group = parser.add_argument_group("LatexNodes2Text options")
+
+    group.add_argument(
+        "--text-keep-inline-math",
+        action="store_const",
+        const=True,
+        dest="text_keep_inline_math",
+        default=None,
+        help=argparse.SUPPRESS,
+    )
+    group.add_argument(
+        "--no-text-keep-inline-math",
+        action="store_const",
+        const=False,
+        dest="text_keep_inline_math",
+        help=argparse.SUPPRESS,
+    )
+
+    group.add_argument(
+        "--math-mode",
+        action="store",
+        dest="math_mode",
+        choices=["text", "with-delimiters", "verbatim", "remove"],
+        default="text",
+        help="How to handle chunks of math mode LaTeX code. 'text' = convert "
+        "to text like the rest; 'with-delimiters' = same as 'text' but retain "
+        "the original math mode delimiters; 'verbatim' = keep verbatim LaTeX code; "
+        "'remove' = remove from input entirely",
+    )
+
+    group.add_argument(
+        "--fill-text",
+        dest="fill_text",
+        action="store",
+        nargs="?",
+        default=-1,
+        help="Attempt to wrap text to the given width, or 80 columns if option is "
+        "specified with no argument",
+    )
+
+    group.add_argument(
+        "--keep-comments",
+        action="store_const",
+        const=True,
+        dest="keep_comments",
+        default=False,
+    )
+    group.add_argument(
+        "--no-keep-comments",
+        action="store_const",
+        const=False,
+        dest="keep_comments",
+        help="Keep LaTeX comments in text output (default no)",
+    )
+
+    class ListWithHiddenItems(list):
+        def __init__(self, thelist, hiddenitems):
+            super(ListWithHiddenItems, self).__init__(thelist)
+            self.hiddenitems = hiddenitems
+
+        def __contains__(self, value):
+            return (
+                super(ListWithHiddenItems, self).__contains__(value)
+                or value in self.hiddenitems
+            )
+
+    strict_latex_spaces_choices = ListWithHiddenItems(
+        # the list
+        ["off", "on"]
+        + list(k for k in _strict_latex_spaces_predef.keys() if k != "default"),
+        # hidden items: Value is accepted, but not shown in list of choices
+        ["default"],
+    )
+    group.add_argument(
+        "--strict-latex-spaces",
+        choices=strict_latex_spaces_choices,
+        dest="strict_latex_spaces",
+        default="macros",
+        help="How to handle whitespace. See documentation for the class "
+        "LatexNodes2Text().",
+    )
+
+    group.add_argument(
+        "--keep-braced-groups",
+        action="store_const",
+        const=True,
+        dest="keep_braced_groups",
+        default=False,
+    )
+    group.add_argument(
+        "--no-keep-braced-groups",
+        action="store_const",
+        const=False,
+        dest="keep_braced_groups",
+        help="Keep LaTeX {braced groups} in text output (default no)",
+    )
+
+    group.add_argument(
+        "--keep-braced-groups-minlen",
+        type=int,
+        default=2,
+        dest="keep_braced_groups_minlen",
+        help="Only apply --keep-braced-groups to groups that contain at least "
+        "this many characters",
+    )
+
+    group = parser.add_argument_group("General options")
+
+    group.add_argument(
+        "-q",
+        "--quiet",
+        dest="logging_level",
+        action="store_const",
+        const=logging.ERROR,
+        default=logging.INFO,
+        help="Suppress warning messages",
+    )
+    group.add_argument(
+        "-v",
+        "--verbose",
+        dest="logging_level",
+        action="store_const",
+        const=logging.DEBUG,
+        help="Verbose output",
+    )
+    group.add_argument(
+        "--version",
+        action="version",
+        version="pylatexenc {}".format(version_str),
+        help="Show version information and exit",
+    )
+    group.add_argument(
+        "--help", action="help", help="Show this help information and exit"
+    )
+
+    args = parser.parse_args(argv)
+
+    logging.basicConfig()
+    logging.getLogger().setLevel(args.logging_level)
+    logger = logging.getLogger(__name__)
+
+    if (
+        args.parser_keep_inline_math is not None
+        or args.text_keep_inline_math is not None
+    ):
+        logger.warning(
+            "Options --parser-keep-inline-math and --text-keep-inline-math are "
+            "deprecated and no longer have any effect.  Please use "
+            "--math-mode=... instead."
+        )
+
+    latex = ""
+    if args.code:
+        if args.files:
+            logger.error(
+                "Cannot specify both FILEs and --code option. "
+                "Use --help option for more information."
+            )
+            sys.exit(1)
+        latex = args.code
+    else:
+        for line in fileinput.input(files=args.files):
+            latex += line
+
+    if args.fill_text != -1:
+        if args.fill_text is not None and len(args.fill_text):
+            fill_text = int(args.fill_text)
+        else:
+            fill_text = True
+    else:
+        fill_text = None
+
+    lw = latexwalker.LatexWalker(
+        latex, tolerant_parsing=args.tolerant_parsing, strict_braces=args.strict_braces
+    )
+
+    (nodelist, pos, len_) = lw.get_latex_nodes()
+
+    ln2t = LatexNodes2Text(
+        math_mode=args.math_mode,
+        keep_comments=args.keep_comments,
+        strict_latex_spaces=args.strict_latex_spaces,
+        keep_braced_groups=args.keep_braced_groups,
+        keep_braced_groups_minlen=args.keep_braced_groups_minlen,
+        fill_text=fill_text,
+    )
+
+    print(ln2t.nodelist_to_text(nodelist))
+
+
+def run_main():
+
+    try:
+
+        main()
+
+    except SystemExit:
+        raise
+    except:  # lgtm [py/catch-base-exception]
+        import pdb
+        import traceback
+
+        traceback.print_exc()
+        pdb.post_mortem()
+
+
+if __name__ == "__main__":
+
+    main()
+    # run_main() # debug

great_ai/utilities/external/pylatexenc/latexencode/__init__.py

@@ -0,0 +1,325 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2018 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+r"""
+The `latexencode` module provides a set of routines that allows you to
+convert a unicode string to LaTeX escape sequences.
+
+For basic usage you can use the :py:func:`unicode_to_latex()` function
+directly::
+
+  >>> print(unicode_to_latex('À votre santé'))
+  \`A votre sant\'e
+  >>> print(unicode_to_latex('The length of samples #3 & #4 is 3μm'))
+  The length of samples \#3 \& \#4 is 3\ensuremath{\mu}m
+
+The conversion is handled by the class :py:class:`UnicodeToLatexEncoder`.  If
+you are converting multiple strings, you may create an instance with the flags
+you like and invoke its method
+:py:meth:`~UnicodeToLatexEncoder.unicode_to_latex()` as many times as necessary::
+
+  >>> u = UnicodeToLatexEncoder(unknown_char_policy='replace')
+  >>> print(u.unicode_to_latex('À votre santé'))
+  \`A votre sant\'e
+  >>> print(u.unicode_to_latex('The length of samples #3 & #4 is 3μm'))
+  The length of samples \#3 \& \#4 is 3\ensuremath{\mu}m
+  >>> print(u.unicode_to_latex('À votre santé: 乾杯'))
+  \`A votre sant\'e: {\bfseries ?}{\bfseries ?}
+
+Example using custom conversion rules::
+  >>> import re
+  >>> u = UnicodeToLatexEncoder(
+  ...     conversion_rules=[
+  ...         UnicodeToLatexConversionRule(rule_type=RULE_REGEX, rule=[
+  ...             (re.compile(r'-->'), r'\\textrightarrow'),
+  ...             (re.compile(r'<--'), r'\\textleftarrow'),
+  ...         ]),
+  ...         'defaults'
+  ...     ]
+  ... )
+  >>> print(u.unicode_to_latex("Cheers --> À votre santé"))
+  Cheers {\textrightarrow} \`A votre sant\'e
+
+See :py:class:`UnicodeToLatexEncoder` and
+:py:class:`UnicodeToLatexConversionRule`.  Note for regex rules, the replacement
+text is expanded like the second argument of `re.sub()` and backslashes need to
+be escaped even inside raw strings.
+
+.. versionadded:: 2.0
+
+   The class :py:class:`UnicodeToLatexEncoder` along with its helper functions
+   and classes were introduced in `pylatexenc 2.0`.
+
+   The earlier function :py:func:`utf8tolatex()` that was available in
+   `pylatexenc 1.x` is still provided unchanged, so code written for `pylatexenc
+   1.x` should work without changes.  New code is however strongly encouraged to
+   employ the new API.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import functools
+import itertools
+import logging
+import sys
+import unicodedata
+
+if sys.version_info.major > 2:
+    unicode = str  # need to support unicode() w/ no arguments
+    basestring = str
+    # use MappingProxyType for keeping
+    # inspect function argument names
+    from inspect import getfullargspec
+    from types import MappingProxyType as _MappingProxyType
+else:
+    _MappingProxyType = dict
+    # inspect function argument names -- simulate getfullargspec with getargspec (argh...)
+    from inspect import getargspec as getfullargspec
+
+logger = logging.getLogger(__name__)
+
+
+from .. import _util
+from ._partial_latex_encoder import PartialLatexToLatexEncoder
+from ._unicode_to_latex_encoder import (
+    RULE_CALLABLE,
+    RULE_DICT,
+    RULE_REGEX,
+    UnicodeToLatexConversionRule,
+    UnicodeToLatexEncoder,
+    get_builtin_conversion_rules,
+    get_builtin_uni2latex_dict,
+)
+
+# ------------------------------------------------
+
+
+# ------------------------------------------------
+
+
+# ------------------------------------------------
+
+
+_u2l_obj_cache = {}
+
+
+def unicode_to_latex(
+    s,
+    non_ascii_only=False,
+    replacement_latex_protection="braces",
+    unknown_char_policy="keep",
+    unknown_char_warning=True,
+):
+    r"""
+    Shorthand for constructing a :py:class:`UnicodeToLatexEncoder` instance and
+    calling its :py:meth:`~UnicodeToLatexEncoder.unicode_to_latex()` method.
+
+    The :py:class:`UnicodeToLatexEncoder` instances for given option settings
+    are cached, making repeated calls to :py:func:`unicode_to_latex()` possible
+    without creating a new instance upon each call.
+
+    The parameters `non_ascii_only`, `replacement_latex_protection`,
+    `unknown_char_policy`, and `unknown_char_warning` are directly passed on to
+    the :py:class:`UnicodeToLatexEncoder` constructor.  See the class doc for
+    :py:class:`UnicodeToLatexEncoder` for more information about what they do.
+
+    You may only use arguments to this function that are python hashable (like
+    `True`, `False`, or simple strings) to help us keep a cache of previously
+    constructed :py:class:`UnicodeToLatexEncoder` instances.  For instance, it
+    is not possible to provide a callable to `unknown_char_policy`.  It is also
+    not possible to specify custom conversion rules with this helper function.
+    If you need any of these features, simply create a
+    :py:class:`UnicodeToLatexEncoder` instance directly.
+    """
+
+    key = (
+        non_ascii_only,
+        replacement_latex_protection,
+        unknown_char_policy,
+        unknown_char_warning,
+    )
+
+    if key in _u2l_obj_cache:
+        u = _u2l_obj_cache[key]
+    else:
+        u = UnicodeToLatexEncoder(
+            non_ascii_only=non_ascii_only,
+            replacement_latex_protection=replacement_latex_protection,
+            unknown_char_policy=unknown_char_policy,
+            unknown_char_warning=unknown_char_warning,
+        )
+        _u2l_obj_cache[key] = u
+
+    return u.unicode_to_latex(s)
+
+
+# ------------------------------------------------------------------------------
+
+# Don't change pylatexenc 1.x function:
+
+
+def _get_deprecated_utf82latex():
+    #
+    # Don't issue a deprecation warning, because utf8tolatex() uses the
+    # `utf82latex` dict even if it isn't modified by the user.
+    #
+    #     _util.pylatexenc_deprecated_2(
+    #         "The module-level dictionary `pylatexenc.latexencode.utf82latex` is deprecated "
+    #         "and might be removed in a future version of `pylatexenc`.",
+    #     )
+
+    # return a copy of the dict so that the user can modify the module-level
+    # `utf82latex` dict without influencing the behavior of the new
+    # `unicode_to_latex()` routines. (E.g., if two python modules use
+    # pylatexenc.latexencode, we don't want one python module's use of
+    # `utf2tolatex()` to influence the behavior of another module's use of
+    # `unicode_to_latex()`.  If both modules use `utf8tolatex()`, we can't avoid
+    # this influence.)
+    from ._uni2latexmap import uni2latex as _uni2latex
+
+    return _uni2latex.copy()
+
+
+utf82latex = _util.LazyDict(generate_dict_fn=_get_deprecated_utf82latex)
+"""
+.. deprecated:: 2.0
+
+   Pylatexenc 1.x exposed the module-level dictionary `utf82latex` that could be
+   modified to alter the behavior of `utf8tolatex()`.
+
+   If you would like to obtain a copy of the built-in unicode to text
+   dictionary, see :py:func:`get_builtin_uni2latex_dict()`.  If you would like
+   to alter the behavior of :py:func:`utf8tolatex()`, you should use
+   :py:class:`UnicodeToLatexEncoder` which provides a rich interface for
+   specifying rules how to convert chars to LaTeX escapes.
+
+   For backwards compatibility, you can still modify the module-level dictionary
+   `utf82latex` (but you can't assign a new object to it) and this will directly
+   modify the global built-in dictionary of known latex escapes.  This is not
+   recommended however, and the `utf82latex` module-level dictionary might be
+   removed in the future.
+
+   .. warning::
+
+      Modifying the `utf82latex` module-level dictionary is not recommended.
+      Doing so will alter the behavior of the `utf8tolatex()` function also for
+      all other modules that also use `pylatexenc`!
+"""
+
+
+def utf8tolatex(
+    s,
+    non_ascii_only=False,
+    brackets=True,
+    substitute_bad_chars=False,
+    fail_bad_chars=False,
+):
+    """
+    .. note::
+
+       Since `pylatexenc 2.0`, it is recommended to use the the
+       :py:func:`unicode_to_latex()` function or the
+       :py:class:`UnicodeToLatexEncoder` class instead of the earlier function
+       `utf8tolatex()`.
+
+       The new routines provide much more flexibility and versatility.  For
+       instance, you can specify custom escape sequences for certain characters.
+       Some cheap benchmarks seem to indicate that the new routines are not
+       significantly slower than the `utf8tolatex()` function.  Also, the name
+       `utf8tolatex()` was poorly chosen, since the argument is in fact not
+       'utf-8'-encoded but rather a Python unicode string object.
+
+       The function `utf8tolatex()` is still provided unchanged from `pylatexenc
+       1.x`.  We do not plan to remove this function in the near future so it is
+       not (yet) considered as deprecated and we will continue to provide it in
+       near future versions of `pylatexenc`.  Bug reports, improvements, and new
+       features will however be directed to :py:func:`UnicodeToLatexEncoder()`.
+
+    Encode a UTF-8 string to a LaTeX snippet.
+
+    If `non_ascii_only` is set to `True`, then usual (ascii) characters such as ``#``,
+    ``{``, ``}`` etc. will not be escaped.  If set to `False` (the default), they are
+    escaped to their respective LaTeX escape sequences.
+
+    If `brackets` is set to `True` (the default), then LaTeX macros are enclosed in
+    brackets.  For example, ``sant\N{LATIN SMALL LETTER E WITH ACUTE}`` is replaced by
+    ``sant{\\'e}`` if `brackets=True` and by ``sant\\'e`` if `brackets=False`.
+
+    .. warning::
+        Using `brackets=False` might give you an invalid LaTeX string, so avoid
+        it! (for instance, ``ma\N{LATIN SMALL LETTER I WITH CIRCUMFLEX}tre`` will be
+        replaced incorrectly by ``ma\\^\\itre`` resulting in an unknown macro ``\\itre``).
+
+    If `substitute_bad_chars=True`, then any non-ascii character for which no LaTeX escape
+    sequence is known is replaced by a question mark in boldface. Otherwise (by default),
+    the character is left as it is.
+
+    If `fail_bad_chars=True`, then a `ValueError` is raised if we cannot find a
+    character substitution for any non-ascii character.
+
+    .. versionchanged:: 1.3
+
+        Added `fail_bad_chars` switch
+    """
+
+    s = unicode(s)  # make sure s is unicode
+    s = unicodedata.normalize("NFC", s)
+
+    if not s:
+        return ""
+
+    result = ""
+    for ch in s:
+        # logger.longdebug("Encoding char %r", ch)
+        if non_ascii_only and ord(ch) < 127:
+            result += ch
+        else:
+            # use the `utf82latex` dict -- not `_uni2latex` which should NOT be
+            # modified externally even for backwards-compatible code
+            lch = utf82latex.get(ord(ch), None)
+            if lch is not None:
+                # add brackets if needed, i.e. if we have a substituting macro.
+                # note: in condition, beware, that lch might be of zero length.
+                result += "{" + lch + "}" if brackets and lch[0:1] == "\\" else lch
+            elif (ord(ch) >= 32 and ord(ch) <= 127) or (ch in "\n\r\t"):
+                # ordinary printable ascii char, just add it
+                result += ch
+            else:
+                # non-ascii char
+                msg = "Character cannot be encoded into LaTeX: U+%04X - `%s'" % (
+                    ord(ch),
+                    ch,
+                )
+                if fail_bad_chars:
+                    raise ValueError(msg)
+
+                logger.warning(msg)
+                if substitute_bad_chars:
+                    result += r"{\bfseries ?}"
+                else:
+                    # keep unescaped char
+                    result += ch
+
+    return result

great_ai/utilities/external/pylatexenc/latexencode/__main__.py

@@ -0,0 +1,148 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2019 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+
+import argparse
+import fileinput
+import logging
+import sys
+
+from ..latexencode import unicode_to_latex
+from ..version import version_str
+
+
+def main(argv=None):
+
+    if argv is None:
+        argv = sys.argv[1:]
+
+    parser = argparse.ArgumentParser(prog="latexencode", add_help=False)
+    parser.add_argument(
+        "files",
+        metavar="FILE",
+        nargs="*",
+        help="Input files (if none specified, read from stdandard input)",
+    )
+
+    parser.add_argument(
+        "--non-ascii-only",
+        action="store_const",
+        const=True,
+        dest="non_ascii_only",
+        default=False,
+    )
+    parser.add_argument(
+        "--no-non-ascii-only",
+        action="store_const",
+        const=False,
+        dest="non_ascii_only",
+        help="The option --non-ascii-only specifies that only non-ascii characters "
+        "are to be encoded into LaTeX sequences, and not characters like '$' "
+        "even though they might have a special LaTeX meaning.",
+    )
+
+    parser.add_argument(
+        "--replacement-latex-protection",
+        choices=(
+            "braces",
+            "braces-all",
+            "braces-almost-all",
+            "braces-after-macro",
+            "none",
+        ),
+        dest="replacement_latex_protection",
+        default="braces",
+        help=r"How to protect replacement latex code from producing invalid latex code "
+        r"when concatenated in a longer string.  One of 'braces', 'braces-all', "
+        r"'braces-almost-all', 'braces-after-macro', 'none'.  Example: using "
+        r"choice 'braces' we avoid the invalid replacement 'a→b' -> 'a\tob' "
+        r"with instead 'a{\to}b'.",
+    )
+
+    parser.add_argument(
+        "--unknown-char-policy",
+        choices=("keep", "replace", "ignore", "fail"),
+        dest="unknown_char_policy",
+        default="keep",
+        help="How to deal with nonascii characters with no known latex code equivalent.",
+    )
+
+    parser.add_argument(
+        "-q",
+        "--quiet",
+        dest="logging_level",
+        action="store_const",
+        const=logging.ERROR,
+        default=logging.INFO,
+        help="Suppress warning messages",
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version="pylatexenc {}".format(version_str),
+        help="Show version information and exit",
+    )
+    parser.add_argument(
+        "--help", action="help", help="Show this help information and exit"
+    )
+
+    args = parser.parse_args(argv)
+
+    logging.basicConfig()
+    logging.getLogger().setLevel(args.logging_level)
+
+    latex = ""
+    for line in fileinput.input(files=args.files):
+        latex += line
+
+    result = unicode_to_latex(
+        latex,
+        non_ascii_only=args.non_ascii_only,
+        replacement_latex_protection=args.replacement_latex_protection,
+        unknown_char_policy=args.unknown_char_policy,
+    )
+
+    sys.stdout.write(result)
+
+
+def run_main():
+    try:
+
+        main()
+
+    except SystemExit:
+        raise
+    except:  # lgtm [py/catch-base-exception]
+        import pdb
+        import traceback
+
+        traceback.print_exc()
+        pdb.post_mortem()
+
+
+if __name__ == "__main__":
+
+    # run_main()  ## DEBUG
+    main()

great_ai/utilities/external/pylatexenc/latexencode/_partial_latex_encoder.py

@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2021 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+# import sys
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+from ._unicode_to_latex_encoder import (
+    RULE_CALLABLE,
+    UnicodeToLatexConversionRule,
+    UnicodeToLatexEncoder,
+)
+
+# if sys.version_info.major == 2:
+#     bytes = str
+#     str = unicode
+
+
+class PartialLatexToLatexEncoder(UnicodeToLatexEncoder):
+    r"""
+    Encode a string while preserving some (fuzzily detected) LaTeX constructs
+    that the input string already has (e.g. accent macros or inline math modes).
+
+    Sometimes you need to fully LaTeX-encode a string that already has some
+    LaTeX constructs.  For instance, titles of bibliographic entries might
+    include some inline math or accents, but they might also include unicode
+    characters that need to be encoded.  Using a
+    :py:class:`UnicodeToLatexEncoder` on such strings would result in ugly
+    doubly-escaped strings such as ``\textbackslash{}'\{e\}``.  Instead,
+    constructs such as ``\'{e}`` should be preserved while other characters
+    and/or constructs (say '&' or '%') as well as unicode characters should be
+    encoded.
+
+    This class offers a simple partial solution: Characters are encoded as per
+    the given `conversion_rules` (or the default conversion rules of
+    :py:class:`UnicodeToLatexEncoder` objects), except that the characters in
+    `keep_latex_chars` are to be interpreted as LaTeX and are not to be further
+    encoded.
+
+    .. versionadded: 2.10
+    """
+
+    def __init__(
+        self,
+        # keyword arguments:
+        keep_latex_chars=r"\${}^_",
+        conversion_rules=None,
+        **kwargs
+    ):
+
+        base_conversion_rules = conversion_rules
+        if base_conversion_rules is None:
+            base_conversion_rules = ["defaults"]
+
+        super(PartialLatexToLatexEncoder, self).__init__(
+            # only a single rule, our own special method that tries to parse
+            # partial latex.
+            conversion_rules=[
+                UnicodeToLatexConversionRule(
+                    rule_type=RULE_CALLABLE,
+                    rule=self._do_partial_latex_encode_step,
+                    replacement_latex_protection="none",
+                )
+            ]
+            + base_conversion_rules,
+            **kwargs
+        )
+
+        self.keep_latex_chars = keep_latex_chars
+
+    def _do_partial_latex_encode_step(self, s, pos):
+        r"""
+        This method is used as a "callable rule" for the
+        :py:class:`UnicodeToLatexEncoder` object.
+
+        The strategy is to see if we have something that looks like a LaTeX char
+        we want to keep.  If so, keep it as is; if not, return `None` so that
+        further rules can be considered by the base unicode encoder.
+        """
+
+        from ..latexwalker import LatexWalker
+
+        if s[pos] in self.keep_latex_chars:
+            # Read a token and if it is a macro, keep the full macro!
+            lw = LatexWalker(s, tolerant_parsing=False)
+
+            tok = lw.get_token(pos, environments=False)
+
+            tok_as_latex = tok.pre_space + s[tok.pos : tok.pos + tok.len]
+
+            # keep the LaTeX token as-is
+            return (tok.pos + tok.len - pos, tok_as_latex)
+
+        return None

great_ai/utilities/external/pylatexenc/latexencode/_unicode_to_latex_encoder.py

@@ -0,0 +1,686 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2021 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import functools
+import itertools
+import logging
+import sys
+import unicodedata
+
+if sys.version_info.major > 2:
+    unicode = str  # need to support unicode() w/ no arguments
+    basestring = str
+    # use MappingProxyType for keeping
+    # inspect function argument names
+    from inspect import getfullargspec
+    from types import MappingProxyType as _MappingProxyType
+else:
+    _MappingProxyType = dict
+    # inspect function argument names -- simulate getfullargspec with getargspec (argh...)
+    from inspect import getargspec as getfullargspec
+
+logger = logging.getLogger(__name__)
+
+
+def get_builtin_uni2latex_dict():
+    r"""
+    Return a dictionary that contains the default collection of known LaTeX
+    escape sequences for unicode characters.
+
+    The keys of the dictionary are integers that correspond to unicode code
+    points (i.e., `ord(char)`).  The values are the corresponding LaTeX
+    replacement strings.
+
+    The returned dictionary may not be modified.  To alter the behavior of
+    :py:func:`unicode_to_latex()`, you should specify custom rules to a new
+    instance of :py:class:`UnicodeToLatexEncoder`.
+
+    .. versionadded:: 2.0
+
+       This function was introduced in `pylatexenc 2.0`.
+    """
+    from ._uni2latexmap import uni2latex as _uni2latex
+
+    return _MappingProxyType(_uni2latex)
+
+
+RULE_DICT = 0
+r"""
+Indicates a rule type that is a dictionary of unicode point values to
+replacement strings. See :py:class:`UnicodeToLatexConversionRule`.
+
+.. versionadded:: 2.0
+
+   This member was introduced in pylatexenc version 2.0.
+"""
+
+RULE_REGEX = 1
+r"""
+Indicates a rule type that is a list (or iterable) of pairs
+`(compiled_regular_expression, replacement_string)`.  See
+:py:class:`UnicodeToLatexConversionRule`.
+
+.. versionadded:: 2.0
+
+   This member was introduced in pylatexenc version 2.0.
+"""
+
+RULE_CALLABLE = 2
+r"""
+Indicates a rule type that is a custom callable.  See
+:py:class:`UnicodeToLatexConversionRule`.
+
+.. versionadded:: 2.0
+
+   This member was introduced in pylatexenc version 2.0.
+"""
+
+
+class UnicodeToLatexConversionRule:
+    r"""
+    Specify a rule how to convert unicode characters into LaTeX escapes.
+
+    .. py:attribute:: rule_type
+    
+       One of :py:data:`RULE_DICT`, :py:data:`RULE_REGEX`, or
+       :py:data:`RULE_CALLABLE`.
+
+    .. py:attribute:: rule
+
+       A specification of the rule itself.  The `rule` attribute is an object
+       that depends on what `rule_type` is set to.  See below.
+
+    .. py:attribute:: replacement_latex_protection
+
+       If non-`None`, then the setting here will override any
+       `replacement_latex_protection` set on
+       :py:class:`UnicodeToLatexConversionRule` objects.  By default the value
+       is `None`, and you can set a replacement_latex_protection globally for
+       all rules on the :py:class:`UnicodeToLatexEncoder` object.
+
+       The use of this attribute is mainly in case you have a fancy rule in
+       which you already guarantee that whatever you output is valid LaTeX even
+       if concatenated with the remainder of the string; in this case you can
+       set `replacement_latex_protection='none'` to avoid unnecessary or
+       unwanted braces around the generated code.
+
+       .. versionadded:: 2.10
+
+          The `replacement_latex_protection` attribute was introduced in
+          `pylatexenc 2.10`.
+
+
+    Constructor syntax::
+    
+        UnicodeToLatexConversionRule(RULE_XXX, <...>)
+        UnicodeToLatexConversionRule(rule_type=RULE_XXX, rule=<...>)
+
+        UnicodeToLatexConversionRule(..., replacement_latex_protection='none')
+
+    Note that you can get some built-in rules via the
+    :py:func:`get_builtin_conversion_rules()` function::
+
+        conversion_rules = get_builtin_conversion_rules('defaults') # all defaults
+
+
+    Rules types:
+    
+      - `RULE_DICT`: If `rule_type` is `RULE_DICT`, then `rule` should be a
+        dictionary whose keys are integers representing unicode code points
+        (e.g., `0x210F`), and whose values are corresponding replacement strings
+        (e.g., ``r'\hbar'``).  See :py:func:`get_builtin_uni2latex_dict()` for
+        an example.
+
+      - `RULE_REGEX`: If `rule_type` is `RULE_REGEX`, then `rule` should be an
+        iterable of tuple pairs `(compiled_regular_expression,
+        replacement_string)` where `compiled_regular_expression` was obtained
+        with `re.compile(...)` and `replacement_string` is anything that can be
+        specified as the second (`repl`) argument of `re.sub(...)`.  This can be
+        a replacement string that includes escapes (like ``\1, \2, \g<name>``)
+        for captured sub-expressions or a callable that takes a match object as
+        argument.
+
+        .. note::
+    
+           The replacement string is parsed like the second argument to
+           `re.sub()` and backslashes have a special meaning because they can
+           refer to captured sub-expressions.  For a literal backslash, use two
+           backslashes ``\\`` in raw strings, four backslashes in normal
+           strings.
+
+        Example::
+
+          regex_conversion_rule = UnicodeToLatexConversionRule(
+              rule_type=RULE_REGEX,
+              rule=[
+                  # protect acronyms of capital letters with braces,
+                  # e.g.: ABC -> {ABC}
+                  (re.compile(r'[A-Z]{2,}'), r'{\1}'),
+                  # Additional rules, e.g., "..." -> "\ldots"
+                  (re.compile(r'...'), r'\\ldots'), # note double \\
+              ]
+          )
+
+      - `RULE_CALLABLE`: If `rule_type` is `RULE_CALLABLE`, then `rule` should
+        be a callable that accepts two arguments, the unicode string and the
+        position in the string (an integer).  The callable will be called with
+        the original unicode string as argument and the position of the
+        character that needs to be encoded.  If this rule can encode the given
+        character at the given position, it should return a tuple
+        `(consumed_length, replacement_string)` where `consumed_length` is the
+        number of characters in the unicode string that `replacement_string`
+        represents.  If the character(s) at the given position can't be encoded
+        by this rule, the callable should return `None` to indicate that further
+        rules should be attempted.
+
+        If the callable accepts an additional argument called `u2lobj`, then the
+        :py:class:`UnicodeToLatexEncoder` instance is provided to that argument.
+
+        For example, the following callable should achieve the same effect as
+        the previous example with regexes::
+
+          def convert_stuff(s, pos):
+              m = re.match(r'[A-Z]{2,}', s, pos)
+              if m is not None:
+                  return (m.end()-m.start(), '{'+m.group()+'}')
+              if s.startswith('...', pos): # or  s[pos:pos+3] == '...'
+                  return (3, r'\ldots')
+              return None
+
+
+    .. versionadded:: 2.0
+
+       This class was introduced in `pylatexenc 2.0`.
+    """
+
+    def __init__(
+        self,
+        rule_type,
+        rule=None,
+        # keyword-only, please:
+        replacement_latex_protection=None,
+    ):
+        self.rule_type = rule_type
+        self.rule = rule
+        self.replacement_latex_protection = replacement_latex_protection
+
+    def __repr__(self):
+        return "{}(rule_type={!r}, rule=<{}>, replacement_latex_protection={})".format(
+            self.__class__.__name__,
+            self.rule_type,
+            type(self.rule).__name__,
+            repr(self.replacement_latex_protection),
+        )
+
+
+def get_builtin_conversion_rules(builtin_name):
+    r"""
+    Return a built-in set of conversion rules specified by a given name
+    `builtin_name`.
+
+    There are two builtin conversion rules, with the following names:
+
+      - `'defaults'`: the default conversion rules, a custom-curated list of
+        unicode chars to LaTeX escapes.
+
+      - `'unicode-xml'`: the conversion rules derived from the `unicode.xml` file
+        maintained at https://www.w3.org/TR/xml-entity-names/#source by David
+        Carlisle.
+
+    The return value is a list of :py:class:`UnicodeToLatexConversionRule`
+    objects that can be either directly specified to the `conversion_rules=`
+    argument of :py:class:`UnicodeToLatexEncoder`, or included in a larger list
+    that can be provided to that argument.
+
+    .. versionadded:: 2.0
+
+       This function was introduced in `pylatexenc 2.0`.
+    """
+    if builtin_name == "defaults":
+        return [
+            UnicodeToLatexConversionRule(
+                rule_type=RULE_DICT, rule=get_builtin_uni2latex_dict()
+            )
+        ]
+    if builtin_name == "unicode-xml":
+        from . import _uni2latexmap_xml
+
+        return [
+            UnicodeToLatexConversionRule(
+                rule_type=RULE_DICT, rule=_uni2latexmap_xml.uni2latex
+            )
+        ]
+    raise ValueError("Unknown builtin rule set: {}".format(builtin_name))
+
+
+class UnicodeToLatexEncoder(object):
+    r"""
+    Encode a string with unicode characters into a LaTeX snippet.
+
+    The following general attributes can be specified as keyword arguments to
+    the constructor.  Note: These attributes must be specified to the
+    constructor and may NOT be subsequently modified.  This is because in the
+    constructor we pre-compile some rules and flags to optimize calls to
+    :py:meth:`unicode_to_text()`.
+
+    .. py:attribute:: non_ascii_only
+
+       Whether we should convert only non-ascii characters into LaTeX sequences,
+       or also all known ascii characters with special LaTeX meaning such as
+       '\\\\', '$', '&', etc.
+
+       If `non_ascii_only` is set to `True` (the default is `False`), then
+       conversion rules are not applied at positions in the string where an
+       ASCII character is encountered.
+
+    .. py:attribute:: conversion_rules
+
+       The conversion rules, specified as a list of
+       :py:class:`UnicodeToLatexConversionRule` objects.  For each position in
+       the string, the rules will be applied in the given sequence until a
+       replacement string is found.
+
+       Instead of a :py:class:`UnicodeToLatexConversionRule` object you may also
+       specify a string specifying a built-in rule (e.g., 'defaults'), which
+       will be expanded to the corresponding rules according to
+       :py:func:`get_builtin_conversion_rules()`.
+
+       If you specify your own list of rules using this argument, you will
+       probably want to include presumably at the end of your list the element
+       'defaults' to include all built-in default conversion rules.  To override
+       built-in rules, simply add your custom rules earlier in the list.
+       Example::
+
+         conversion_rules = [
+             # our custom rules
+             UnicodeToLatexConversionRule(RULE_REGEX, [
+                 # double \\ needed, see UnicodeToLatexConversionRule
+                 ( re.compile(r'...'), r'\\ldots' ),
+                 ( re.compile(r'î'), r'\\^i' ),
+             ]),
+             # plus all the default rules
+             'defaults'
+         ]
+         u = UnicodeToLatexEncoder(conversion_rules=conversion_rules)
+
+    .. py:attribute:: replacement_latex_protection
+
+       How to "protect" LaTeX replacement text that looks like it could be
+       interpreted differently if concatenated to arbitrary strings before and
+       after.
+
+       Currently in the default scheme only one situation is recognized: if the
+       replacement string ends with a latex macro invocation with a non-symbol
+       macro name, e.g. ``\textemdash`` or ``\^\i``.  Indeed, if we naively
+       replace these texts in an arbitrary string (like ``maître``), we might
+       get an invalid macro invocation (like ``ma\^\itre`` which causes un known
+       macro name ``\itre``).
+
+       Possible protection schemes are:
+
+         - 'braces' (the default):  Any suspicious replacement text (that
+           might look fragile) is placed in curly braces ``{...}``.
+
+         - 'braces-all':  All replacement latex escapes are surrounded in
+           protective curly braces ``{...}``, regardless of whether or not they
+           might be deemed "fragile" or "unsafe".
+
+         - 'braces-almost-all':  Almost all replacement latex escapes are
+           surrounded in protective curly braces ``{...}``.  This option
+           emulates closely the behavior of `brackets=True` of the function
+           `utf8tolatex()` in `pylatexenc 1.x`, though I'm not sure it is really
+           useful.  [Specifically, all those replacement strings that start with
+           a backslash are surrounded by curly braces].
+
+         - 'braces-after-macro':  In the situation where the replacement latex
+           code ends with a string-named macro, then a pair of empty braces is
+           added at the end of the replacement text to protect the macro.
+
+         - 'none': No protection is applied, even in "unsafe" cases.  This is
+           not recommended, as this will likely result in invalid LaTeX
+           code. (Note this is the string 'none', not Python's built-in `None`.)
+
+         - any callable object: The callable should take a single argument, the
+           replacement latex string associated with a piece of the input (maybe
+           a special character) that has been encoded; it should return the
+           actual string to append to the output string.
+
+         .. versionadded:: 2.10
+
+            You can specify a callable object to `replacement_latex_protection`
+            since `pylatexenc 2.10`.
+
+    .. py:attribute:: unknown_char_policy
+
+       What to do when a non-ascii character is encountered without any known
+       substitution macro.  The attribute `unknown_char_policy` can be set to one of:
+
+         - 'keep': keep the character as is;
+
+         - 'replace': replace the character by a boldface question mark;
+
+         - 'ignore': ignore the character from the input entirely and don't
+           output anything for it;
+
+         - 'fail': raise a `ValueError` exception;
+
+         - 'unihex': output the unicode hexadecimal code (U+XXXX) of the
+           character in typewriter font;
+
+         - a Python callable --- will be called with argument the character that
+           could not be encoded.  (If the callable accepts a second argument
+           called 'u2lobj', then the `UnicodeToLatexEncoder` instance is
+           provided to that argument.)  The return value of the callable is used
+           as LaTeX replacement code.
+
+    .. py:attribute:: unknown_char_warning
+
+       In addition to the `unknown_char_policy`, this attribute indicates
+       whether or not (`True` or `False`) one should generate a warning when a
+       nonascii character without any known latex representation is
+       encountered. (Default: True)
+
+    .. py:attribute:: latex_string_class
+
+       The return type of :py:meth:`unicode_to_latex()`.  Normally this is a
+       simple unicode string (`str` on `Python 3` or `unicode` on `Python 2`).
+
+       But you can specify your custom string type via the `latex_string_class`
+       argument.  The `latex_string_class` will be invoked with no arguments to
+       construct an empty object (so `latex_string_class` can be either an
+       object that can be constructed with no arguments or it can be a function
+       with no arguments that return a fresh object instance).  The object must
+       support the operation "+=", i.e., you should overload the ``__iadd__()``
+       method.
+
+       For instance, you can record the chunks that would have been appended
+       into a single string as follows::
+
+           class LatexChunkList:
+               def __init__(self):
+                   self.chunks = []
+
+               def __iadd__(self, s):
+                   self.chunks.append(s)
+                   return self
+
+           u = UnicodeToLatexEncoder(latex_string_class=LatexChunkList,
+                                     replacement_latex_protection='none')
+           result = u.unicode_to_latex("é → α")
+           # result.chunks == [ r"\'e", ' ', r'\textrightarrow', ' ',
+           #                    r'\ensuremath{\alpha}' ]
+
+    .. warning::
+
+       None of the above attributes should be modified after constructing the
+       object.  The values specified to the class constructor are final and
+       cannot be changed.  [Indeed, the class constructor "compiles" these
+       attribute values into a data structure that makes
+       :py:meth:`unicode_to_text()` slightly more efficient.]
+
+    .. versionadded:: 2.0
+
+       This class was introduced in `pylatexenc 2.0`.
+    """
+
+    def __init__(self, **kwargs):
+        self.non_ascii_only = kwargs.pop("non_ascii_only", False)
+        self.conversion_rules = kwargs.pop("conversion_rules", ["defaults"])
+        self.replacement_latex_protection = kwargs.pop(
+            "replacement_latex_protection", "braces"
+        )
+        self.unknown_char_policy = kwargs.pop("unknown_char_policy", "keep")
+        self.unknown_char_warning = kwargs.pop("unknown_char_warning", True)
+        self.latex_string_class = kwargs.pop("latex_string_class", unicode)
+
+        if kwargs:
+            logger.warning(
+                "Ignoring unknown keyword arguments: %s", ",".join(kwargs.keys())
+            )
+
+        super(UnicodeToLatexEncoder, self).__init__(**kwargs)
+
+        # build generator that expands built-in conversion rules
+        expanded_conversion_rules = itertools.chain.from_iterable(
+            (get_builtin_conversion_rules(r) if isinstance(r, basestring) else [r])
+            for r in self.conversion_rules
+        )
+
+        #
+        # now "pre-compile" some stuff so that calls to unicode_to_latex() can
+        # hopefully execute faster
+        #
+
+        # "pre-compile" rules and check rule types:
+        self._compiled_rules = []
+        for rule in expanded_conversion_rules:
+            if rule.rule_type == RULE_DICT:
+                self._compiled_rules.append(
+                    functools.partial(self._apply_rule_dict, rule.rule, rule)
+                )
+            elif rule.rule_type == RULE_REGEX:
+                self._compiled_rules.append(
+                    functools.partial(self._apply_rule_regex, rule.rule, rule)
+                )
+            elif rule.rule_type == RULE_CALLABLE:
+                thecallable = rule.rule
+                if "u2lobj" in getfullargspec(thecallable)[0]:
+                    thecallable = functools.partial(rule.rule, u2lobj=self)
+                self._compiled_rules.append(
+                    functools.partial(self._apply_rule_callable, thecallable, rule)
+                )
+            else:
+                raise TypeError("Invalid rule type: {}".format(rule.rule_type))
+
+        # bad char policy:
+        if isinstance(self.unknown_char_policy, basestring):
+            self._do_unknown_char = self._get_method_fn(
+                "do_unknown_char", self.unknown_char_policy, what="unknown_char_policy"
+            )
+        elif callable(self.unknown_char_policy):
+            fn = self.unknown_char_policy
+            if "u2lobj" in getfullargspec(fn)[0]:
+                self._do_unknown_char = functools.partial(
+                    self.unknown_char_policy, u2lobj=self
+                )
+            else:
+                self._do_unknown_char = self.unknown_char_policy
+        else:
+            raise TypeError(
+                "Invalid argument for unknown_char_policy: {!r}".format(
+                    self.unknown_char_policy
+                )
+            )
+
+        # bad char warning:
+        if not self.unknown_char_warning:
+            self._do_warn_unknown_char = lambda ch: None  # replace method by no-op
+
+        # set a method that will skip ascii characters if required:
+        if self.non_ascii_only:
+            self._maybe_skip_ascii = self._check_do_skip_ascii
+        else:
+            self._maybe_skip_ascii = lambda s, p: False
+
+        # set a method to protect replacement latex code, if necessary:
+        self._apply_protection = self._get_replacement_latex_fn(
+            self.replacement_latex_protection
+        )
+
+    def _get_method_fn(self, base, name, what):
+        selfmethname = "_" + base + "_" + name.replace("-", "_")
+        if not hasattr(self, selfmethname):
+            raise ValueError("Invalid {}: {}".format(what, name))
+        return getattr(self, selfmethname)
+
+    def _get_replacement_latex_fn(self, replacement_latex_protection):
+        if callable(replacement_latex_protection):
+            return replacement_latex_protection
+        return self._get_method_fn(
+            "apply_protection",
+            replacement_latex_protection,
+            what="replacement_latex_protection",
+        )
+
+    def unicode_to_latex(self, s):
+        """
+        Convert unicode characters in the string `s` into latex escape sequences,
+        according to the rules and options given to the constructor.
+        """
+
+        s = unicode(s)  # make sure s is unicode
+        s = unicodedata.normalize("NFC", s)
+
+        class _NS:
+            pass
+
+        p = _NS()
+        p.latex = self.latex_string_class()
+        p.pos = 0
+
+        while p.pos < len(s):
+
+            if self._maybe_skip_ascii(s, p):
+                continue
+
+            for compiledrule in self._compiled_rules:
+                if compiledrule(s, p):
+                    break
+            else:
+                # for-else, see
+                # https://docs.python.org/2/tutorial/controlflow.html\
+                # #break-and-continue-statements-and-else-clauses-on-loops
+                ch = s[p.pos]
+                o = ord(ch)
+                if (o >= 32 and o <= 127) or (ch in "\n\r\t"):
+                    p.latex += ch
+                    p.pos += 1
+                else:
+                    self._do_warn_unknown_char(ch)
+                    p.latex += self._do_unknown_char(ch)
+                    p.pos += 1
+
+        return p.latex
+
+    def _check_do_skip_ascii(self, s, p):
+        if ord(s[p.pos]) < 127:
+            # skip, we only want to convert non-ascii chars
+            p.latex += s[p.pos]
+            p.pos += 1
+            return True
+        return False
+
+    def _apply_rule_dict(self, ruledict, rule, s, p):
+        o = ord(s[p.pos])
+        if o in ruledict:
+            self._apply_replacement(p, ruledict[o], 1, rule)
+            return True
+        return None
+
+    def _apply_rule_regex(self, ruleregexes, rule, s, p):
+        for regex, repl in ruleregexes:
+            m = regex.match(s, p.pos)
+            if m is not None:
+                if callable(repl):
+                    replstr = repl(m)
+                else:
+                    replstr = m.expand(repl)
+                self._apply_replacement(p, replstr, m.end() - m.start(), rule)
+                return True
+        return None
+
+    def _apply_rule_callable(self, rulecallable, rule, s, p):
+        res = rulecallable(s, p.pos)
+        if res is None:
+            return None
+        (consumed, repl) = res
+        self._apply_replacement(p, repl, consumed, rule)
+        return True
+
+    def _apply_replacement(self, p, repl, numchars, ruleobj):
+        # check for possible replacement latex protection, like braces.
+
+        protect_fn = self._apply_protection
+
+        # maybe the rule object has overridden the replacement_latex_protection to use.
+        if ruleobj.replacement_latex_protection is not None:
+            protect_fn = self._get_replacement_latex_fn(
+                ruleobj.replacement_latex_protection
+            )
+
+        repl = protect_fn(repl)
+        p.latex += repl
+        p.pos += numchars
+
+    def _apply_protection_none(self, repl):
+        # no protection
+        return repl
+
+    def _apply_protection_braces(self, repl):
+        k = repl.rfind("\\")
+        if k >= 0 and repl[k + 1 :].isalpha():
+            # has dangling named macro, apply protection.
+            return "{" + repl + "}"
+        return repl
+
+    def _apply_protection_braces_almost_all(self, repl):
+        if repl[0:1] == "\\":
+            return "{" + repl + "}"
+        return repl
+
+    def _apply_protection_braces_all(self, repl):
+        return "{" + repl + "}"
+
+    def _apply_protection_braces_after_macro(self, repl):
+        k = repl.rfind("\\")
+        if k >= 0 and repl[k + 1 :].isalpha():
+            # has dangling named macro, apply protection.
+            return repl + "{}"
+        return repl
+
+    # policies for "bad chars":
+    def _do_unknown_char_keep(self, ch):
+        return ch
+
+    def _do_unknown_char_replace(self, ch):
+        return r"{\bfseries ?}"
+
+    def _do_unknown_char_ignore(self, ch):
+        return ""
+
+    def _do_unknown_char_fail(self, ch):
+        raise ValueError(
+            "No known latex representation for character: U+%04X - ‘%s’" % (ord(ch), ch)
+        )
+
+    def _do_unknown_char_unihex(self, ch):
+        return r"\ensuremath{\langle}\texttt{U+%04X}\ensuremath{\rangle}" % (ord(ch))
+
+    def _do_warn_unknown_char(self, ch):
+        logger.warning(
+            "No known latex representation for character: U+%04X - ‘%s’", ord(ch), ch
+        )

great_ai/utilities/external/pylatexenc/latexwalker/__main__.py

@@ -0,0 +1,230 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2018 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+import argparse
+import fileinput
+import json
+import logging
+import sys
+
+from ..latexwalker import LatexWalker, disp_node, make_json_encoder
+from ..version import version_str
+
+
+def main(argv=None):
+
+    if argv is None:
+        argv = sys.argv[1:]
+
+    parser = argparse.ArgumentParser(prog="latexwalker", add_help=False)
+
+    parser.add_argument(
+        "--output-format",
+        metavar="FORMAT",
+        dest="output_format",
+        choices=["human", "json"],
+        default="human",
+        help='Requested output format for the node tree ("human" or "json")',
+    )
+    parser.add_argument(
+        "--json-indent",
+        metavar="NUMSPACES",
+        dest="json_indent",
+        type=int,
+        default=2,
+        help="Indentation in JSON output (specify number of spaces "
+        "per indentation level)",
+    )
+    parser.add_argument(
+        "--json-compact",
+        dest="json_indent",
+        action="store_const",
+        const=None,
+        help="Output compact JSON",
+    )
+
+    parser.add_argument(
+        "--keep-inline-math",
+        action="store_const",
+        const=True,
+        dest="keep_inline_math",
+        default=True,
+        help=argparse.SUPPRESS,
+    )
+    parser.add_argument(
+        "--no-keep-inline-math",
+        action="store_const",
+        const=False,
+        dest="keep_inline_math",
+        help=argparse.SUPPRESS,
+    )
+
+    parser.add_argument(
+        "--tolerant-parsing",
+        action="store_const",
+        const=True,
+        dest="tolerant_parsing",
+        default=True,
+    )
+    parser.add_argument(
+        "--no-tolerant-parsing",
+        action="store_const",
+        const=False,
+        dest="tolerant_parsing",
+        help="Tolerate syntax errors when parsing, and attempt "
+        "to continue (default yes)",
+    )
+
+    # I'm not sure this flag is useful and if it should be exposed at all.
+    # Accept it, but make it hidden.
+    parser.add_argument(
+        "--strict-braces",
+        action="store_const",
+        const=True,
+        dest="strict_braces",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
+    parser.add_argument(
+        "--no-strict-braces",
+        action="store_const",
+        const=False,
+        dest="strict_braces",
+        # help="Report errors for mismatching LaTeX braces (default no)"
+        help=argparse.SUPPRESS,
+    )
+
+    parser.add_argument(
+        "-q",
+        "--quiet",
+        dest="logging_level",
+        action="store_const",
+        const=logging.ERROR,
+        default=logging.INFO,
+        help="Suppress warning messages",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        dest="logging_level",
+        action="store_const",
+        const=logging.DEBUG,
+        help="Verbose output",
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version="pylatexenc {}".format(version_str),
+        help="Show version information and exit",
+    )
+    parser.add_argument(
+        "--help", action="help", help="Show this help information and exit"
+    )
+
+    parser.add_argument(
+        "--code",
+        "-c",
+        action="store",
+        default=None,
+        metavar="LATEX_CODE",
+        help="Convert the given LATEX_CODE to unicode text instead of reading "
+        "from FILE or standard input.  You cannot specify FILEs if you use this "
+        "option, and any standard input is ignored.",
+    )
+
+    parser.add_argument(
+        "files",
+        metavar="FILE",
+        nargs="*",
+        help="Input files (if none specified, read from stdandard input)",
+    )
+
+    args = parser.parse_args(argv)
+
+    logging.basicConfig()
+    logging.getLogger().setLevel(args.logging_level)
+    logger = logging.getLogger(__name__)
+
+    latex = ""
+    if args.code:
+        if args.files:
+            logger.error(
+                "Cannot specify both FILEs and --code option. "
+                "Use --help option for more information."
+            )
+            sys.exit(1)
+        latex = args.code
+    else:
+        for line in fileinput.input(files=args.files):
+            latex += line
+
+    latexwalker = LatexWalker(
+        latex, tolerant_parsing=args.tolerant_parsing, strict_braces=args.strict_braces
+    )
+
+    (nodelist, pos, len_) = latexwalker.get_latex_nodes()
+
+    if args.output_format == "human":
+        print("\n--- NODES ---\n")
+        for n in nodelist:
+            disp_node(n)
+        print("\n-------------\n")
+        return
+
+    if args.output_format == "json":
+        json.dump(
+            {
+                "nodelist": nodelist,
+            },
+            sys.stdout,
+            cls=make_json_encoder(latexwalker),
+            indent=args.json_indent,
+        )
+        sys.stdout.write("\n")
+        return
+
+    raise ValueError("Invalid output format: " + args.output_format)
+
+
+def run_main():
+
+    try:
+
+        main()
+
+    except SystemExit:
+        raise
+    except:  # lgtm [py/catch-base-exception]
+        import pdb
+        import traceback
+
+        traceback.print_exc()
+        pdb.post_mortem()
+
+
+if __name__ == "__main__":
+
+    # run_main() # debug
+    main()

great_ai/utilities/external/pylatexenc/latexwalker/_defaultspecs.py

@@ -0,0 +1,459 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2019 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+
+# Internal module. May change without notice.
+
+
+from ..macrospec import (
+    EnvironmentSpec,
+    MacroSpec,
+    MacroStandardArgsParser,
+    VerbatimArgsParser,
+    std_environment,
+    std_macro,
+    std_specials,
+)
+
+specs = [
+    #
+    # CATEGORY: latex-base
+    #
+    (
+        "latex-base",
+        {
+            "macros": [
+                std_macro("documentclass", True, 1),
+                std_macro("usepackage", True, 1),
+                std_macro("RequirePackage", True, 1),
+                std_macro("selectlanguage", True, 1),
+                std_macro("setlength", True, 2),
+                std_macro("addlength", True, 2),
+                std_macro("setcounter", True, 2),
+                std_macro("addcounter", True, 2),
+                std_macro("newcommand", "*{[[{"),
+                std_macro("renewcommand", "*{[[{"),
+                std_macro("providecommand", "*{[[{"),
+                std_macro("newenvironment", "*{[[{{"),
+                std_macro("renewenvironment", "*{[[{{"),
+                std_macro("provideenvironment", "*{[[{{"),
+                std_macro("DeclareMathOperator", "*{{"),
+                std_macro("hspace", "*{"),
+                std_macro("vspace", "*{"),
+                MacroSpec(
+                    "mbox",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                # \title, \author, \date
+                MacroSpec("title", "{"),
+                MacroSpec("author", "{"),
+                MacroSpec("date", "{"),
+                # (Note: single backslash) end of line with optional no-break ('*') and
+                # additional vertical spacing, e.g. \\*[2mm]
+                #
+                # Special for this command: don't allow an optional spacing argument
+                # [2mm] to be separated by spaces from the rest of the macro.  This
+                # emulates the behavior in AMS environments, and avoids some errors;
+                # e.g. in "\begin{align} A=0 \\ [C,D]=0 \end{align}" the "[C,D]"
+                # does not get captured as an optional macro argument.
+                MacroSpec(
+                    "\\",
+                    args_parser=MacroStandardArgsParser(
+                        "*[", optional_arg_no_space=True
+                    ),
+                ),
+                std_macro("item", True, 0),
+                # \input{someotherfile}
+                std_macro("input", False, 1),
+                std_macro("include", False, 1),
+                std_macro("includegraphics", True, 1),
+                std_macro("chapter", "*[{"),
+                std_macro("section", "*[{"),
+                std_macro("subsection", "*[{"),
+                std_macro("subsubsection", "*[{"),
+                std_macro("pagagraph", "*[{"),
+                std_macro("subparagraph", "*[{"),
+                std_macro("bibliography", "{"),
+                std_macro("emph", False, 1),
+                MacroSpec(
+                    "textrm",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "textit",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "textbf",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "textmd",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "textsc",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "textsf",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "textsl",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "texttt",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "textup",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "text",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                std_macro("mathrm", False, 1),  # only allowed in math mode anyway
+                std_macro("mathbb", False, 1),  # only allowed in math mode anyway
+                std_macro("mathbf", False, 1),
+                std_macro("mathit", False, 1),
+                std_macro("mathsf", False, 1),
+                std_macro("mathtt", False, 1),
+                std_macro("mathcal", False, 1),
+                std_macro("mathscr", False, 1),
+                std_macro("mathfrak", False, 1),
+                std_macro("label", False, 1),
+                std_macro("ref", False, 1),
+                std_macro("autoref", False, 1),
+                std_macro("cref", False, 1),
+                std_macro("Cref", False, 1),
+                std_macro("eqref", False, 1),
+                std_macro("url", False, 1),
+                std_macro("hypersetup", False, 1),
+                std_macro("footnote", True, 1),
+                std_macro("keywords", False, 1),
+                std_macro("hphantom", True, 1),
+                std_macro("vphantom", True, 1),
+                std_macro("'", False, 1),
+                std_macro("`", False, 1),
+                std_macro('"', False, 1),
+                std_macro("c", False, 1),
+                std_macro("^", False, 1),
+                std_macro("~", False, 1),
+                std_macro("H", False, 1),
+                std_macro("k", False, 1),
+                std_macro("=", False, 1),
+                std_macro("b", False, 1),
+                std_macro(".", False, 1),
+                std_macro("d", False, 1),
+                std_macro("r", False, 1),
+                std_macro("u", False, 1),
+                std_macro("v", False, 1),
+                MacroSpec(
+                    "ensuremath",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[True]),
+                ),
+                std_macro("not", False, 1),
+                std_macro("vec", False, 1),
+                std_macro("dot", False, 1),
+                std_macro("hat", False, 1),
+                std_macro("check", False, 1),
+                std_macro("breve", False, 1),
+                std_macro("acute", False, 1),
+                std_macro("grave", False, 1),
+                std_macro("tilde", False, 1),
+                std_macro("bar", False, 1),
+                std_macro("ddot", False, 1),
+                std_macro("frac", False, 2),
+                std_macro("nicefrac", False, 2),
+                std_macro("sqrt", True, 1),
+                MacroSpec("overline", "{"),
+                MacroSpec("underline", "{"),
+                MacroSpec("widehat", "{"),
+                MacroSpec("widetilde", "{"),
+                MacroSpec("wideparen", "{"),
+                MacroSpec("overleftarrow", "{"),
+                MacroSpec("overrightarrow", "{"),
+                MacroSpec("overleftrightarrow", "{"),
+                MacroSpec("underleftarrow", "{"),
+                MacroSpec("underrightarrow", "{"),
+                MacroSpec("underleftrightarrow", "{"),
+                MacroSpec("overbrace", "{"),
+                MacroSpec("underbrace", "{"),
+                MacroSpec("overgroup", "{"),
+                MacroSpec("undergroup", "{"),
+                MacroSpec("overbracket", "{"),
+                MacroSpec("underbracket", "{"),
+                MacroSpec("overlinesegment", "{"),
+                MacroSpec("underlinesegment", "{"),
+                MacroSpec("overleftharpoon", "{"),
+                MacroSpec("overrightharpoon", "{"),
+                MacroSpec("xleftarrow", "[{"),
+                MacroSpec("xrightarrow", "[{"),
+                std_macro("ket", False, 1),
+                std_macro("bra", False, 1),
+                std_macro("braket", False, 2),
+                std_macro("ketbra", False, 2),
+                std_macro("texorpdfstring", False, 2),
+                # xcolor commands
+                MacroSpec("definecolor", "[{{{"),
+                MacroSpec("providecolor", "[{{{"),
+                MacroSpec("colorlet", "[{[{"),
+                MacroSpec("color", "[{"),
+                MacroSpec("textcolor", "[{{"),
+                MacroSpec("pagecolor", "[{"),
+                MacroSpec("nopagecolor", ""),
+                MacroSpec("colorbox", "[{{"),
+                MacroSpec("fcolorbox", "[{[{{"),
+                MacroSpec("boxframe", "{{{"),
+                MacroSpec("rowcolors", "*[{{{"),
+            ],
+            "environments": [
+                # NOTE: Starred variants (as in \begin{equation*}) are not specified as
+                # for macros with an argspec='*'.  Rather, we need to define a separate
+                # spec for the starred variant as the star really is part of the
+                # environment name.  If you specify argspec='*', the parser will try to
+                # look for an expression of the form '\begin{equation}*'
+                std_environment("figure", "["),
+                std_environment("figure*", "["),
+                std_environment("table", "["),
+                std_environment("table*", "["),
+                std_environment("abstract", None),
+                std_environment("tabular", "{"),
+                std_environment("tabular*", "{{"),
+                std_environment("tabularx", "{[{"),
+                std_environment("array", "[{"),
+                std_environment("equation", None, is_math_mode=True),
+                std_environment("equation*", None, is_math_mode=True),
+                std_environment("eqnarray", None, is_math_mode=True),
+                std_environment("eqnarray*", None, is_math_mode=True),
+                # AMS environments
+                std_environment("align", None, is_math_mode=True),
+                std_environment("align*", None, is_math_mode=True),
+                std_environment("gather", None, is_math_mode=True),
+                std_environment("gather*", None, is_math_mode=True),
+                std_environment("flalign", None, is_math_mode=True),
+                std_environment("flalign*", None, is_math_mode=True),
+                std_environment("multline", None, is_math_mode=True),
+                std_environment("multline*", None, is_math_mode=True),
+                std_environment("alignat", "{", is_math_mode=True),
+                std_environment("alignat*", "{", is_math_mode=True),
+                std_environment("split", None, is_math_mode=True),
+            ],
+            "specials": [
+                std_specials("&"),
+                # TODO --- for this, we need to parse their argument but don't use
+                #          the standard args parser because we need to be able to
+                #          accept arguments like "x_\mathrm{initial}"
+                #
+                # std_specials('^'),
+                # std_specials('_'),
+            ],
+        },
+    ),
+    #
+    # CATEGORY: nonascii-specials
+    #
+    (
+        "nonascii-specials",
+        {
+            "macros": [],
+            "environments": [],
+            "specials": [
+                std_specials("~"),
+                # cf. https://tex.stackexchange.com/a/439652/32188 "fake ligatures":
+                std_specials("``"),
+                std_specials("''"),
+                std_specials("--"),
+                std_specials("---"),
+                std_specials("!`"),
+                std_specials("?`"),
+            ],
+        },
+    ),
+    #
+    # CATEGORY: verbatim
+    #
+    (
+        "verbatim",
+        {
+            "macros": [
+                MacroSpec(
+                    "verb",
+                    args_parser=VerbatimArgsParser(verbatim_arg_type="verb-macro"),
+                ),
+            ],
+            "environments": [
+                EnvironmentSpec(
+                    "verbatim",
+                    args_parser=VerbatimArgsParser(
+                        verbatim_arg_type="verbatim-environment"
+                    ),
+                ),
+            ],
+            "specials": [
+                # optionally users could include the specials "|" like in latex-doc
+                # for verbatim |\like \this|...
+            ],
+        },
+    ),
+    #
+    # CATEGORY: theorems
+    #
+    (
+        "theorems",
+        {
+            "macros": [],
+            "environments": [
+                std_environment("theorem", "["),
+                std_environment("proposition", "["),
+                std_environment("lemma", "["),
+                std_environment("corollary", "["),
+                std_environment("definition", "["),
+                std_environment("conjecture", "["),
+                std_environment("remark", "["),
+                #
+                std_environment("proof", "["),
+                # short names
+                std_environment("thm", "["),
+                std_environment("prop", "["),
+                std_environment("lem", "["),
+                std_environment("cor", "["),
+                std_environment("conj", "["),
+                std_environment("rem", "["),
+                std_environment("defn", "["),
+            ],
+            "specials": [],
+        },
+    ),
+    #
+    # CATEGORY: enumitem
+    #
+    (
+        "enumitem",
+        {
+            "macros": [],
+            "environments": [
+                std_environment("enumerate", "["),
+                std_environment("itemize", "["),
+                std_environment("description", "["),
+            ],
+            "specials": [],
+        },
+    ),
+    #
+    # CATEGORY: natbib
+    #
+    (
+        "natbib",
+        {
+            "macros": [
+                std_macro("cite", "*[[{"),
+                std_macro("citet", "*[[{"),
+                std_macro("citep", "*[[{"),
+                std_macro("citealt", "*[[{"),
+                std_macro("citealp", "*[[{"),
+                std_macro("citeauthor", "*[[{"),
+                std_macro("citefullauthor", "[[{"),
+                std_macro("citeyear", "[[{"),
+                std_macro("citeyearpar", "[[{"),
+                std_macro("Citet", "*[[{"),
+                std_macro("Citep", "*[[{"),
+                std_macro("Citealt", "*[[{"),
+                std_macro("Citealp", "*[[{"),
+                std_macro("Citeauthor", "*[[{"),
+                std_macro("citetext", "{"),
+                std_macro("citenum", "{"),
+                std_macro("defcitealias", "{{"),
+                std_macro("citetalias", "[[{"),
+                std_macro("citepalias", "[[{"),
+            ],
+            "environments": [],
+            "specials": [],
+        },
+    ),
+    #
+    # CATEGORY: latex-ethuebung
+    #
+    (
+        "latex-ethuebung",
+        {
+            "macros": [
+                # ethuebung
+                std_macro("UebungLoesungFont", False, 1),
+                std_macro("UebungHinweisFont", False, 1),
+                std_macro("UebungExTitleFont", False, 1),
+                std_macro("UebungSubExTitleFont", False, 1),
+                std_macro("UebungTipsFont", False, 1),
+                std_macro("UebungLabel", False, 1),
+                std_macro("UebungSubLabel", False, 1),
+                std_macro("UebungLabelEnum", False, 1),
+                std_macro("UebungLabelEnumSub", False, 1),
+                std_macro("UebungSolLabel", False, 1),
+                std_macro("UebungHinweisLabel", False, 1),
+                std_macro("UebungHinweiseLabel", False, 1),
+                std_macro("UebungSolEquationLabel", False, 1),
+                std_macro("UebungTipsLabel", False, 1),
+                std_macro("UebungTipsEquationLabel", False, 1),
+                std_macro("UebungsblattTitleSeries", False, 1),
+                std_macro("UebungsblattTitleSolutions", False, 1),
+                std_macro("UebungsblattTitleTips", False, 1),
+                std_macro("UebungsblattNumber", False, 1),
+                std_macro("UebungsblattTitleFont", False, 1),
+                std_macro("UebungTitleCenterVSpacing", False, 1),
+                std_macro("UebungAttachedSolutionTitleTop", False, 1),
+                std_macro("UebungAttachedSolutionTitleFont", False, 1),
+                std_macro("UebungAttachedSolutionTitle", False, 1),
+                std_macro("UebungTextAttachedSolution", False, 1),
+                std_macro("UebungDueByLabel", False, 1),
+                std_macro("UebungDueBy", False, 1),
+                std_macro("UebungLecture", False, 1),
+                std_macro("UebungProf", False, 1),
+                std_macro("UebungLecturer", False, 1),
+                std_macro("UebungSemester", False, 1),
+                std_macro("UebungLogoFile", False, 1),
+                std_macro("UebungLanguage", False, 1),
+                std_macro("UebungStyle", False, 1),
+                #
+                std_macro("uebung", "{["),
+                std_macro("exercise", "{["),
+                std_macro("keywords", False, 1),
+                std_macro("subuebung", False, 1),
+                std_macro("subexercise", False, 1),
+                std_macro("pdfloesung", True, 1),
+                std_macro("pdfsolution", True, 1),
+                std_macro("exenumfulllabel", False, 1),
+                std_macro("hint", False, 1),
+                std_macro("hints", False, 1),
+                std_macro("hinweis", False, 1),
+                std_macro("hinweise", False, 1),
+            ],
+            "environments": [],
+            "specials": [],
+        },
+    ),
+]

great_ai/utilities/external/pylatexenc/macrospec/__init__.py

@@ -0,0 +1,785 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2019 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+r"""
+Provides classes and helper functions to describe a LaTeX context of known
+macros and environments, specifying how they should be parsed by
+:py:mod:`pylatexenc.latexwalker`.
+
+.. versionadded:: 2.0
+
+   The entire module :py:mod:`pylatexenc.macrospec` was introduced in
+   `pylatexenc 2.0`.
+"""
+
+
+import sys
+
+if sys.version_info.major > 2:
+    # Py3
+    def unicode(s):
+        return s
+
+    _basestring = str
+    _str_from_unicode = lambda x: x
+    _unicode_from_str = lambda x: x
+else:
+    # Py2
+    _basestring = basestring
+    _str_from_unicode = lambda x: unicode(x).encode("utf-8")
+    _unicode_from_str = lambda x: x.decode("utf-8")
+
+
+# ------------------------------------------------------------------------------
+
+from ._argparsers import (
+    MacroStandardArgsParser,
+    ParsedMacroArgs,
+    ParsedVerbatimArgs,
+    VerbatimArgsParser,
+)
+
+# ------------------------------------------------------------------------------
+
+
+class MacroSpec(object):
+    r"""
+    Stores the specification of a macro.
+
+    This stores the macro name and instructions on how to parse the macro
+    arguments.
+
+    .. py:attribute:: macroname
+
+       The name of the macro, without the leading backslash.
+
+    .. py:attribute:: args_parser
+
+       The parser instance that can understand this macro's arguments.  For
+       standard LaTeX macros this is usually a
+       :py:class:`MacroStandardArgsParser` instance.
+
+       If you specify a string, then for convenience this is interpreted as an
+       argspec argument for :py:class:`MacroStandardArgsParser` and such an
+       instance is automatically created.
+    """
+
+    def __init__(self, macroname, args_parser=MacroStandardArgsParser(), **kwargs):
+        super(MacroSpec, self).__init__(**kwargs)
+        self.macroname = macroname
+        if isinstance(args_parser, _basestring):
+            self.args_parser = MacroStandardArgsParser(args_parser)
+        else:
+            self.args_parser = args_parser
+
+    def parse_args(self, *args, **kwargs):
+        r"""
+        Shorthand for calling the :py:attr:`args_parser`\ 's `parse_args()` method.
+        See :py:class:`MacroStandardArgsParser`.
+        """
+        return self.args_parser.parse_args(*args, **kwargs)
+
+    def __repr__(self):
+        return "MacroSpec(macroname=%r, args_parser=%r)" % (
+            self.macroname,
+            self.args_parser,
+        )
+
+
+class EnvironmentSpec(object):
+    r"""
+    Stores the specification of a LaTeX environment.
+
+    This stores the environment name and instructions on how to parse any
+    arguments provided after ``\begin{environment}<args>``.
+
+    .. py:attribute:: environmentname
+
+       The name of the environment, i.e., the argument of ``\begin{...}`` and
+       ``\end{...}``.
+
+    .. py:attribute:: args_parser
+
+       The parser instance that can understand this environment's arguments.
+       For standard LaTeX environment this is usually a
+       :py:class:`MacroStandardArgsParser` instance.
+
+       If you specify a string, then for convenience this is interpreted as an
+       argspec argument for :py:class:`MacroStandardArgsParser` and such an
+       instance is automatically created.
+
+    .. py:attribute:: is_math_mode
+
+       A boolean that indicates whether or not the contents is to be interpreted
+       in Math Mode.  This would be True for environments like
+       ``\begin{equation}``, ``\begin{align}``, etc., but False for
+       ``\begin{figure}``, etc.
+
+    .. note::
+
+       Starred variants of environments (as in ``\begin{equation*}``) must not
+       be specified using an argspec as for macros (e.g., `argspec='*'`).
+       Rather, we need to define a separate environment spec for the starred
+       variant with the star in the name itself (``EnvironmentSpec('equation*',
+       None)``) because the star really is part of the environment name.  If you
+       happened to use ``EnvironmentSpec('equation', '*')``, then the parser
+       would recognize the expression ``\begin{equation}*`` but not
+       ``\begin{equation*}``.
+    """
+
+    def __init__(
+        self,
+        environmentname,
+        args_parser=MacroStandardArgsParser(),
+        is_math_mode=False,
+        **kwargs
+    ):
+        super(EnvironmentSpec, self).__init__(**kwargs)
+        self.environmentname = environmentname
+        if isinstance(args_parser, _basestring):
+            self.args_parser = MacroStandardArgsParser(args_parser)
+        else:
+            self.args_parser = args_parser
+        self.is_math_mode = is_math_mode
+
+    def parse_args(self, *args, **kwargs):
+        r"""
+        Shorthand for calling the :py:attr:`args_parser`\ 's `parse_args()` method.
+        See :py:class:`MacroStandardArgsParser`.
+        """
+        return self.args_parser.parse_args(*args, **kwargs)
+
+    def __repr__(self):
+        return (
+            "EnvironmentSpec(environmentname=%r, args_parser=%r, is_math_mode=%r)"
+            % (self.environmentname, self.args_parser, self.is_math_mode)
+        )
+
+
+class SpecialsSpec(object):
+    r"""
+    Specification of a LaTeX "special char sequence": an active char, a
+    ligature, or some other non-macro char sequence that has a special meaning.
+
+    For instance, '&', '~', and '``' are considered as "specials".
+
+    .. py:attribute:: specials_chars
+
+       The string (one or several characters) that has a special meaning. E.g.,
+       '&', '~', '``', etc.
+
+    .. py:attribute:: args_parser
+
+       A parser (e.g. :py:class:`MacroStandardArgsParser`) that is invoked when
+       the specials is encountered.  Can/should be set to `None` if the specials
+       should not parse any arguments (e.g. '~').
+    """
+
+    def __init__(self, specials_chars, args_parser=None, **kwargs):
+        super(SpecialsSpec, self).__init__(**kwargs)
+        self.specials_chars = specials_chars
+        self.args_parser = args_parser
+
+    def parse_args(self, *args, **kwargs):
+        r"""
+        Basically a shorthand for calling the :py:attr:`args_parser`\ 's
+        `parse_args()` method.  See :py:class:`MacroStandardArgsParser`.
+
+        If however the py:attr:`args_parser` attribute is `None`, then this
+        method returns `None`.
+        """
+        if self.args_parser is None:
+            return None
+        return self.args_parser.parse_args(*args, **kwargs)
+
+    def __repr__(self):
+        return "SpecialsSpec(specials_chars=%r, args_parser=%r)" % (
+            self.specials_chars,
+            self.args_parser,
+        )
+
+
+# ------------------------------------------------------------------------------
+
+
+def std_macro(macname, *args, **kwargs):
+    r"""
+    Return a macro specification for the given macro.  Syntax::
+
+      spec = std_macro(macname, argspec)
+      #  or
+      spec = std_macro(macname, optarg, numargs)
+      #  or
+      spec = std_macro( (macname, argspec), )
+      #  or
+      spec = std_macro( (macname, optarg, numargs), )
+      #  or
+      spec = std_macro( spec ) # spec is already a `MacroSpec` -- no-op
+
+    - `macname` is the name of the macro, without the leading backslash.
+
+    - `argspec` is a string either characters "\*", "{" or "[", in which star
+      indicates an optional asterisk character (e.g. starred macro variants),
+      each curly brace specifies a mandatory argument and each square bracket
+      specifies an optional argument in square brackets.  For example, "{{\*[{"
+      expects two mandatory arguments, then an optional star, an optional
+      argument in square brackets, and then another mandatory argument.
+
+      `argspec` may also be `None`, which is the same as ``argspec=''``.
+
+    - `optarg` may be one of `True`, `False`, or `None`, corresponding to these
+      possibilities:
+
+      + if `True`, the macro expects as first argument an optional argument in
+        square brackets. Then, `numargs` specifies the number of additional
+        mandatory arguments to the command, given in usual curly braces (or
+        simply as one TeX token like a single macro)
+
+      + if `False`, the macro only expects a number of mandatory arguments given
+        by `numargs`. The mandatory arguments are given in usual curly braces
+        (or simply as one TeX token like a single macro)
+
+      + if `None`, then `numargs` is a string like `argspec` above.  I.e.,
+        ``std_macro(macname, None, argspec)`` is the same as
+        ``std_macro(macname, argspec)``.
+
+    - `numargs`: depends on `optarg`, see above.
+
+    To make environment specifications (:py:class:`EnvironmentSpec`) instead of
+    a macro specification, use the function :py:func:`std_environment()`
+    instead.
+
+    The helper function :py:func:`std_environment()` is a shorthand for calling
+    this function with additional keyword arguments.  An optional keyword
+    argument `make_environment_spec=True` to the present function may be
+    specified to return an `EnvironmentSpec` instead of a `MacroSpec`.  In this
+    case, you can further specify the `environment_is_math_mode=True|False` to
+    specify whether of not the environment represents a math mode.
+    """
+
+    if isinstance(macname, tuple):
+        if len(args) != 0:
+            raise TypeError(
+                "No positional arguments expected if first argument is a tuple"
+            )
+        args = tuple(macname[1:])
+        macname = macname[0]
+
+    if isinstance(macname, MacroSpec):
+        if len(args) != 0:
+            raise TypeError(
+                "No positional arguments expected if first argument is a MacroSpec"
+            )
+        return macname
+
+    if isinstance(macname, EnvironmentSpec):
+        if len(args) != 0:
+            raise TypeError(
+                "No positional arguments expected if first argument is a EnvironmentSpec"
+            )
+        return macname
+
+    if len(args) == 1:
+        # std_macro(macname, argspec)
+        argspec = args[0]
+    elif len(args) != 2:
+        raise TypeError(
+            "Wrong number of arguments for std_macro, macname={!r}, args={!r}".format(
+                macname, args
+            )
+        )
+    elif not args[0] and isinstance(args[1], _basestring):
+        # argspec given in numargs
+        argspec = args[1]
+    else:
+        argspec = ""
+        if args[0]:
+            argspec = "["
+        argspec += "{" * args[1]
+
+    if kwargs.get("make_environment_spec", False):
+        return EnvironmentSpec(
+            macname,
+            args_parser=MacroStandardArgsParser(argspec),
+            is_math_mode=kwargs.get("environment_is_math_mode", False),
+        )
+    return MacroSpec(macname, args_parser=MacroStandardArgsParser(argspec))
+
+
+def std_environment(envname, *args, **kwargs):
+    r"""
+    Return an environment specification for the given environment.  Syntax::
+
+      spec = std_environment(envname, argspec, is_math_mode=True|False)
+      #  or
+      spec = std_environment(envname, optarg, numargs, is_math_mode=True|False)
+      #  or
+      spec = std_environment( (envname, argspec), is_math_mode=True|False)
+      #  or
+      spec = std_environment( (envname, optarg, numargs), is_math_mode=True|False)
+      #  or
+      spec = std_environment( spec ) # spec is already a `EnvironmentSpec` -- no-op
+
+    - `envname` is the name of the environment, i.e., the argument to
+      ``\begin{...}``.
+
+    - `argspec` is a string either characters "\*", "{" or "[", in which star
+      indicates an optional asterisk character (e.g. starred environment
+      variants), each curly brace specifies a mandatory argument and each square
+      bracket specifies an optional argument in square brackets.  For example,
+      "{{\*[{" expects two mandatory arguments, then an optional star, an
+      optional argument in square brackets, and then another mandatory argument.
+
+      `argspec` may also be `None`, which is the same as ``argspec=''``.
+
+    .. note::
+
+       See :py:class:`EnvironmentSpec` for an important remark about starred
+       variants for environments.  TL;DR: a starred verison of an environment is
+       defined as a separate `EnvironmentSpec` with the star in the name and
+       *not* using an ``argspec='*'``.
+
+    - `optarg` may be one of `True`, `False`, or `None`, corresponding to these
+      possibilities:
+
+      + if `True`, the environment expects as first argument an optional argument in
+        square brackets. Then, `numargs` specifies the number of additional
+        mandatory arguments to the command, given in usual curly braces (or
+        simply as one TeX token like a single environment)
+
+      + if `False`, the environment only expects a number of mandatory arguments given
+        by `numargs`. The mandatory arguments are given in usual curly braces
+        (or simply as one TeX token like a single environment)
+
+      + if `None`, then `numargs` is a string like `argspec` above.  I.e.,
+        ``std_environment(envname, None, argspec)`` is the same as
+        ``std_environment(envname, argspec)``.
+
+    - `numargs`: depends on `optarg`, see above.
+
+    - `is_math_mode`: if set to True, then the environment represents a math
+      mode environment (e.g., 'equation', 'align', 'gather', etc.), i.e., whose
+      contents should be parsed in an appropriate math mode.  Note that
+      `is_math_mode` *must* be given as a keyword argument, in contrast to all
+      other arguments which must be positional (non-keyword) arguments.
+    """
+    is_math_mode = kwargs.pop("is_math_mode", False)
+    kwargs2 = dict(kwargs)
+    kwargs2.update(make_environment_spec=True, environment_is_math_mode=is_math_mode)
+    return std_macro(envname, *args, **kwargs2)
+
+
+def std_specials(specials_chars):
+    r"""
+    Return a latex specials specification for the given character sequence.  Syntax::
+
+      spec = std_specials(specials_chars)
+
+    where `specials_chars` is the sequence of characters that has a special
+    LaTeX meaning, e.g. ``&`` or ``''``.
+
+    This helper function only allows to create specs for simple specials without
+    any argument parsing.  For more complicated specials, you can instantiate a
+    :py:class:`SpecialsSpec` directly.
+    """
+    return SpecialsSpec(specials_chars, args_parser=None)
+
+
+# ------------------------------------------------------------------------------
+
+
+class LatexContextDb(object):
+    r"""
+    Store a database of specifications of known macros, environments, and other
+    latex specials.  This might be, e.g., how many arguments a macro accepts, or
+    how to determine the text representation of a macro or environment.
+
+    When used with :py:class:`pylatexenc.latexwalker.LatexWalker`, the
+    specifications describe mostly rules for parsing arguments of macros and
+    environments, and which sequences of characters to consider as "latex
+    specials".  Specifications for macros, environments, and other specials are
+    stored as :py:class:`MacroSpec`, :py:class:`EnvironmentSpec`, and
+    :py:class:`SpecialsSpec` instances, respectively.
+    When used with :py:class:`pylatexenc.latex2text.LatexNodes2Text`, the
+    specifications for macros, environments, and other specials are stored as
+    :py:class:`pylatexenc.latex2text.MacroTextSpec` ,
+    :py:class:`pylatexenc.latex2text.EnvironmentTextSpec`, and
+    :py:class:`pylatexenc.latex2text.SpecialsTextSpec` instances, respectively.
+
+    In fact, the objects stored in this database may be of any type, except that
+    macro specifications must have an attribute `macroname`, environment
+    specifications must have an attribute `environmentname`, and specials
+    specification must have an attribute `specials_chars`.
+
+    The `LatexContextDb` instance is meant to be (pseudo-)immutable.  Once
+    constructed and all the definitions added with
+    :py:meth:`add_context_category()`, one should refrain from modifying it
+    directly after providing it to, e.g., a
+    :py:class:`~pylatexenc.latexwalker.LatexWalker` object.  The reason is that
+    the latex walker keeps track of what the latex context was when parsing
+    nodes, and modifying the context will modify that stored information, too.
+    Instead of being tempted to modify the object, create a new one with
+    :py:meth:`filter_context()`.
+
+    See :py:func:`pylatexenc.latexwalker.get_default_latex_context_db()` for the
+    default latex context for `latexwalker` with a default collection of known
+    latex macros and environments.
+    See :py:func:`pylatexenc.latex2text.get_default_latex_context_db()` for the
+    default latex context for `latex2text` with a set of text replacements for a
+    collection of known macros and environments.
+    """
+
+    def __init__(self, **kwargs):
+        super(LatexContextDb, self).__init__(**kwargs)
+
+        self.category_list = []
+        self.d = {}
+
+        self.unknown_macro_spec = None
+        self.unknown_environment_spec = None
+        self.unknown_specials_spec = None
+
+    def add_context_category(
+        self,
+        category,
+        macros=[],
+        environments=[],
+        specials=[],
+        prepend=False,
+        insert_before=None,
+        insert_after=None,
+    ):
+        r"""
+        Register a category of macro and environment specifications in the context
+        database.
+
+        The category name `category` must not already exist in the database.
+
+        The argument `macros` is an iterable (e.g., a list) of macro
+        specification objects.  The argument `environments` is an iterable
+        (e.g., a list) of environment spec objects.  Similarly, the `specials`
+        argument is an iterable of latex specials spec instances.
+
+        If you specify `prepend=True`, then macro and environment lookups will
+        prioritize this category over other categories.  Categories are normally
+        searched for in the order they are registered to the database; if you
+        specify `prepend=True`, then the new category is prepended to the
+        existing list so that it is searched first.
+
+        If `insert_before` is not `None`, then it must be a string; the
+        definitions are inserted in the category list immediately before the
+        given category name, or at the beginning of the list if the given
+        category doesn't exist.  If `insert_after` is not `None`, then it must
+        be a string; the definitions are inserted in the category list
+        immediately after the given category name, or at the end of the list if
+        the given category doesn't exist.
+
+        You may only specify one of `prepend=True`, `insert_before='...'` or
+        `insert_after='...'`.
+        """
+
+        if category in self.category_list:
+            raise ValueError(
+                "Category {} is already registered in the context database".format(
+                    category
+                )
+            )
+
+        # ensure only one of these options is set
+        if len([x for x in (prepend, insert_before, insert_after) if x]) > 1:
+            raise TypeError(
+                "add_context_category(): You may only specify one of "
+                "prepend=True, insert_before=... or insert_after=..."
+            )
+
+        if prepend:
+            self.category_list.insert(0, category)
+        elif insert_before:
+            if insert_before in self.category_list:
+                i = self.category_list.index(insert_before)
+            else:
+                i = 0
+            self.category_list.insert(i, category)
+        elif insert_after:
+            if insert_after in self.category_list:
+                i = (
+                    self.category_list.index(insert_after) + 1
+                )  # insert after found category
+            else:
+                i = len(self.category_list)
+            self.category_list.insert(i, category)
+        else:
+            self.category_list.append(category)
+
+        self.d[category] = {
+            "macros": dict((m.macroname, m) for m in macros),
+            "environments": dict((e.environmentname, e) for e in environments),
+            "specials": dict((s.specials_chars, s) for s in specials),
+        }
+
+    def set_unknown_macro_spec(self, macrospec):
+        r"""
+        Set the macro spec to use when encountering a macro that is not in the
+        database.
+        """
+        self.unknown_macro_spec = macrospec
+
+    def set_unknown_environment_spec(self, environmentspec):
+        r"""
+        Set the environment spec to use when encountering a LaTeX environment that
+        is not in the database.
+        """
+        self.unknown_environment_spec = environmentspec
+
+    def set_unknown_specials_spec(self, specialsspec):
+        r"""
+        Set the latex specials spec to use when encountering a LaTeX environment
+        that is not in the database.
+        """
+        self.unknown_specials_spec = specialsspec
+
+    def categories(self):
+        r"""
+        Return a list of valid category names that are registered in the current
+        database context.
+        """
+        return list(self.category_list)
+
+    def get_macro_spec(self, macroname):
+        r"""
+        Look up a macro specification by macro name.  The macro name is searched for
+        in all categories one by one and the first match is returned.
+
+        Returns a macro spec instance that matches the given `macroname`.  If
+        the macro name was not found, we return the default macro specification
+        set by :py:meth:`set_unknown_macro_spec()` or `None` if no such spec was
+        set.
+        """
+        for cat in self.category_list:
+            # search categories in the given order
+            if macroname in self.d[cat]["macros"]:
+                return self.d[cat]["macros"][macroname]
+        return self.unknown_macro_spec
+
+    def get_environment_spec(self, environmentname):
+        r"""
+        Look up an environment specification by environment name.  The environment
+        name is searched for in all categories one by one and the first match is
+        returned.
+
+        Returns the environment spec.  If the environment name was not found, we
+        return the default environment specification set by
+        :py:meth:`set_unknown_environment_spec()` or `None` if no such spec was
+        set.
+        """
+        for cat in self.category_list:
+            # search categories in the given order
+            if environmentname in self.d[cat]["environments"]:
+                return self.d[cat]["environments"][environmentname]
+        return self.unknown_environment_spec
+
+    def get_specials_spec(self, specials_chars):
+        r"""
+        Look up a "latex specials" specification by character sequence.  The
+        sequence name is searched for in all categories one by one and the first
+        match is returned.
+
+        If you are parsing a chunk of LaTeX code, you should use
+        :py:meth:`test_for_specials()` instead.  Unlike
+        :py:meth:`test_for_specials()`, :py:meth:`get_specials_spec()` returns
+        the first match regardless of matched length.  [Rationale: we only need
+        to worry about matching the longest specials sequence when parsing LaTeX
+        code.  Calling `get_specials_spec()` means one has already parsed the
+        sequence and one is looking up additional specs on it.]
+
+        Returns the specials spec.  If the latex specials was not found, we
+        return the default latex specials specification set by
+        :py:meth:`set_unknown_specials_spec()` or `None` if no such spec was
+        set.
+        """
+        for cat in self.category_list:
+            # search categories in the given order
+            if specials_chars in self.d[cat]["specials"]:
+                return self.d[cat]["specials"][specials_chars]
+        return self.unknown_specials_spec
+
+    def test_for_specials(self, s, pos, parsing_state=None):
+        r"""
+        Test the given position in the string for any LaTeX specials.  The lookup
+        proceeds by searching for in all categories one by one and the first
+        match is returned, except that the longest match accross all categories
+        is returned.  For instance, a match of '``' in a later category will
+        take precedence over a match of '`' in a earlier-searched category.
+
+        Returns a specials spec instance, or `None` if no specials are detected
+        at the position `pos`.
+        """
+        best_match_len = 0
+        best_match_s = None
+        for cat in self.category_list:
+            # search categories in the given order
+            for specials_chars in self.d[cat]["specials"].keys():
+                if len(specials_chars) > best_match_len and s.startswith(
+                    specials_chars, pos
+                ):
+                    best_match_s = self.d[cat]["specials"][specials_chars]
+                    best_match_len = len(specials_chars)
+
+        return best_match_s  # this is None if no match
+
+    def iter_macro_specs(self, categories=None):
+        r"""
+        Yield the macro specs corresponding to all macros in the given categories.
+
+        If `categories` is `None`, then the known macro specs from all
+        categories are provided in one long iterable sequence.  Otherwise,
+        `categories` should be a list or iterable of category names (e.g.,
+        'latex-base') of macro specs to return.
+
+        The macro specs from the different categories specified are concatenated
+        into one long sequence which is yielded spec by spec.
+        """
+
+        if categories is None:
+            categories = self.category_list
+
+        for c in categories:
+            if c not in self.category_list:
+                raise ValueError(
+                    "Invalid latex macro spec db category: {!r} (Expected one of {!r})".format(
+                        c, self.category_list
+                    )
+                )
+            for spec in self.d[c]["macros"].values():
+                yield spec
+
+    def iter_environment_specs(self, categories=None):
+        r"""
+        Yield the environment specs corresponding to all environments in the given
+        categories.
+
+        If `categories` is `None`, then the known environment specs from all
+        categories are provided in one long iterable sequence.  Otherwise,
+        `categories` should be a list or iterable of category names (e.g.,
+        'latex-base') of environment specs to return.
+
+        The environment specs from the different categories specified are
+        concatenated into one long sequence which is yielded spec by spec.
+        """
+
+        if categories is None:
+            categories = self.category_list
+
+        for c in categories:
+            if c not in self.category_list:
+                raise ValueError(
+                    "Invalid latex environment spec db category: {!r} (Expected one of {!r})".format(
+                        c, self.category_list
+                    )
+                )
+            for spec in self.d[c]["environments"].values():
+                yield spec
+
+    def iter_specials_specs(self, categories=None):
+        r"""
+        Yield the specials specs corresponding to all environments in the given
+        categories.
+
+        If `categories` is `None`, then the known specials specs from all
+        categories are provided in one long iterable sequence.  Otherwise,
+        `categories` should be a list or iterable of category names (e.g.,
+        'latex-base') of specials specs to return.
+
+        The specials specs from the different categories specified are
+        concatenated into one long sequence which is yielded spec by spec.
+        """
+
+        if categories is None:
+            categories = self.category_list
+
+        for c in categories:
+            if c not in self.category_list:
+                raise ValueError(
+                    "Invalid latex environment spec db category: {!r} (Expected one of {!r})".format(
+                        c, self.category_list
+                    )
+                )
+            for spec in self.d[c]["specials"].values():
+                yield spec
+
+    def filter_context(self, keep_categories=[], exclude_categories=[], keep_which=[]):
+        r"""
+        Return a new :py:class:`LatexContextDb` instance where we only keep
+        certain categories of macro and environment specifications.
+
+        If `keep_categories` is set to a nonempty list, then the returned
+        context will not contain any definitions that do not correspond to the
+        specified categories.
+
+        If `exclude_categories` is set to a nonempty list, then the returned
+        context will not contain any definitions that correspond to the
+        specified categories.
+
+        It is explicitly fine to have category names in `keep_categories` and
+        `exclude_categories` that don't exist in the present object
+        (cf. :py:meth:`categories()`).
+
+        The argument `keep_which`, if non-empty, specifies which definitions to
+        keep.  It should be a subset of the list ['macros', 'environments',
+        'specials'].
+
+        The returned context will make a copy of the dictionaries that store the
+        macro and environment specifications, but the specification classes (and
+        corresponding argument parsers) might correspond to the same instances.
+        I.e., the returned context is not a full deep copy.
+        """
+
+        new_context = LatexContextDb()
+
+        new_context.unknown_macro_spec = self.unknown_macro_spec
+        new_context.unknown_environment_spec = self.unknown_environment_spec
+        new_context.unknown_specials_spec = self.unknown_specials_spec
+
+        keep_macros = not keep_which or "macros" in keep_which
+        keep_environments = not keep_which or "environments" in keep_which
+        keep_specials = not keep_which or "specials" in keep_which
+
+        for cat in self.category_list:
+            if keep_categories and cat not in keep_categories:
+                continue
+            if exclude_categories and cat in exclude_categories:
+                continue
+
+            # include this category
+            new_context.add_context_category(
+                cat,
+                macros=self.d[cat]["macros"].values() if keep_macros else [],
+                environments=self.d[cat]["environments"].values()
+                if keep_environments
+                else [],
+                specials=self.d[cat]["specials"].values() if keep_specials else [],
+            )
+
+        return new_context

great_ai/utilities/external/pylatexenc/macrospec/_argparsers.py

@@ -0,0 +1,497 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2019 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+
+# Internal module. Internal API may move, disappear or otherwise change at any
+# time and without notice.
+
+
+import sys
+
+if sys.version_info.major > 2:
+    # Py3
+    def unicode(s):
+        return s
+
+    _basestring = str
+    _str_from_unicode = lambda x: x
+    _unicode_from_str = lambda x: x
+else:
+    # Py2
+    _basestring = basestring
+    _str_from_unicode = lambda x: unicode(x).encode("utf-8")
+    _unicode_from_str = lambda x: x.decode("utf-8")
+
+
+class ParsedMacroArgs(object):
+    r"""
+    Parsed representation of macro arguments.
+
+    The base class provides a simple way of storing the arguments as a list of
+    parsed nodes.
+
+    This base class can be subclassed to store additional information and
+    provide more advanced APIs to access macro arguments for certain categories
+    of macros.
+
+    Arguments:
+
+      - `argnlist` is a list of latexwalker nodes that represent macro
+        arguments.  If the macro arguments are too complicated to store in a
+        list, leave this as `None`.  (But then code that uses the latexwalker
+        must be aware of your own API to access the macro arguments.)
+
+        The difference between `argnlist` and the legacy `nodeargs` is that all
+        options, regardless of optional or mandatory, are stored in the list
+        `argnlist` with possible `None`\ 's at places where optional arguments
+        were not provided.  Previously, whether a first optional argument was
+        included in `nodeoptarg` or `nodeargs` depended on how the macro
+        specification was given.
+
+      - `argspec` is a string or a list that describes how each corresponding
+        argument in `argnlist` represents.  If the macro arguments are too
+        complicated to store in a list, leave this as `None`.  For standard
+        macros and parsed arguments this is a string with characters '*', '[',
+        '{' describing an optional star argument, an optional
+        square-bracket-delimited argument, and a mandatory argument.
+
+    Attributes:
+
+    .. py:attribute:: argnlist
+
+       The list of latexwalker nodes that was provided to the constructor
+
+    .. py:attribute:: argspec
+
+       Argument type specification provided to the constructor
+
+    .. py:attribute:: legacy_nodeoptarg_nodeargs
+
+       A tuple `(nodeoptarg, nodeargs)` that should be exposed as properties in
+       :py:class:`~pylatexenc.latexwalker.LatexMacroNode` to provide (as best as
+       possible) compatibility with pylatexenc < 2.
+
+       This is either `(<1st optional arg node>, <list of remaining args>)` if
+       the first argument is optional and all remaining args are mandatory; or
+       it is `(None, <list of args>)` for any other argument structure.
+    """
+
+    def __init__(self, argnlist=[], argspec="", **kwargs):
+        super(ParsedMacroArgs, self).__init__(**kwargs)
+
+        self.argnlist = argnlist
+        self.argspec = argspec
+
+        # for LatexMacroNode to provide some kind of compatibility with pylatexenc < 2
+        self.legacy_nodeoptarg_nodeargs = self._get_legacy_attribs(
+            self.argspec, self.argnlist
+        )
+
+    def _get_legacy_attribs(self, argspec, argnlist):
+        nskip = 0
+        while argspec.startswith("*"):
+            argspec = argspec[1:]
+            nskip += 1
+        if argspec[0:1] == "[" and all(x == "{" for x in argspec[1:]):
+            return (argnlist[nskip], argnlist[nskip + 1 :])
+        else:
+            return (None, argnlist)
+
+    def to_json_object(self):
+        r"""
+        Called when we export the node structure to JSON when running latexwalker in
+        command-line.
+
+        Return a representation of the current parsed arguments in an object,
+        typically a dictionary, that can easily be exported to JSON.  The object
+        may contain latex nodes and other parsed-argument objects, as we use a
+        custom JSON encoder that understands these types.
+
+        Subclasses may
+        """
+
+        return dict(
+            argspec=self.argspec,
+            argnlist=self.argnlist,
+        )
+
+    def __repr__(self):
+        return "{}(argspec={!r}, argnlist={!r})".format(
+            self.__class__.__name__, self.argspec, self.argnlist
+        )
+
+
+class MacroStandardArgsParser(object):
+    r"""
+    Parses the arguments to a LaTeX macro.
+
+    This class parses a simple macro argument specification with a specified
+    arrangement of optional and mandatory arguments.
+
+    This class also serves as base class for more advanced argument parsers
+    (e.g. for a ``\verb+...+`` macro argument parser).  In such cases,
+    subclasses should attempt to provide the most suitable `argspec` (and
+    `argnlist` for the corresponding :py:class:`ParsedMacroArgs`) for their use,
+    if appropriate, or set them to `None`.
+
+    Arguments:
+
+      - `argspec`: must be a string in which each character corresponds to an
+        argument.  The character '{' represents a mandatory argument (single
+        token or LaTeX group) and the character '[' denotes an optional argument
+        delimited by braces.  The character '\*' denotes a possible star char at
+        that position in the argument list, a corresponding
+        ``latexwalker.LatexCharsNode('*')`` (or `None` if no star) will be
+        inserted in the argument node list.  For instance, the string '\*{[[{'
+        would be suitable to specify the signature of the '\\newcommand' macro.
+
+        Currently, the argspec string may only contain the characters '\*', '{'
+        and '['.
+
+        The `argspec` may also be `None`, which is the same as specifying an
+        empty string.
+
+      - `optional_arg_no_space`: If set to `True`, then an optional argument
+        cannot have any whitespace between the preceeding tokens and the '['
+        character.  Set this to `True` in cases such as for ``\\`` in AMS-math
+        environments, where AMS apparently introduced a patch to prevent a
+        bracket on a new line after ``\\`` from being interpreted as the
+        optional argument to ``\\``.
+
+      - `args_math_mode`: Either `None`, or a list of the same length as
+        `argspec`.  If a list is given, then each item must be `True`, `False`,
+        or `None`.  The corresponding argument (cf. `argspec`) is then
+        respectively parsed in math mode (`True`), in text mode (`False`), or
+        with the mode unchanged (`None`).  If `args_math_mode` is `None`, then
+        all arguments are parsed in the same mode as the current mode.
+
+      - additional unrecognized keyword arguments are passed on to superclasses
+        in case of multiple inheritance
+
+    Attributes:
+
+    .. py:attribute:: argspec
+
+       Argument type specification provided to the constructor.
+
+    .. py:attribute:: optional_arg_no_space
+
+       See the corresponding constructor argument.
+
+    .. py:attribute:: args_math_mode
+
+       See the corresponding constructor argument.
+    """
+
+    def __init__(
+        self, argspec=None, optional_arg_no_space=False, args_math_mode=None, **kwargs
+    ):
+        super(MacroStandardArgsParser, self).__init__(**kwargs)
+        self.argspec = argspec if argspec else ""
+        self.optional_arg_no_space = optional_arg_no_space
+        self.args_math_mode = args_math_mode
+        # catch bugs, make sure that argspec is a string with only accepted chars
+        if not isinstance(self.argspec, _basestring) or not all(
+            x in "*[{" for x in self.argspec
+        ):
+            raise TypeError(
+                "argspec must be a string containing chars '*', '[', '{{' only: {!r}".format(
+                    self.argspec
+                )
+            )
+        # non-documented attribute that makes us ignore any leading '*'.  We use
+        # this to emulate pylatexenc 1.x behavior when using the MacrosDef()
+        # function explicitly
+        self._like_pylatexenc1x_ignore_leading_star = False
+
+    def parse_args(self, w, pos, parsing_state=None):
+        r"""
+        Parse the arguments encountered at position `pos` in the
+        :py:class:`~pylatexenc.latexwalker.LatexWalker` instance `w`.
+
+        You may override this function to provide custom parsing of complicated
+        macro arguments (say, ``\verb+...+``).  The method will be called by
+        keyword arguments, so the argument names should not be altered.
+
+        The argument `w` is the :py:class:`pylatexenc.latexwalker.LatexWalker`
+        object that is currently parsing LaTeX code.  You can call methods like
+        `w.get_goken()`, `w.get_latex_expression()` etc., to parse and read
+        arguments.
+
+        The argument `parsing_state` is the current parsing state in the
+        :py:class:`~pylatexenc.latexwalker.LatexWalker` (e.g., are we currently
+        in math mode?).  See doc for
+        :py:class:`~pylatexenc.latexwalker.ParsingState`.
+
+        This function should return a tuple `(argd, pos, len)` where:
+
+        - `argd` is a :py:class:`ParsedMacroArgs` instance, or an instance of a
+          subclass of :py:class:`ParsedMacroArgs`.  The base `parse_args()`
+          provided here returns a :py:class:`ParsedMacroArgs` instance.
+
+        - `pos` is the position of the first parsed content.  It should be the
+          same as the `pos` argument, except if there is whitespace at that
+          position in which case the returned `pos` would have to be the
+          position where the argument contents start.
+
+        - `len` is the length of the parsed expression.  You will probably want
+          to continue parsing stuff at the index `pos+len` in the string.
+        """
+
+        from .. import latexwalker
+
+        if parsing_state is None:
+            parsing_state = w.make_parsing_state()
+
+        argnlist = []
+
+        if self.args_math_mode is not None and len(self.args_math_mode) != len(
+            self.argspec
+        ):
+            raise ValueError(
+                "Invalid args_math_mode={!r} for argspec={!r}!".format(
+                    self.args_math_mode, self.argspec
+                )
+            )
+
+        def get_inner_parsing_state(j):
+            if self.args_math_mode is None:
+                return parsing_state
+            amm = self.args_math_mode[j]
+            if amm is None or amm == parsing_state.in_math_mode:
+                return parsing_state
+            if amm == True:
+                return parsing_state.sub_context(in_math_mode=True)
+            return parsing_state.sub_context(in_math_mode=False)
+
+        p = pos
+
+        if self._like_pylatexenc1x_ignore_leading_star:
+            # ignore any leading '*' character
+            tok = w.get_token(p)
+            if tok.tok == "char" and tok.arg == "*":
+                p = tok.pos + tok.len
+
+        for j, argt in enumerate(self.argspec):
+            if argt == "{":
+                (node, np, nl) = w.get_latex_expression(
+                    p, strict_braces=False, parsing_state=get_inner_parsing_state(j)
+                )
+                p = np + nl
+                argnlist.append(node)
+
+            elif argt == "[":
+
+                if self.optional_arg_no_space and p < len(w.s) and w.s[p].isspace():
+                    # don't try to read optional arg, we don't allow space
+                    argnlist.append(None)
+                    continue
+
+                optarginfotuple = w.get_latex_maybe_optional_arg(
+                    p, parsing_state=get_inner_parsing_state(j)
+                )
+                if optarginfotuple is None:
+                    argnlist.append(None)
+                    continue
+                (node, np, nl) = optarginfotuple
+                p = np + nl
+                argnlist.append(node)
+
+            elif argt == "*":
+                # possible star.
+                tok = w.get_token(p)
+                if tok.tok == "char" and tok.arg.startswith("*"):
+                    # has star
+                    argnlist.append(
+                        w.make_node(
+                            latexwalker.LatexCharsNode,
+                            parsing_state=get_inner_parsing_state(j),
+                            chars="*",
+                            pos=tok.pos,
+                            len=1,
+                        )
+                    )
+                    p = tok.pos + 1
+                else:
+                    argnlist.append(None)
+
+            else:
+                raise LatexWalkerError(
+                    "Unknown macro argument kind for macro: {!r}".format(argt)
+                )
+
+        parsed = ParsedMacroArgs(
+            argspec=self.argspec,
+            argnlist=argnlist,
+        )
+
+        return (parsed, pos, p - pos)
+
+    def __repr__(self):
+        return (
+            "{}(argspec={!r}, optional_arg_no_space={!r}, args_math_mode={!r})".format(
+                self.__class__.__name__,
+                self.argspec,
+                self.optional_arg_no_space,
+                self.args_math_mode,
+            )
+        )
+
+
+class ParsedVerbatimArgs(ParsedMacroArgs):
+    r"""
+    Parsed representation of arguments to LaTeX verbatim constructs, such as
+    ``\begin{verbatim}...\end{verbatim}`` or ``\verb|...|``.
+
+    Instances of `ParsedVerbatimArgs` are returned by the args parser
+    :py:class:`VerbatimArgsParser`.
+
+    Arguments:
+
+      - `verbatim_chars_node` --- a properly initialized
+        :py:class:`pylatexenc.latexwalker.LatexCharsNode` that stores the
+        verbatim text provided.  It is used to initialize the base class
+        :py:class:`ParsedMacroArgs` to expose a single mandatory argument with
+        the given verbatim text.  The `verbatim_text` attribute is initialized
+        from this node, too.
+
+      - `verbatim_delimiters` --- a 2-item tuple of characters used to delimit
+        the verbatim arguemnt (in case of a ``\verb+...+`` macro) or `None`.
+
+    Attributes:
+
+    .. py:attribute:: verbatim_text
+
+       The verbatim text that was provided
+
+    .. py:attribute:: verbatim_delimiters
+
+       If the verbatim text was specified as an argument to ``\verb$...$``, then
+       this is set to a 2-item tuple that specifies the begin and end
+       delimiters.  Otherwise, the attribute is `None`.
+    """
+
+    def __init__(self, verbatim_chars_node, verbatim_delimiters=None, **kwargs):
+
+        # provide argspec/argnlist to the parent class so that any code that is
+        # not "verbatim environment-aware" sees this simply as the argument to
+        # an empty verbatim environment
+        super(ParsedVerbatimArgs, self).__init__(
+            argspec="{", argnlist=[verbatim_chars_node], **kwargs
+        )
+
+        self.verbatim_text = verbatim_chars_node.chars
+        self.verbatim_delimiters = verbatim_delimiters
+
+    def __repr__(self):
+        return "{}(verbatim_text={!r}, verbatim_delimiters={!r})".format(
+            self.__class__.__name__, self.verbatim_text, self.verbatim_delimiters
+        )
+
+
+class VerbatimArgsParser(MacroStandardArgsParser):
+    r"""
+    Parses the arguments to various LaTeX "verbatim" constructs such as
+    ``\begin{verbatim}...\end{verbatim}`` environment or ``\verb+...+``.
+
+    This class also serves to illustrate how to write custom parsers for
+    complicated macro arguments.  See also :py:class:`MacroStandardArgsParser`.
+
+    Arguments:
+
+    .. py:attribute:: verbatim_arg_type
+
+      One of 'verbatim-environment' or 'verb-macro'.
+    """
+
+    def __init__(self, verbatim_arg_type, **kwargs):
+        super(VerbatimArgsParser, self).__init__(argspec="{", **kwargs)
+        self.verbatim_arg_type = verbatim_arg_type
+
+    def parse_args(self, w, pos, parsing_state=None):
+
+        from .. import latexwalker
+
+        if self.verbatim_arg_type == "verbatim-environment":
+            # simply scan the string until we find '\end{verbatim}'.  That's
+            # exactly how LaTeX processes it.
+            endverbpos = w.s.find(r"\end{verbatim}", pos)
+            if endverbpos == -1:
+                raise latexwalker.LatexWalkerParseError(
+                    s=w.s, pos=pos, msg=r"Cannot find matching \end{verbatim}"
+                )
+            # do NOT include the "\end{verbatim}", latexwalker will expect to
+            # see it:
+            len_ = endverbpos - pos
+
+            argd = ParsedVerbatimArgs(
+                verbatim_chars_node=w.make_node(
+                    latexwalker.LatexCharsNode,
+                    parsing_state=parsing_state,
+                    chars=w.s[pos : pos + len_],
+                    pos=pos,
+                    len=len_,
+                )
+            )
+            return (argd, pos, len_)
+
+        if self.verbatim_arg_type == "verb-macro":
+            # read the next nonwhitespace char. This is the delimiter of the
+            # argument
+            while w.s[pos].isspace():
+                pos += 1
+                if pos >= len(w.s):
+                    raise latexwalker.LatexWalkerParseError(
+                        s=w.s, pos=pos, msg=r"Missing argument to \verb command"
+                    )
+            verbdelimchar = w.s[pos]
+            beginpos = pos + 1
+            endpos = w.s.find(verbdelimchar, beginpos)
+            if endpos == -1:
+                raise latexwalker.LatexWalkerParseError(
+                    s=w.s,
+                    pos=pos,
+                    msg=r"End of stream reached while reading argument to \verb command",
+                )
+
+            verbarg = w.s[beginpos:endpos]
+
+            argd = ParsedVerbatimArgs(
+                verbatim_chars_node=w.make_node(
+                    latexwalker.LatexCharsNode,
+                    parsing_state=parsing_state,
+                    chars=verbarg,
+                    pos=beginpos,
+                    len=endpos - beginpos,
+                ),
+                verbatim_delimiters=(verbdelimchar, verbdelimchar),
+            )
+
+            return (argd, pos, endpos + 1 - pos)  # include delimiters in pos/len
+
+    def __repr__(self):
+        return "{}(verbatim_arg_type={!r})".format(
+            self.__class__.__name__, self.verbatim_arg_type
+        )

great_ai/utilities/external/pylatexenc/version.py

@@ -0,0 +1,58 @@
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2021 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+
+#
+# Self-note: Checklist
+#
+#   1) First some checks:
+#
+#       - Set below in this file ' version_str = "X.Xb" ' (beta version for next
+#         release) for the following tests.
+#
+#       - tests pass: https://travis-ci.org/github/phfaist/pylatexenc
+#
+#       - LGTM looks good: https://lgtm.com/projects/g/phfaist/pylatexenc/
+#
+#       - python package creation works: (python setup.py sdist, pip install
+#         dist/pylatexenc-xxx.tar.gz)
+#
+#   2) update change log (doc/changes.rst)
+#
+#   3) bump version number here
+#
+#   4) git commit any remaining changes
+#
+#   5) " git tag vX.X -am '<message>' "
+#
+#   6) " git push && git push --tags "
+#
+#   7) on github.com, fill in release details with a summary of changes etc.
+#
+#   8) create the source package for PyPI (" python3 setup.py sdist ")
+#
+#   8) upload package to PyPI (twine upload dist/pylatexenc-X.X.tar.gz -r realpypi)
+#
+
+version_str = "2.10"

great_ai/utilities/get_sentences.py

@@ -0,0 +1,42 @@
+import re
+from string import punctuation
+from typing import List
+
+from syntok.segmenter import segment
+from syntok.tokenizer import Tokenizer
+
+from .data import sentence_ending_punctuations
+
+punctuations_pattern = re.compile(f"\\s*[{re.escape(punctuation)}]+\\s*")
+
+
+def get_sentences(
+    text: str,
+    ignore_partial: bool = False,
+    true_case: bool = False,
+    remove_punctuation: bool = False,
+) -> List[str]:
+    tokenizer = Tokenizer(
+        emit_hyphen_or_underscore_sep=True, replace_not_contraction=False
+    )
+    token_stream = tokenizer.tokenize(text)
+
+    def process(sentence: str) -> str:
+        if true_case:
+            sentence = sentence[0].lower() + sentence[1:]  # very crude method
+        if remove_punctuation:
+            sentence = re.sub(punctuations_pattern, " ", sentence)
+        return sentence.strip()
+
+    sentences = [
+        process(tokenizer.to_text(sentence)) for sentence in segment(token_stream)
+    ]
+
+    if ignore_partial:
+        sentences = [
+            sentence
+            for sentence in sentences
+            if sentence[0].isupper() and sentence[-1] in sentence_ending_punctuations
+        ]
+
+    return sentences

great_ai/utilities/language/__init__.py

@@ -0,0 +1,3 @@
+from .english_name_of_language import english_name_of_language
+from .is_english import is_english
+from .predict_language import predict_language

great_ai/utilities/language/english_name_of_language.py

@@ -0,0 +1,10 @@
+from typing import Optional
+
+from langcodes import Language
+
+
+def english_name_of_language(language_code: Optional[str]) -> str:
+    if not language_code:
+        language_code = "und"
+
+    return Language.get(language_code).display_name()

great_ai/utilities/language/is_english.py

@@ -0,0 +1,11 @@
+from typing import Optional
+
+from langcodes import standardize_tag, tag_distance
+
+
+def is_english(language_code: Optional[str]) -> bool:
+    if not language_code:
+        language_code = "und"
+
+    language_code = standardize_tag(language_code)
+    return tag_distance(language_code, "en") < 15

great_ai/utilities/language/predict_language.py

@@ -0,0 +1,16 @@
+from typing import Optional
+
+from langcodes import Language
+from langdetect import LangDetectException, detect
+
+
+def predict_language(text: Optional[str]) -> str:
+    if not text:
+        return Language.make().to_tag()
+
+    try:
+        language_code = detect(text)
+    except LangDetectException:
+        return Language.make().to_tag()
+
+    return Language.get(language_code).to_tag()

great_ai/utilities/logger/__init__.py

@@ -0,0 +1,2 @@
+from .custom_formatter import CustomFormatter
+from .get_logger import get_logger

great_ai/utilities/logger/colors.py

@@ -0,0 +1,6 @@
+GREY = "\x1b[38;21m"
+BLUE = "\x1b[38;5;39m"
+YELLOW = "\x1b[38;5;226m"
+RED = "\x1b[38;5;196m"
+BOLD_RED = "\x1b[31;1m"
+RESET = "\x1b[0m"

great_ai/utilities/logger/custom_formatter.py

@@ -0,0 +1,21 @@
+import logging
+
+from .colors import BLUE, BOLD_RED, GREY, RED, RESET, YELLOW
+
+
+class CustomFormatter(logging.Formatter):
+    def __init__(self, fmt: str):
+        super().__init__()
+        self._fmt = fmt
+        self._color_mapping = {
+            logging.DEBUG: GREY + self._fmt + RESET,
+            logging.INFO: BLUE + self._fmt + RESET,
+            logging.WARNING: YELLOW + self._fmt + RESET,
+            logging.ERROR: RED + self._fmt + RESET,
+            logging.CRITICAL: BOLD_RED + self._fmt + RESET,
+        }
+
+    def format(self, record: logging.LogRecord) -> str:
+        log_fmt = self._color_mapping.get(record.levelno)
+        formatter = logging.Formatter(log_fmt)
+        return formatter.format(record)

great_ai/utilities/logger/get_logger.py

@@ -0,0 +1,26 @@
+import logging
+from typing import Dict
+
+from .custom_formatter import CustomFormatter
+
+loggers: Dict[str, logging.Logger] = {}
+
+
+def get_logger(
+    name: str, level: int = logging.INFO, disable_colors: bool = False
+) -> logging.Logger:
+    if name not in loggers:
+        logger = logging.getLogger(name)
+        logger.setLevel(level)
+
+        fmt = "%(asctime)s | %(levelname)8s | %(message)s"
+
+        stdout_handler = logging.StreamHandler()
+        stdout_handler.setLevel(level)
+        if not disable_colors:
+            stdout_handler.setFormatter(CustomFormatter(fmt))
+
+        logger.addHandler(stdout_handler)
+        loggers[name] = logger
+
+    return loggers[name]

great_ai/utilities/match_names/__init__.py

@@ -0,0 +1 @@
+from .match_names import get_name_match_score, get_name_parts, match_names

great_ai/utilities/match_names/config.py

@@ -0,0 +1,5 @@
+last_name_weight = 1.5
+first_name_weight = 0.9
+infixes_weight = 0.5
+initials_weight = 0.6
+match_threshold = 1.51

great_ai/utilities/match_names/match_names.py

@@ -0,0 +1,162 @@
+import functools
+import re
+from typing import List, Optional, Tuple
+
+from ..clean import clean
+from .config import (
+    first_name_weight,
+    infixes_weight,
+    initials_weight,
+    last_name_weight,
+    match_threshold,
+)
+from .name_parts import NameParts
+
+
+def match_names(n1: Optional[str], n2: Optional[str]) -> bool:
+    score = get_name_match_score(n1, n2)
+    return score > match_threshold
+
+
+def get_name_match_score(n1: Optional[str], n2: Optional[str]) -> float:
+    if not n1 or not n2:
+        return 0
+
+    p1 = get_name_parts(n1)
+    p2 = get_name_parts(n2)
+
+    for f1 in p1.first_names:
+        for f2 in p2.first_names:
+            if f1[0] == f2[0]:
+                p1.initials = [i for i in p1.initials if i != f1[0]]
+                p2.initials = [i for i in p2.initials if i != f1[0]]
+
+    last_name_score = last_name_weight * _match_percentage(p1.last_names, p2.last_names)
+    first_name_score = first_name_weight * _match_percentage(
+        p1.first_names, p2.first_names
+    )
+    initials_score = initials_weight * _match_percentage(p1.initials, p2.initials)
+    infixes_score = min(0, infixes_weight * _match_percentage(p1.infixes, p2.infixes))
+
+    return last_name_score + first_name_score + initials_score + infixes_score
+
+
+def get_name_parts(name: str) -> NameParts:
+    result = _get_name_parts(name)
+    return result.copy()
+
+
+@functools.lru_cache(maxsize=None)
+def _get_name_parts(name: str) -> NameParts:
+    name = _clean_name(name)
+
+    first_names, name = _get_parenthesized_first_names(name)
+    initials, name = _get_initials(name)
+
+    last_names = []
+    infixes, name = _get_infixes(name)
+    if "," in name:
+        [last_name, *rest] = name.split(",")
+        rest_combined = " ".join(rest)
+        last_names.extend(last_name.split(" "))
+        first_names.extend(rest_combined.split(" "))
+    else:
+        parts = name.strip().split(" ")
+        if parts:
+            last_names.append(parts[-1])
+            parts.pop(-1)
+            first_names.extend(parts)
+        else:
+            last_names.append(name)
+
+    first_names = [f for f in first_names if f]
+    for f in first_names:
+        if all(f[0] != i for i in initials):
+            initials.append(f[0])
+
+    return NameParts(
+        first_names=first_names,
+        initials=initials,
+        infixes=infixes,
+        last_names=[
+            name
+            for last_name in last_names
+            for name in (last_name.split("-") if "-" in last_name else [last_name])
+            if name
+        ],
+    )
+
+
+def _clean_name(name: str) -> str:
+    name = clean(name, convert_to_ascii=True)
+
+    name = re.sub(
+        r"""
+            (MD|PhD|Ir|dr|Dr|DR|MSc|MA|BSc|BA|Prof)
+            [,.]?
+            [ ]*
+        """,
+        "",
+        name,
+        flags=re.VERBOSE,
+    )
+
+    subs_made = 1
+    while subs_made:
+        name, subs_made = re.subn(r"([A-Z]\.)\s+([A-Z])\b", r"\g<1>\g<2>", name)
+
+    name = name.replace(r"\s+-\s+", "-")
+    name = " ".join(p for p in name.split(" ") if p)
+    name = re.sub(r"^([^,.]+) ([A-Z])$", r"\g<1>, \g<2>.", name)
+    return name
+
+
+def _get_parenthesized_first_names(name: str) -> Tuple[List[str], str]:
+    groups = re.search(r"\(([a-zA-Z ]+)\)", name)
+    if groups:
+        return (
+            groups.group(1).split(" "),
+            _remove_between_indices(name, groups.start(), groups.end()),
+        )
+
+    return [], name
+
+
+def _get_initials(n: str) -> Tuple[List[str], str]:
+    initials = []
+
+    groups = re.search(r"\b(?P<abr1>([A-Z]\.)*)(?P<abr2>[A-Z])(\.|$)", n)
+    if groups:
+        first_name_abbreviations = [
+            *groups.group("abr1").split("."),
+            groups.group("abr2"),
+        ]
+        first_name_abbreviations = [f for f in first_name_abbreviations if f != ""]
+        initials.extend(first_name_abbreviations)
+        n = _remove_between_indices(n, groups.start(), groups.end())
+
+    return initials, n
+
+
+def _get_infixes(n: str) -> Tuple[List[str], str]:
+    infixes = ["de", "van", "der", "den", "te", "ter", "ten"]
+
+    parts = n.split(" ")
+    return [p for p in parts if p.lower() in infixes], " ".join(
+        p for p in parts if p.lower() not in infixes
+    )
+
+
+def _remove_between_indices(s: str, start: int, end: int) -> str:
+    return s[:start] + s[end:]
+
+
+def _match_percentage(l1: List[str], l2: List[str]) -> float:
+    if not l1 or not l2:
+        return 0
+
+    s1 = set(l1)
+    s2 = set(l2)
+    common_q = 2 * len(s1 & s2) / (len(s1) + len(s2))
+    different_q = min(len(s1 - s2) / len(s1), len(s2 - s1) / len(s2))
+    return common_q - different_q

great_ai/utilities/match_names/name_parts.py

@@ -0,0 +1,10 @@
+from typing import List
+
+from pydantic import BaseModel
+
+
+class NameParts(BaseModel):
+    first_names: List[str]
+    initials: List[str]
+    infixes: List[str]
+    last_names: List[str]

great_ai/utilities/parallel_map/__init__.py

@@ -0,0 +1,3 @@
+from .parallel_map import parallel_map
+from .threaded_parallel_map import threaded_parallel_map
+from .worker_exception import WorkerException

great_ai/utilities/parallel_map/get_config.py

@@ -0,0 +1,56 @@
+import os
+from math import ceil
+from typing import Callable, Iterable, Optional, Sequence, Union
+
+from ..logger import get_logger
+from .parallel_map_configuration import ParallelMapConfiguration
+
+logger = get_logger("parallel_map")
+
+
+def get_config(
+    *,
+    function: Callable,
+    input_values: Union[Sequence, Iterable],
+    chunk_size: Optional[int],
+    concurrency: Optional[int],
+) -> ParallelMapConfiguration:
+
+    is_input_sequence = hasattr(input_values, "__len__")
+
+    if concurrency is None:
+        concurrency = len(os.sched_getaffinity(0))
+    assert concurrency >= 1, "At least one mapper process has to be created"
+
+    if chunk_size is None:
+        if is_input_sequence:
+            chunk_size = max(1, ceil(len(input_values) / concurrency / 10))
+        else:
+            raise ValueError(
+                "The argument for `values` does not implement `__len__`, therefore, you must provide a `chunk_size`"
+            )
+    assert chunk_size >= 1, "Chunks have to contain at least one element"
+
+    chunk_count: Optional[int] = None
+    if is_input_sequence:
+        chunk_count = ceil(len(input_values) / chunk_size)
+        if chunk_count < concurrency:
+            logger.warning(
+                f"Limiting concurrency to {chunk_count} because there are only {chunk_count} chunks"
+            )
+            concurrency = chunk_count
+
+    if concurrency == 1:
+        logger.warning("Running in series, there is no reason for parallelism")
+
+    input_length = len(input_values) if is_input_sequence else None
+
+    config = ParallelMapConfiguration(
+        concurrency=concurrency,
+        chunk_count=chunk_count,
+        chunk_size=chunk_size,
+        input_length=input_length,
+        function_name=function.__name__ if hasattr(function, "__name__") else "unknown",
+    )
+
+    return config

great_ai/utilities/parallel_map/manage_communication.py

@@ -0,0 +1,70 @@
+import multiprocessing as mp
+import queue
+import traceback
+from typing import Dict, Iterable, TypeVar, Union
+
+from ..chunk import chunk
+from ..logger import get_logger
+from .worker_exception import WorkerException
+
+logger = get_logger("parallel_map")
+
+T = TypeVar("T")
+V = TypeVar("V")
+
+
+def manage_communication(
+    *,
+    input_values: Iterable[T],
+    chunk_size: int,
+    input_queue: Union[mp.Queue, queue.Queue],
+    output_queue: Union[mp.Queue, queue.Queue],
+    unordered: bool,
+    ignore_exceptions: bool,
+) -> Iterable[V]:
+    chunks = iter(chunk(enumerate(input_values), chunk_size=chunk_size))
+    indexed_results: Dict[int, V] = {}
+    next_output_index = 0
+    read_input_length = 0
+    is_iteration_over = False
+    while not is_iteration_over or next_output_index < read_input_length:
+        if not is_iteration_over:
+            try:
+                next_chunk = next(chunks)
+                input_queue.put(next_chunk)
+                read_input_length += len(next_chunk)
+            except StopIteration:
+                is_iteration_over = True
+            except Exception as e:
+                if not ignore_exceptions:
+                    raise
+                else:
+                    logger.error(
+                        f"Exception {e} encountered in input, traceback:\n{traceback.format_exc()}"
+                    )
+
+        try:
+            result_chunk = output_queue.get_nowait()
+
+            for index, value, exception in result_chunk:
+                if exception is not None:
+                    e, tb = exception
+                    if not ignore_exceptions:
+                        raise WorkerException from e
+                    else:
+                        logger.error(
+                            f"Exception {e} encountered in worker, traceback:\n{tb}"
+                        )
+                if unordered:
+                    yield value
+                    next_output_index += 1
+                else:
+                    indexed_results[index] = value
+
+            if not unordered:
+                while next_output_index in indexed_results:
+                    yield indexed_results[next_output_index]
+                    del indexed_results[next_output_index]
+                    next_output_index += 1
+        except queue.Empty:
+            pass

great_ai/utilities/parallel_map/manage_serial.py

@@ -0,0 +1,36 @@
+import traceback
+from typing import Callable, Iterable, TypeVar
+
+from ..logger import get_logger
+from .worker_exception import WorkerException
+
+logger = get_logger("parallel_map")
+
+T = TypeVar("T")
+V = TypeVar("V")
+
+
+def manage_serial(
+    *,
+    function: Callable[[T], V],
+    input_values: Iterable[T],
+    ignore_exceptions: bool,
+) -> Iterable[V]:
+    try:
+        for v in input_values:
+            try:
+                yield function(v)
+            except Exception as e:
+                if not ignore_exceptions:
+                    raise WorkerException from e
+                else:
+                    logger.error(
+                        f"Exception {e} encountered in worker, traceback:\n{traceback.format_exc()}"
+                    )
+    except Exception as e:
+        if not ignore_exceptions:
+            raise
+        else:
+            logger.error(
+                f"Exception {e} encountered in input, traceback:\n{traceback.format_exc()}"
+            )

great_ai/utilities/parallel_map/mapper_function.py

@@ -0,0 +1,43 @@
+import multiprocessing as mp
+import queue
+import threading
+import traceback
+from typing import Callable, Union
+
+import dill
+
+
+def mapper_function(
+    input_queue: Union[mp.Queue, queue.Queue],
+    output_queue: Union[mp.Queue, queue.Queue],
+    should_stop: Union[mp.Event, threading.Event],
+    map_function: Union[bytes, Callable],
+) -> None:
+    try:
+        if isinstance(map_function, bytes):
+            map_function = dill.loads(map_function)
+
+        last_chunk = None
+        while not should_stop.wait(0.1):
+            if last_chunk is None:
+                try:
+                    input_chunk = input_queue.get_nowait()
+                    last_chunk = []
+                    for i, v in input_chunk:
+                        result, exception = None, None
+                        try:
+                            result = map_function(v)
+                        except Exception as e:
+                            exception = e, traceback.format_exc()
+                        last_chunk.append((i, result, exception))
+                except queue.Empty:
+                    pass
+
+            if last_chunk is not None:
+                try:
+                    output_queue.put_nowait(last_chunk)
+                    last_chunk = None
+                except queue.Full:
+                    pass
+    except (KeyboardInterrupt, BrokenPipeError):
+        pass

great_ai/utilities/parallel_map/parallel_map.py

@@ -0,0 +1,145 @@
+import multiprocessing as mp
+from typing import Callable, Iterable, Literal, Optional, Sequence, TypeVar, overload
+
+import dill
+
+from .get_config import get_config
+from .manage_communication import manage_communication
+from .manage_serial import manage_serial
+from .mapper_function import mapper_function
+from .worker_exception import WorkerException
+
+T = TypeVar("T")
+V = TypeVar("V")
+
+
+@overload
+def parallel_map(
+    function: Callable[[T], V],
+    input_values: Sequence[T],
+    *,
+    chunk_size: Optional[int],
+    concurrency: Optional[int],
+    unordered: Optional[bool],
+    ignore_exceptions: Optional[Literal[False]],
+) -> Iterable[V]:
+    ...
+
+
+@overload
+def parallel_map(
+    function: Callable[[T], V],
+    input_values: Iterable[T],
+    *,
+    chunk_size: int,
+    concurrency: Optional[int],
+    unordered: Optional[bool],
+    ignore_exceptions: Optional[Literal[False]],
+) -> Iterable[V]:
+    ...
+
+
+@overload
+def parallel_map(
+    function: Callable[[T], V],
+    input_values: Sequence[T],
+    *,
+    chunk_size: Optional[int],
+    concurrency: Optional[int],
+    unordered: Optional[bool],
+    ignore_exceptions: True,
+) -> Iterable[Optional[V]]:
+    ...
+
+
+@overload
+def parallel_map(
+    function: Callable[[T], V],
+    input_values: Iterable[T],
+    *,
+    chunk_size: int,
+    concurrency: Optional[int],
+    unordered: Optional[bool],
+    ignore_exceptions: True,
+) -> Iterable[Optional[V]]:
+    ...
+
+
+def parallel_map(
+    function,
+    input_values,
+    *,
+    chunk_size=None,
+    concurrency=None,
+    unordered=False,
+    ignore_exceptions=False,
+):
+    config = get_config(
+        function=function,
+        input_values=input_values,
+        chunk_size=chunk_size,
+        concurrency=concurrency,
+    )
+
+    if config.concurrency == 1:
+        yield from manage_serial(
+            function=function,
+            input_values=input_values,
+            ignore_exceptions=ignore_exceptions,
+        )
+
+    ctx = (
+        mp.get_context("fork")
+        if "fork" in mp.get_all_start_methods()
+        else mp.get_context("spawn")
+    )
+    ctx.freeze_support()
+    manager = ctx.Manager()
+    input_queue = manager.Queue(config.concurrency * 2)
+    output_queue = manager.Queue(config.concurrency * 2)
+
+    should_stop = ctx.Event()
+    serialized_map_function = dill.dumps(function, byref=True, recurse=True)
+
+    processes = [
+        ctx.Process(
+            name=f"parallel_map_{config.function_name}_{i}",
+            target=mapper_function,
+            daemon=True,
+            kwargs=dict(
+                input_queue=input_queue,
+                output_queue=output_queue,
+                should_stop=should_stop,
+                map_function=serialized_map_function,
+            ),
+        )
+        for i in range(config.concurrency)
+    ]
+
+    for p in processes:
+        p.start()
+
+    try:
+        yield from manage_communication(
+            input_values=input_values,
+            chunk_size=config.chunk_size,
+            input_queue=input_queue,
+            output_queue=output_queue,
+            unordered=unordered,
+            ignore_exceptions=ignore_exceptions,
+        )
+        should_stop.set()
+    except WorkerException:
+        should_stop.set()
+        raise
+    except Exception:
+        for p in processes:
+            p.terminate()
+            p.kill()
+        raise
+    finally:
+        for p in processes:
+            p.join()  # terminated processes have to be joined else they remain zombies
+            p.close()
+
+        manager.shutdown()

great_ai/utilities/parallel_map/parallel_map_configuration.py

@@ -0,0 +1,15 @@
+from typing import Optional
+
+from pydantic import BaseModel
+
+from ..logger import get_logger
+
+logger = get_logger("parallel_map")
+
+
+class ParallelMapConfiguration(BaseModel):
+    concurrency: int
+    chunk_count: Optional[int]
+    chunk_size: int
+    input_length: Optional[int]
+    function_name: str

great_ai/utilities/parallel_map/threaded_parallel_map.py

@@ -0,0 +1,121 @@
+import queue
+import threading
+from typing import Callable, Iterable, Literal, Optional, Sequence, TypeVar, overload
+
+from .get_config import get_config
+from .manage_communication import manage_communication
+from .manage_serial import manage_serial
+from .mapper_function import mapper_function
+
+T = TypeVar("T")
+V = TypeVar("V")
+
+
+@overload
+def threaded_parallel_map(
+    function: Callable[[T], V],
+    input_values: Sequence[T],
+    *,
+    chunk_size: Optional[int],
+    concurrency: Optional[int],
+    unordered: Optional[bool],
+    ignore_exceptions: Optional[Literal[False]],
+) -> Iterable[V]:
+    ...
+
+
+@overload
+def threaded_parallel_map(
+    function: Callable[[T], V],
+    input_values: Iterable[T],
+    *,
+    chunk_size: int,
+    concurrency: Optional[int],
+    unordered: Optional[bool],
+    ignore_exceptions: Optional[Literal[False]],
+) -> Iterable[V]:
+    ...
+
+
+@overload
+def threaded_parallel_map(
+    function: Callable[[T], V],
+    input_values: Sequence[T],
+    *,
+    chunk_size: Optional[int],
+    concurrency: Optional[int],
+    unordered: Optional[bool],
+    ignore_exceptions: True,
+) -> Iterable[Optional[V]]:
+    ...
+
+
+@overload
+def threaded_parallel_map(
+    function: Callable[[T], V],
+    input_values: Iterable[T],
+    *,
+    chunk_size: int,
+    concurrency: Optional[int],
+    unordered: Optional[bool],
+    ignore_exceptions: True,
+) -> Iterable[Optional[V]]:
+    ...
+
+
+def threaded_parallel_map(
+    function,
+    input_values,
+    *,
+    chunk_size=None,
+    concurrency=None,
+    unordered=False,
+    ignore_exceptions=False,
+):
+    config = get_config(
+        function=function,
+        input_values=input_values,
+        chunk_size=chunk_size,
+        concurrency=concurrency,
+    )
+
+    if config.concurrency == 1:
+        yield from manage_serial(
+            function=function,
+            input_values=input_values,
+            ignore_exceptions=ignore_exceptions,
+        )
+
+    input_queue = queue.Queue(config.concurrency * 2)
+    output_queue = queue.Queue(config.concurrency * 2)
+    should_stop = threading.Event()
+
+    threads = [
+        threading.Thread(
+            name=f"threaded_parallel_map_{config.function_name}_{i}",
+            target=mapper_function,
+            daemon=True,
+            kwargs=dict(
+                input_queue=input_queue,
+                output_queue=output_queue,
+                should_stop=should_stop,
+                map_function=function,
+            ),
+        )
+        for i in range(config.concurrency)
+    ]
+
+    for t in threads:
+        t.start()
+
+    yield from manage_communication(
+        input_values=input_values,
+        chunk_size=config.chunk_size,
+        input_queue=input_queue,
+        output_queue=output_queue,
+        unordered=unordered,
+        ignore_exceptions=ignore_exceptions,
+    )
+    should_stop.set()
+    for t in threads:
+        t.join(1)

great_ai/utilities/parallel_map/worker_exception.py

@@ -0,0 +1,2 @@
+class WorkerException(Exception):
+    pass

great_ai/utilities/unchunk.py

@@ -0,0 +1,9 @@
+from typing import Iterable, Optional, TypeVar
+
+T = TypeVar("T")
+
+
+def unchunk(chunks: Iterable[Optional[Iterable[T]]]) -> Iterable[T]:
+    for chunk in chunks:
+        if chunk is not None:
+            yield from chunk

great_ai/utilities/unique.py

@@ -0,0 +1,34 @@
+from typing import Any, Callable, Iterable, List, TypeVar
+
+T = TypeVar("T")
+
+
+def unique(values: Iterable[T], *, key: Callable[[T], Any] = lambda v: v) -> List[T]:
+    """Keep only the first occurrences while maintaining original order.
+
+    The equality check used for deduplication can be overridden using the `key` argument.
+
+    Examples:
+        >>> unique([1, 1, 5, 3, 3])
+        [1, 5, 3]
+        >>> unique([{'a': 1, 'b': 2}, {'a': 1, 'b': 3}], key=lambda v: v['a'])
+        [{'a': 1, 'b': 2}]
+        >>> unique([{'a': 1, 'b': 2}, {'a': 1, 'b': 3}], key=lambda v: v['b'])
+        [{'a': 1, 'b': 2}, {'a': 1, 'b': 3}]
+
+    Args:
+        values: An iterable containing your values
+        key: Override the identity function of the equality check.
+
+    Returns:
+        A deduplicated list.
+    """
+
+    key_values = {}
+    for v in values:
+        k = key(v)
+        if k not in key_values:
+            # dicts maintain insertion order: https://mail.python.org/pipermail/python-dev/2017-December/151283.html
+            key_values[k] = v
+
+    return list(key_values.values())