Document utilities

great_ai/utilities/__init__.py

@@ -1,15 +1,16 @@
 from .chunk import chunk
 from .clean import clean
-from .config_file import ConfigFile, ParseError
+from .config_file.config_file import ConfigFile
-from .evaluate_ranking import evaluate_ranking
+from .config_file.parse_error import ParseError
+from .evaluate_ranking.evaluate_ranking import evaluate_ranking
 from .get_sentences import get_sentences
-from .language import english_name_of_language, is_english, predict_language
+from .language.english_name_of_language import english_name_of_language
-from .logger import get_logger
+from .language.is_english import is_english
-from .parallel_map import (
+from .language.predict_language import predict_language
-    WorkerException,
+from .logger.get_logger import get_logger
-    parallel_map,
+from .parallel_map.parallel_map import parallel_map
-    simple_parallel_map,
+from .parallel_map.simple_parallel_map import simple_parallel_map
-    threaded_parallel_map,
+from .parallel_map.threaded_parallel_map import threaded_parallel_map
-)
+from .parallel_map.worker_exception import WorkerException
 from .unchunk import unchunk
 from .unique import unique

great_ai/utilities/chunk.py

@@ -4,6 +4,25 @@ T = TypeVar("T")
 
 
 def chunk(values: Iterable[T], chunk_size: int) -> Iterable[List[T]]:
+    """Turn an iterable of items into an iterable of lists (chunks) of items.
+
+    Each returned chunk is of length `chunk_size` except the last one the length of
+    which is between 1 and `chunk_size`.
+
+    Useful for parallel processing.
+
+    Examples:
+        >>> list(chunk(range(10), chunk_size=3))
+        [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]
+
+    Args:
+        values: The stream of items to pack into chunks.
+        chunk_size: Desired length of each (but the last) chunk.
+
+    Yields:
+        The next chunk.
+    """
+
     assert chunk_size >= 1
 
     result: List[T] = []

great_ai/utilities/clean.py

@@ -6,7 +6,7 @@ import unidecode
 
 from .data import left_regular_punctuations, right_regular_punctuations
 from .external.pylatexenc.latex2text import LatexNodes2Text
-from .logger import get_logger
+from .logger.get_logger import get_logger
 
 logger = get_logger("clean")
 latex = LatexNodes2Text()
@@ -23,6 +23,43 @@ def clean(
     remove_brackets: bool = False,
     convert_to_ascii: bool = False,
 ) -> str:
+    """Clean all XML, LaTeX, PDF-extraction, and Unicode artifacts from the text.
+
+    The cleaning is quite heavy-weight and can be destructive. However, when working
+    with text, this is usually required to achieve sufficient cleanliness before further
+    processing.
+
+    Optionally, the text can be turned into ASCII. Carefully consider whether this is
+    absolutely needed for your use-case.
+
+    Examples:
+        >>> clean('<h2 color="red">Bj\\\\"{o}rn is \\t \\\\textit{happy} 🙂 &lt;3</h2>')
+        'Björn is happy 🙂 <3'
+
+        >>> clean(
+        ...    '<h2 color="red">Bj\\\\"{o}rn    is \\t \\\\textit{happy} 🙂 &lt;3</h2>',
+        ...    convert_to_ascii=True
+        ... )
+        'Bjorn is happy <3'
+
+        >>> clean(
+        ... '<h2 color="red">Bj\\\\"{o}rn       is \\t \\\\textit{happy} 🙂 &lt;3</h2>',
+        ... ignore_xml=True
+        ... )
+        '<h2 color="red">Björn is happy 🙂 lt;3</h2>'
+
+    Args:
+        text: Text to be cleaned.
+        ignore_xml: Do not process/remove XML-tags.
+        ignore_latex: Do not process/remove LaTeX-tags.
+        remove_brackets: Do not remove brackets ([])
+        convert_to_ascii: Strip (or convert) non-ascii characters.
+
+    Returns:
+        The cleaned input text with sensibly collapsed whitespace and optionally no
+            markup.
+    """
+
     if not ignore_xml:
         text = re.sub(r"<[^>]*>", " ", text)
         text = html.unescape(text)

great_ai/utilities/config_file/config_file.py

@@ -2,7 +2,7 @@ import os
 from pathlib import Path
 from typing import Dict, ItemsView, Iterator, KeysView, Mapping, Union, ValuesView
 
-from ..logger import get_logger
+from ..logger.get_logger import get_logger
 from .parse_error import ParseError
 from .pattern import pattern
 
@@ -10,9 +10,63 @@ logger = get_logger("ConfigFile")
 
 
 class ConfigFile(Mapping[str, str]):
+    """A small and safe `INI`-style configuration loader with `dict` and `ENV` support.
+
+    The values can be accessed using both dot- and index-notation. It is compatible
+    with the `dict` interface.
+
+    File format example:
+
+    ```toml
+    # comments are allowed everywhere
+
+    key = value  # you can leave or omit whitespace around the equal-sign
+    my_hashtag = "#great_ai" # the r-value can be quoted with " or ' or `.
+
+    my_var = my_default_value  # Default values can be given to env-vars,
+                               # see next line. The default value must come first.
+
+    my_var = ENV:MY_ENV_VAR    # If the value starts with the `ENV:` prefix,
+                               # it is looked up from the environment variables.
+    ```
+
+    Examples:
+        >>> ConfigFile('tests/utilities/data/simple.conf')
+        ConfigFile(path=tests/utilities/data/simple.conf) {'zeroth_key': 'test', 'first_key': 'András'}
+
+        >>> ConfigFile('tests/utilities/data/simple.conf').zeroth_key
+        'test'
+
+        >>> ConfigFile('tests/utilities/data/simple.conf').second_key
+        Traceback (most recent call last):
+        ...
+        KeyError: 'Key `second_key` is not found in configuration file ...
+
+        >>> a = ConfigFile('tests/utilities/data/simple.conf')
+        >>> {**a}
+        {'zeroth_key': 'test', 'first_key': 'András'}
+
+    """
+
     ENVIRONMENT_VARIABLE_KEY_PREFIX = "ENV"
 
     def __init__(self, path: Union[Path, str], *, ignore_missing: bool = False) -> None:
+        """Load and parse a configuration file.
+
+        Everything is eager-loaded, thus, exceptions may be thrown here.
+
+        Args:
+            path: Local path of the configuration file.
+            ignore_missing: Don't raise an exception on missing environment variables.
+
+        Raises:
+            FileNotFoundError: If there is no file at the specified path.
+            ParseError: If the provided file does not conform to the expected format.
+            KeyError: If there is duplication in the keys.
+            ValueError: If an environment variable is referenced but it is not set in
+                the system and `ignore_missing=False`.
+        """
+
         if not isinstance(path, Path):
             path = Path(path)
 
@@ -28,6 +82,7 @@ class ConfigFile(Mapping[str, str]):
 
     @property
     def path(self) -> Path:
+        """Original path from where the configuration was loaded."""
         return self._path
 
     def _parse(self) -> None:
@@ -54,16 +109,20 @@ class ConfigFile(Mapping[str, str]):
             if value.startswith(f"{self.ENVIRONMENT_VARIABLE_KEY_PREFIX}:"):
                 _, value = value.split(":")
                 if value not in os.environ:
-                    issue = f'The value of `{key}` contains the "{self.ENVIRONMENT_VARIABLE_KEY_PREFIX}` prefix but `{value}` is not defined as an environment variable'
+                    issue = f"""The value of `{key}` contains the "{
+                        self.ENVIRONMENT_VARIABLE_KEY_PREFIX
+                    }` prefix but `{value}` is not defined as an environment variable"""
                     if already_exists:
                         logger.warning(
-                            f"{issue}, using the default value defined above (`{self._key_values[key]}`)"
+                            f"""{issue}, using the default value defined above (`{
+                                self._key_values[key]
+                            }`)"""
                         )
                         continue
                     elif self._ignore_missing:
                         logger.warning(issue)
                     else:
-                        raise KeyError(
+                        raise ValueError(
                             f"{issue} and no default value has been provided"
                         )
                 else:

great_ai/utilities/evaluate_ranking/__init__.py

@@ -1,2 +0,0 @@
-from .draw_f1_iso_lines import draw_f1_iso_lines
-from .evaluate_ranking import evaluate_ranking

great_ai/utilities/evaluate_ranking/evaluate_ranking.py

@@ -23,6 +23,26 @@ def evaluate_ranking(
     reverse_order: bool = False,
     plot: bool = True,
 ) -> Dict[T, float]:
+    """Render the Precision-Recall curve of a ranking.
+
+    And improved version of scikit-learn's [PR-curve](https://scikit-learn.org/stable/auto_examples/model_selection/plot_precision_recall.html#sphx-glr-auto-examples-model-selection-plot-precision-recall-py)
+
+    Args:
+        expected: Expected ordering of the elements
+            (rank if it's an integer, alphabetical if a string)
+        actual_scores: Actual ranking scores (need not be on the same scale as
+            `expected`)
+        title: Title of the plot.
+        disable_interpolation: Do not interpolate.
+        axes: Matplotlib axes for ploting inside a subplot.
+        output_svg: If specified, save the chart as an svg to the given Path.
+        reverse_order: Reverse the ranking specified by `expected`.
+        plot: Display a plot on the screen.
+
+    Returns:
+        Precision values at given recall.
+    """
+
     assert 0 <= target_recall <= 1
 
     if plot and axes is None:

great_ai/utilities/get_sentences.py

@@ -16,6 +16,32 @@ def get_sentences(
     true_case: bool = False,
     remove_punctuation: bool = False,
 ) -> List[str]:
+    """Return the list of sentences found in the input text.
+
+    Use [syntok](https://github.com/fnl/syntok) to segment the sentences. Further
+    processing can be enabled with optional arguments.
+
+    Examples:
+        >>> get_sentences('This is a sentence. This is a half')
+        ['This is a sentence.', 'This is a half']
+
+        >>> get_sentences('This is a sentence. This is a half', ignore_partial=True)
+        ['This is a sentence.']
+
+        >>> get_sentences('I like Apple.', true_case=True, remove_punctuation=True)
+        ['i like Apple']
+
+    Args:
+        text: Text to be segmented into sentences.
+        ignore_partial: Filter out sentences that are not capitalised/don't end with a
+        punctuation.
+        true_case: Crude method: lowercase the first word of each sentence.
+        remove_punctuation: Remove all kinds of punctuation.
+
+    Returns:
+        The found sentences (with partial sentences optionally filtered out).
+    """
+
     tokenizer = Tokenizer(
         emit_hyphen_or_underscore_sep=True, replace_not_contraction=False
     )

great_ai/utilities/language/__init__.py

@@ -1,3 +0,0 @@
-from .english_name_of_language import english_name_of_language
-from .is_english import is_english
-from .predict_language import predict_language

great_ai/utilities/language/english_name_of_language.py

@@ -4,6 +4,26 @@ from langcodes import Language
 
 
 def english_name_of_language(language_code: Optional[str]) -> str:
+    """Human-friendly English name of language from its `language_code`.
+
+    A thin wrapper over [langcodes](https://github.com/rspeer/langcodes) for convenient
+    language tagging.
+
+    Examples:
+        >>> english_name_of_language('en-US')
+        'English (United States)'
+
+        >>> english_name_of_language('und')
+        'Unknown language'
+
+    Args:
+        language_code: Language code, for example, returned by
+            [great_ai.utilities.predict_language][].
+
+    Returns:
+        English name of language.
+    """
+
     if not language_code:
         language_code = "und"
 

great_ai/utilities/language/is_english.py

@@ -4,6 +4,28 @@ from langcodes import standardize_tag, tag_distance
 
 
 def is_english(language_code: Optional[str]) -> bool:
+    """Decide whether the `language_code` is of an English language.
+
+    A thin wrapper over [langcodes](https://github.com/rspeer/langcodes) for convenient
+    language tagging.
+
+    Examples:
+        >>> is_english('en-US')
+        True
+
+        >>> is_english(None)
+        False
+
+        >>> is_english('und')
+        False
+
+    Args:
+        language_code: Language code, for example, returned by
+            `[great_ai.utilities.predict_language][].
+
+    Returns:
+        Boolean indicating whether it's English.
+    """
     if not language_code:
         language_code = "und"
 

great_ai/utilities/language/predict_language.py

@@ -5,6 +5,23 @@ from langdetect import LangDetectException, detect
 
 
 def predict_language(text: Optional[str]) -> str:
+    """Predict the language code from text.
+
+    A thin wrapper over [langcodes](https://github.com/rspeer/langcodes) for convenient
+    language tagging.
+
+    Examples:
+        >>> predict_language('This is a sentence.')
+        'en'
+
+    Args:
+        text: Text used for prediction.
+
+    Returns:
+        The predicted language code (en, en-US) or `und` if a prediction could not be
+            made.
+    """
+
     if not text:
         return Language.make().to_tag()
 

great_ai/utilities/logger/__init__.py

@@ -1,2 +0,0 @@
-from .custom_formatter import CustomFormatter
-from .get_logger import get_logger

great_ai/utilities/logger/get_logger.py

@@ -9,6 +9,8 @@ loggers: Dict[str, logging.Logger] = {}
 def get_logger(
     name: str, level: int = logging.INFO, disable_colors: bool = False
 ) -> logging.Logger:
+    """Return a customised logger used throughout the GreatAI codebase."""
+
     if name not in loggers:
         logger = logging.getLogger(name)
         logger.setLevel(level)

great_ai/utilities/parallel_map/__init__.py

@@ -1,4 +0,0 @@
-from .parallel_map import parallel_map
-from .simple_parallel_map import simple_parallel_map
-from .threaded_parallel_map import threaded_parallel_map
-from .worker_exception import WorkerException

great_ai/utilities/parallel_map/get_config.py

@@ -2,7 +2,7 @@ import os
 from math import ceil
 from typing import Callable, Iterable, Optional, Sequence, Union
 
-from ..logger import get_logger
+from ..logger.get_logger import get_logger
 from .parallel_map_configuration import ParallelMapConfiguration
 
 logger = get_logger("parallel_map")

great_ai/utilities/parallel_map/manage_communication.py

@@ -4,7 +4,7 @@ import traceback
 from typing import Dict, Iterable, List, TypeVar, Union
 
 from ..chunk import chunk
-from ..logger import get_logger
+from ..logger.get_logger import get_logger
 from .map_result import MapResult
 from .worker_exception import WorkerException
 

great_ai/utilities/parallel_map/parallel_map.py

@@ -83,6 +83,49 @@ def parallel_map(
     concurrency: Optional[int] = None,
     unordered: bool = False,
 ) -> Iterable[Optional[V]]:
+    """Execute a map operation on an iterable stream.
+
+    A custom parallel map operation supporting both synchronous and `async` map
+    functions. The `func` function is serialised with `dill`. Exceptions encountered in
+    the map function are sent to the host process where they are either raised (default)
+    or ignored.
+
+    The new processes are forked if the OS allows it, otherwise, new Python processes
+    are bootstrapped which can incur some startup cost. Each process processes a single
+    chunk at once.
+
+    Examples:
+        >>> import math
+        >>> list(parallel_map(math.sqrt, [9, 4, 1], concurrency=2))
+        [3.0, 2.0, 1.0]
+
+    Args:
+        func: The function that should be applied to each element of `input_values`.
+            It can `async`, in that case, a new event loop is started for each chunk.
+        input_values: An iterable of items that `func` is applied to.
+        chunk_size: Tune the number of items processed in each step. Larger numbers
+            result in smaller communication overhead but less parallelism at the start
+            and end. If `chunk_size` has a `__len__` property, the `chunk_size` is
+            calculated automatically if not given.
+        ignore_exceptions: Ignore chunks if `next()` raises an exception on
+            `input_values`. And return `None` if `func` raised an exception in a worker
+            process.
+        concurrency: Number of new processes to start. Shouldn't be too much more than
+            the number of physical cores.
+        unordered: Do not preserve the order of the elements, yield them as soon as they
+            have been processed. This decreases the latency caused by
+            difficult-to-process items.
+
+    Yields:
+        The next result obtained from applying `func` to each input value. May
+            contain `None`-s if `ignore_exceptions=True`. May have different order than
+            the input if `unordered=True`.
+
+    Raises:
+        WorkerException: If there was an error in the `func` function in a background
+            process and `ignore_exceptions=False`.
+    """
+
     config = get_config(
         function=func,
         input_values=input_values,

great_ai/utilities/parallel_map/parallel_map_configuration.py

@@ -2,7 +2,7 @@ from typing import Optional
 
 from pydantic import BaseModel
 
-from ..logger import get_logger
+from ..logger.get_logger import get_logger
 
 logger = get_logger("parallel_map")
 

great_ai/utilities/parallel_map/simple_parallel_map.py

@@ -15,6 +15,36 @@ def simple_parallel_map(
     chunk_size: Optional[int] = None,
     concurrency: Optional[int] = None,
 ) -> List[V]:
+    """Execute a map operation on an list mimicking the API of the built-in `map()`.
+
+    A thin-wrapper over [parallel_map][great_ai.utilities.parallel_map.parallel_map.parallel_map].
+    For more options, consult the documentation of
+    [parallel_map][great_ai.utilities.parallel_map.parallel_map.parallel_map].
+
+    Examples:
+        >>> import math
+        >>> list(simple_parallel_map(math.sqrt, [9, 4, 1]))
+        [3.0, 2.0, 1.0]
+
+    Args:
+        func: The function that should be applied to each element of `input_values`.
+            It can `async`, in that case, a new event loop is started for each chunk.
+        input_values: An iterable of items that `func` is applied to.
+        chunk_size: Tune the number of items processed in each step. Larger numbers
+            result in smaller communication overhead but less parallelism at the start
+            and end. If `chunk_size` has a `__len__` property, the `chunk_size` is
+            calculated automatically if not given.
+        concurrency: Number of new processes to start. Shouldn't be too much more than
+            the number of physical cores.
+
+    Returns:
+        An iterable of results obtained from applying `func` to each input value.
+
+    Raises:
+        WorkerException: If there was an error in the `func` function in a background
+            process.
+    """
+
     input_values = list(input_values)  # in case the input is mistakenly not a sequence
     return list(
         tqdm(

great_ai/utilities/parallel_map/threaded_parallel_map.py

@@ -81,6 +81,47 @@ def threaded_parallel_map(
     concurrency: Optional[int] = None,
     unordered: bool = False,
 ) -> Iterable[Optional[V]]:
+    """Execute a map operation on an iterable stream.
+
+    Similar to [parallel_map][great_ai.utilities.parallel_map.parallel_map.parallel_map]
+    but uses threads instead of processes. Hence, it is not helpful in CPU-bound
+    situations.
+
+    A custom parallel map operation supporting both synchronous and `async` map
+    functions. Exceptions encountered in the map function are sent to the host thread
+    where they are either raised (default) or ignored. Each process processes a single
+    chunk at once.
+
+    Examples:
+        >>> list(threaded_parallel_map(lambda x: x ** 2, [1, 2, 3]))
+        [1, 4, 9]
+
+    Args:
+        func: The function that should be applied to each element of `input_values`.
+            It can `async`, in that case, a new event loop is started for each chunk.
+        input_values: An iterable of items that `func` is applied to.
+        chunk_size: Tune the number of items processed in each step. Larger numbers
+            result in smaller communication overhead but less parallelism at the start
+            and end. If `chunk_size` has a `__len__` property, the `chunk_size` is
+            calculated automatically if not given.
+        ignore_exceptions: Ignore chunks if `next()` raises an exception on
+            `input_values`. And return `None` if `func` raised an exception in a worker
+            process.
+        concurrency: Number of new threads to start.
+        unordered: Do not preserve the order of the elements, yield them as soon as they
+            have been processed. This decreases the latency caused by
+            difficult-to-process items.
+
+    Yields:
+        The next result obtained from applying `func` to each input value. May
+            contain `None`-s if `ignore_exceptions=True`. May have different order than
+            the input if `unordered=True`.
+
+    Raises:
+        WorkerException: If there was an error in the `func` function in a background
+            thread and `ignore_exceptions=False`.
+    """
+
     config = get_config(
         function=func,
         input_values=input_values,

great_ai/utilities/unchunk.py

@@ -4,6 +4,22 @@ T = TypeVar("T")
 
 
 def unchunk(chunks: Iterable[Optional[Iterable[T]]]) -> Iterable[T]:
+    """Turn a stream of chunks of items into a stream of items (flatten operation).
+
+    The inverse operation of [chunk][great_ai.utilities.chunk.chunk].
+    Useful for parallel processing.
+
+    Examples:
+        >>> list(unchunk([[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]))
+        [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+    Args:
+        chunks: Stream of chunks to unpack.
+
+    Yields:
+        The next item in the flattened iterable.
+    """
+
     for chunk in chunks:
         if chunk is not None:
             yield from chunk