diff --git a/great_ai/utilities/__init__.py b/great_ai/utilities/__init__.py index 65c2a17..2b52017 100644 --- a/great_ai/utilities/__init__.py +++ b/great_ai/utilities/__init__.py @@ -1,15 +1,16 @@ from .chunk import chunk from .clean import clean -from .config_file import ConfigFile, ParseError -from .evaluate_ranking import evaluate_ranking +from .config_file.config_file import ConfigFile +from .config_file.parse_error import ParseError +from .evaluate_ranking.evaluate_ranking import evaluate_ranking from .get_sentences import get_sentences -from .language import english_name_of_language, is_english, predict_language -from .logger import get_logger -from .parallel_map import ( - WorkerException, - parallel_map, - simple_parallel_map, - threaded_parallel_map, -) +from .language.english_name_of_language import english_name_of_language +from .language.is_english import is_english +from .language.predict_language import predict_language +from .logger.get_logger import get_logger +from .parallel_map.parallel_map import parallel_map +from .parallel_map.simple_parallel_map import simple_parallel_map +from .parallel_map.threaded_parallel_map import threaded_parallel_map +from .parallel_map.worker_exception import WorkerException from .unchunk import unchunk from .unique import unique diff --git a/great_ai/utilities/chunk.py b/great_ai/utilities/chunk.py index 13db74d..a0663d1 100644 --- a/great_ai/utilities/chunk.py +++ b/great_ai/utilities/chunk.py @@ -4,6 +4,25 @@ T = TypeVar("T") def chunk(values: Iterable[T], chunk_size: int) -> Iterable[List[T]]: + """Turn an iterable of items into an iterable of lists (chunks) of items. + + Each returned chunk is of length `chunk_size` except the last one the length of + which is between 1 and `chunk_size`. + + Useful for parallel processing. + + Examples: + >>> list(chunk(range(10), chunk_size=3)) + [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]] + + Args: + values: The stream of items to pack into chunks. + chunk_size: Desired length of each (but the last) chunk. + + Yields: + The next chunk. + """ + assert chunk_size >= 1 result: List[T] = [] diff --git a/great_ai/utilities/clean.py b/great_ai/utilities/clean.py index 378320b..521ae52 100644 --- a/great_ai/utilities/clean.py +++ b/great_ai/utilities/clean.py @@ -6,7 +6,7 @@ import unidecode from .data import left_regular_punctuations, right_regular_punctuations from .external.pylatexenc.latex2text import LatexNodes2Text -from .logger import get_logger +from .logger.get_logger import get_logger logger = get_logger("clean") latex = LatexNodes2Text() @@ -23,6 +23,43 @@ def clean( remove_brackets: bool = False, convert_to_ascii: bool = False, ) -> str: + """Clean all XML, LaTeX, PDF-extraction, and Unicode artifacts from the text. + + The cleaning is quite heavy-weight and can be destructive. However, when working + with text, this is usually required to achieve sufficient cleanliness before further + processing. + + Optionally, the text can be turned into ASCII. Carefully consider whether this is + absolutely needed for your use-case. + + Examples: + >>> clean('

Bj\\\\"{o}rn is \\t \\\\textit{happy} 🙂 <3

') + 'Björn is happy 🙂 <3' + + >>> clean( + ... '

Bj\\\\"{o}rn is \\t \\\\textit{happy} 🙂 <3

', + ... convert_to_ascii=True + ... ) + 'Bjorn is happy <3' + + >>> clean( + ... '

Bj\\\\"{o}rn is \\t \\\\textit{happy} 🙂 <3

', + ... ignore_xml=True + ... ) + '

Björn is happy 🙂 lt;3

' + + Args: + text: Text to be cleaned. + ignore_xml: Do not process/remove XML-tags. + ignore_latex: Do not process/remove LaTeX-tags. + remove_brackets: Do not remove brackets ([]) + convert_to_ascii: Strip (or convert) non-ascii characters. + + Returns: + The cleaned input text with sensibly collapsed whitespace and optionally no + markup. + """ + if not ignore_xml: text = re.sub(r"<[^>]*>", " ", text) text = html.unescape(text) diff --git a/great_ai/utilities/config_file/config_file.py b/great_ai/utilities/config_file/config_file.py index 0161842..948984c 100644 --- a/great_ai/utilities/config_file/config_file.py +++ b/great_ai/utilities/config_file/config_file.py @@ -2,7 +2,7 @@ import os from pathlib import Path from typing import Dict, ItemsView, Iterator, KeysView, Mapping, Union, ValuesView -from ..logger import get_logger +from ..logger.get_logger import get_logger from .parse_error import ParseError from .pattern import pattern @@ -10,9 +10,63 @@ logger = get_logger("ConfigFile") class ConfigFile(Mapping[str, str]): + """A small and safe `INI`-style configuration loader with `dict` and `ENV` support. + + The values can be accessed using both dot- and index-notation. It is compatible + with the `dict` interface. + + File format example: + + ```toml + # comments are allowed everywhere + + key = value # you can leave or omit whitespace around the equal-sign + my_hashtag = "#great_ai" # the r-value can be quoted with " or ' or `. + + my_var = my_default_value # Default values can be given to env-vars, + # see next line. The default value must come first. + + my_var = ENV:MY_ENV_VAR # If the value starts with the `ENV:` prefix, + # it is looked up from the environment variables. + ``` + + Examples: + >>> ConfigFile('tests/utilities/data/simple.conf') + ConfigFile(path=tests/utilities/data/simple.conf) {'zeroth_key': 'test', 'first_key': 'András'} + + >>> ConfigFile('tests/utilities/data/simple.conf').zeroth_key + 'test' + + >>> ConfigFile('tests/utilities/data/simple.conf').second_key + Traceback (most recent call last): + ... + KeyError: 'Key `second_key` is not found in configuration file ... + + >>> a = ConfigFile('tests/utilities/data/simple.conf') + >>> {**a} + {'zeroth_key': 'test', 'first_key': 'András'} + + """ + ENVIRONMENT_VARIABLE_KEY_PREFIX = "ENV" def __init__(self, path: Union[Path, str], *, ignore_missing: bool = False) -> None: + """Load and parse a configuration file. + + Everything is eager-loaded, thus, exceptions may be thrown here. + + Args: + path: Local path of the configuration file. + ignore_missing: Don't raise an exception on missing environment variables. + + Raises: + FileNotFoundError: If there is no file at the specified path. + ParseError: If the provided file does not conform to the expected format. + KeyError: If there is duplication in the keys. + ValueError: If an environment variable is referenced but it is not set in + the system and `ignore_missing=False`. + """ + if not isinstance(path, Path): path = Path(path) @@ -28,6 +82,7 @@ class ConfigFile(Mapping[str, str]): @property def path(self) -> Path: + """Original path from where the configuration was loaded.""" return self._path def _parse(self) -> None: @@ -54,16 +109,20 @@ class ConfigFile(Mapping[str, str]): if value.startswith(f"{self.ENVIRONMENT_VARIABLE_KEY_PREFIX}:"): _, value = value.split(":") if value not in os.environ: - issue = f'The value of `{key}` contains the "{self.ENVIRONMENT_VARIABLE_KEY_PREFIX}` prefix but `{value}` is not defined as an environment variable' + issue = f"""The value of `{key}` contains the "{ + self.ENVIRONMENT_VARIABLE_KEY_PREFIX + }` prefix but `{value}` is not defined as an environment variable""" if already_exists: logger.warning( - f"{issue}, using the default value defined above (`{self._key_values[key]}`)" + f"""{issue}, using the default value defined above (`{ + self._key_values[key] + }`)""" ) continue elif self._ignore_missing: logger.warning(issue) else: - raise KeyError( + raise ValueError( f"{issue} and no default value has been provided" ) else: diff --git a/great_ai/utilities/evaluate_ranking/__init__.py b/great_ai/utilities/evaluate_ranking/__init__.py index 94226bb..e69de29 100644 --- a/great_ai/utilities/evaluate_ranking/__init__.py +++ b/great_ai/utilities/evaluate_ranking/__init__.py @@ -1,2 +0,0 @@ -from .draw_f1_iso_lines import draw_f1_iso_lines -from .evaluate_ranking import evaluate_ranking diff --git a/great_ai/utilities/evaluate_ranking/evaluate_ranking.py b/great_ai/utilities/evaluate_ranking/evaluate_ranking.py index f5e4c51..38b3a92 100644 --- a/great_ai/utilities/evaluate_ranking/evaluate_ranking.py +++ b/great_ai/utilities/evaluate_ranking/evaluate_ranking.py @@ -23,6 +23,26 @@ def evaluate_ranking( reverse_order: bool = False, plot: bool = True, ) -> Dict[T, float]: + """Render the Precision-Recall curve of a ranking. + + And improved version of scikit-learn's [PR-curve](https://scikit-learn.org/stable/auto_examples/model_selection/plot_precision_recall.html#sphx-glr-auto-examples-model-selection-plot-precision-recall-py) + + Args: + expected: Expected ordering of the elements + (rank if it's an integer, alphabetical if a string) + actual_scores: Actual ranking scores (need not be on the same scale as + `expected`) + title: Title of the plot. + disable_interpolation: Do not interpolate. + axes: Matplotlib axes for ploting inside a subplot. + output_svg: If specified, save the chart as an svg to the given Path. + reverse_order: Reverse the ranking specified by `expected`. + plot: Display a plot on the screen. + + Returns: + Precision values at given recall. + """ + assert 0 <= target_recall <= 1 if plot and axes is None: diff --git a/great_ai/utilities/get_sentences.py b/great_ai/utilities/get_sentences.py index 1a507dc..d770530 100644 --- a/great_ai/utilities/get_sentences.py +++ b/great_ai/utilities/get_sentences.py @@ -16,6 +16,32 @@ def get_sentences( true_case: bool = False, remove_punctuation: bool = False, ) -> List[str]: + """Return the list of sentences found in the input text. + + Use [syntok](https://github.com/fnl/syntok) to segment the sentences. Further + processing can be enabled with optional arguments. + + Examples: + >>> get_sentences('This is a sentence. This is a half') + ['This is a sentence.', 'This is a half'] + + >>> get_sentences('This is a sentence. This is a half', ignore_partial=True) + ['This is a sentence.'] + + >>> get_sentences('I like Apple.', true_case=True, remove_punctuation=True) + ['i like Apple'] + + Args: + text: Text to be segmented into sentences. + ignore_partial: Filter out sentences that are not capitalised/don't end with a + punctuation. + true_case: Crude method: lowercase the first word of each sentence. + remove_punctuation: Remove all kinds of punctuation. + + Returns: + The found sentences (with partial sentences optionally filtered out). + """ + tokenizer = Tokenizer( emit_hyphen_or_underscore_sep=True, replace_not_contraction=False ) diff --git a/great_ai/utilities/language/__init__.py b/great_ai/utilities/language/__init__.py index 1ec3102..e69de29 100644 --- a/great_ai/utilities/language/__init__.py +++ b/great_ai/utilities/language/__init__.py @@ -1,3 +0,0 @@ -from .english_name_of_language import english_name_of_language -from .is_english import is_english -from .predict_language import predict_language diff --git a/great_ai/utilities/language/english_name_of_language.py b/great_ai/utilities/language/english_name_of_language.py index 9019c4d..372de5c 100644 --- a/great_ai/utilities/language/english_name_of_language.py +++ b/great_ai/utilities/language/english_name_of_language.py @@ -4,6 +4,26 @@ from langcodes import Language def english_name_of_language(language_code: Optional[str]) -> str: + """Human-friendly English name of language from its `language_code`. + + A thin wrapper over [langcodes](https://github.com/rspeer/langcodes) for convenient + language tagging. + + Examples: + >>> english_name_of_language('en-US') + 'English (United States)' + + >>> english_name_of_language('und') + 'Unknown language' + + Args: + language_code: Language code, for example, returned by + [great_ai.utilities.predict_language][]. + + Returns: + English name of language. + """ + if not language_code: language_code = "und" diff --git a/great_ai/utilities/language/is_english.py b/great_ai/utilities/language/is_english.py index 24fb77c..e40fbe9 100644 --- a/great_ai/utilities/language/is_english.py +++ b/great_ai/utilities/language/is_english.py @@ -4,6 +4,28 @@ from langcodes import standardize_tag, tag_distance def is_english(language_code: Optional[str]) -> bool: + """Decide whether the `language_code` is of an English language. + + A thin wrapper over [langcodes](https://github.com/rspeer/langcodes) for convenient + language tagging. + + Examples: + >>> is_english('en-US') + True + + >>> is_english(None) + False + + >>> is_english('und') + False + + Args: + language_code: Language code, for example, returned by + `[great_ai.utilities.predict_language][]. + + Returns: + Boolean indicating whether it's English. + """ if not language_code: language_code = "und" diff --git a/great_ai/utilities/language/predict_language.py b/great_ai/utilities/language/predict_language.py index 1fc3955..475994a 100644 --- a/great_ai/utilities/language/predict_language.py +++ b/great_ai/utilities/language/predict_language.py @@ -5,6 +5,23 @@ from langdetect import LangDetectException, detect def predict_language(text: Optional[str]) -> str: + """Predict the language code from text. + + A thin wrapper over [langcodes](https://github.com/rspeer/langcodes) for convenient + language tagging. + + Examples: + >>> predict_language('This is a sentence.') + 'en' + + Args: + text: Text used for prediction. + + Returns: + The predicted language code (en, en-US) or `und` if a prediction could not be + made. + """ + if not text: return Language.make().to_tag() diff --git a/great_ai/utilities/logger/__init__.py b/great_ai/utilities/logger/__init__.py index 20f9099..e69de29 100644 --- a/great_ai/utilities/logger/__init__.py +++ b/great_ai/utilities/logger/__init__.py @@ -1,2 +0,0 @@ -from .custom_formatter import CustomFormatter -from .get_logger import get_logger diff --git a/great_ai/utilities/logger/get_logger.py b/great_ai/utilities/logger/get_logger.py index 3e6fb90..fbbb62b 100644 --- a/great_ai/utilities/logger/get_logger.py +++ b/great_ai/utilities/logger/get_logger.py @@ -9,6 +9,8 @@ loggers: Dict[str, logging.Logger] = {} def get_logger( name: str, level: int = logging.INFO, disable_colors: bool = False ) -> logging.Logger: + """Return a customised logger used throughout the GreatAI codebase.""" + if name not in loggers: logger = logging.getLogger(name) logger.setLevel(level) diff --git a/great_ai/utilities/parallel_map/__init__.py b/great_ai/utilities/parallel_map/__init__.py index a984057..e69de29 100644 --- a/great_ai/utilities/parallel_map/__init__.py +++ b/great_ai/utilities/parallel_map/__init__.py @@ -1,4 +0,0 @@ -from .parallel_map import parallel_map -from .simple_parallel_map import simple_parallel_map -from .threaded_parallel_map import threaded_parallel_map -from .worker_exception import WorkerException diff --git a/great_ai/utilities/parallel_map/get_config.py b/great_ai/utilities/parallel_map/get_config.py index 98ffbe4..6d42bb7 100644 --- a/great_ai/utilities/parallel_map/get_config.py +++ b/great_ai/utilities/parallel_map/get_config.py @@ -2,7 +2,7 @@ import os from math import ceil from typing import Callable, Iterable, Optional, Sequence, Union -from ..logger import get_logger +from ..logger.get_logger import get_logger from .parallel_map_configuration import ParallelMapConfiguration logger = get_logger("parallel_map") diff --git a/great_ai/utilities/parallel_map/manage_communication.py b/great_ai/utilities/parallel_map/manage_communication.py index 1fc9630..d6c0740 100644 --- a/great_ai/utilities/parallel_map/manage_communication.py +++ b/great_ai/utilities/parallel_map/manage_communication.py @@ -4,7 +4,7 @@ import traceback from typing import Dict, Iterable, List, TypeVar, Union from ..chunk import chunk -from ..logger import get_logger +from ..logger.get_logger import get_logger from .map_result import MapResult from .worker_exception import WorkerException diff --git a/great_ai/utilities/parallel_map/parallel_map.py b/great_ai/utilities/parallel_map/parallel_map.py index c4c3bac..81cd0bd 100644 --- a/great_ai/utilities/parallel_map/parallel_map.py +++ b/great_ai/utilities/parallel_map/parallel_map.py @@ -83,6 +83,49 @@ def parallel_map( concurrency: Optional[int] = None, unordered: bool = False, ) -> Iterable[Optional[V]]: + """Execute a map operation on an iterable stream. + + A custom parallel map operation supporting both synchronous and `async` map + functions. The `func` function is serialised with `dill`. Exceptions encountered in + the map function are sent to the host process where they are either raised (default) + or ignored. + + The new processes are forked if the OS allows it, otherwise, new Python processes + are bootstrapped which can incur some startup cost. Each process processes a single + chunk at once. + + Examples: + >>> import math + >>> list(parallel_map(math.sqrt, [9, 4, 1], concurrency=2)) + [3.0, 2.0, 1.0] + + Args: + func: The function that should be applied to each element of `input_values`. + It can `async`, in that case, a new event loop is started for each chunk. + input_values: An iterable of items that `func` is applied to. + chunk_size: Tune the number of items processed in each step. Larger numbers + result in smaller communication overhead but less parallelism at the start + and end. If `chunk_size` has a `__len__` property, the `chunk_size` is + calculated automatically if not given. + ignore_exceptions: Ignore chunks if `next()` raises an exception on + `input_values`. And return `None` if `func` raised an exception in a worker + process. + concurrency: Number of new processes to start. Shouldn't be too much more than + the number of physical cores. + unordered: Do not preserve the order of the elements, yield them as soon as they + have been processed. This decreases the latency caused by + difficult-to-process items. + + Yields: + The next result obtained from applying `func` to each input value. May + contain `None`-s if `ignore_exceptions=True`. May have different order than + the input if `unordered=True`. + + Raises: + WorkerException: If there was an error in the `func` function in a background + process and `ignore_exceptions=False`. + """ + config = get_config( function=func, input_values=input_values, diff --git a/great_ai/utilities/parallel_map/parallel_map_configuration.py b/great_ai/utilities/parallel_map/parallel_map_configuration.py index d8f91d4..6959b94 100644 --- a/great_ai/utilities/parallel_map/parallel_map_configuration.py +++ b/great_ai/utilities/parallel_map/parallel_map_configuration.py @@ -2,7 +2,7 @@ from typing import Optional from pydantic import BaseModel -from ..logger import get_logger +from ..logger.get_logger import get_logger logger = get_logger("parallel_map") diff --git a/great_ai/utilities/parallel_map/simple_parallel_map.py b/great_ai/utilities/parallel_map/simple_parallel_map.py index 3dca6fd..0950a3a 100644 --- a/great_ai/utilities/parallel_map/simple_parallel_map.py +++ b/great_ai/utilities/parallel_map/simple_parallel_map.py @@ -15,6 +15,36 @@ def simple_parallel_map( chunk_size: Optional[int] = None, concurrency: Optional[int] = None, ) -> List[V]: + """Execute a map operation on an list mimicking the API of the built-in `map()`. + + A thin-wrapper over [parallel_map][great_ai.utilities.parallel_map.parallel_map.parallel_map]. + For more options, consult the documentation of + [parallel_map][great_ai.utilities.parallel_map.parallel_map.parallel_map]. + + Examples: + >>> import math + >>> list(simple_parallel_map(math.sqrt, [9, 4, 1])) + [3.0, 2.0, 1.0] + + Args: + func: The function that should be applied to each element of `input_values`. + It can `async`, in that case, a new event loop is started for each chunk. + input_values: An iterable of items that `func` is applied to. + chunk_size: Tune the number of items processed in each step. Larger numbers + result in smaller communication overhead but less parallelism at the start + and end. If `chunk_size` has a `__len__` property, the `chunk_size` is + calculated automatically if not given. + concurrency: Number of new processes to start. Shouldn't be too much more than + the number of physical cores. + + Returns: + An iterable of results obtained from applying `func` to each input value. + + Raises: + WorkerException: If there was an error in the `func` function in a background + process. + """ + input_values = list(input_values) # in case the input is mistakenly not a sequence return list( tqdm( diff --git a/great_ai/utilities/parallel_map/threaded_parallel_map.py b/great_ai/utilities/parallel_map/threaded_parallel_map.py index 9e153b1..faf0b48 100644 --- a/great_ai/utilities/parallel_map/threaded_parallel_map.py +++ b/great_ai/utilities/parallel_map/threaded_parallel_map.py @@ -81,6 +81,47 @@ def threaded_parallel_map( concurrency: Optional[int] = None, unordered: bool = False, ) -> Iterable[Optional[V]]: + """Execute a map operation on an iterable stream. + + Similar to [parallel_map][great_ai.utilities.parallel_map.parallel_map.parallel_map] + but uses threads instead of processes. Hence, it is not helpful in CPU-bound + situations. + + A custom parallel map operation supporting both synchronous and `async` map + functions. Exceptions encountered in the map function are sent to the host thread + where they are either raised (default) or ignored. Each process processes a single + chunk at once. + + Examples: + >>> list(threaded_parallel_map(lambda x: x ** 2, [1, 2, 3])) + [1, 4, 9] + + Args: + func: The function that should be applied to each element of `input_values`. + It can `async`, in that case, a new event loop is started for each chunk. + input_values: An iterable of items that `func` is applied to. + chunk_size: Tune the number of items processed in each step. Larger numbers + result in smaller communication overhead but less parallelism at the start + and end. If `chunk_size` has a `__len__` property, the `chunk_size` is + calculated automatically if not given. + ignore_exceptions: Ignore chunks if `next()` raises an exception on + `input_values`. And return `None` if `func` raised an exception in a worker + process. + concurrency: Number of new threads to start. + unordered: Do not preserve the order of the elements, yield them as soon as they + have been processed. This decreases the latency caused by + difficult-to-process items. + + Yields: + The next result obtained from applying `func` to each input value. May + contain `None`-s if `ignore_exceptions=True`. May have different order than + the input if `unordered=True`. + + Raises: + WorkerException: If there was an error in the `func` function in a background + thread and `ignore_exceptions=False`. + """ + config = get_config( function=func, input_values=input_values, diff --git a/great_ai/utilities/unchunk.py b/great_ai/utilities/unchunk.py index 36d2071..e154808 100644 --- a/great_ai/utilities/unchunk.py +++ b/great_ai/utilities/unchunk.py @@ -4,6 +4,22 @@ T = TypeVar("T") def unchunk(chunks: Iterable[Optional[Iterable[T]]]) -> Iterable[T]: + """Turn a stream of chunks of items into a stream of items (flatten operation). + + The inverse operation of [chunk][great_ai.utilities.chunk.chunk]. + Useful for parallel processing. + + Examples: + >>> list(unchunk([[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]])) + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + + Args: + chunks: Stream of chunks to unpack. + + Yields: + The next item in the flattened iterable. + """ + for chunk in chunks: if chunk is not None: yield from chunk