Document utilities

great_ai/utilities/parallel_map/__init__.py

@@ -1,4 +0,0 @@
-from .parallel_map import parallel_map
-from .simple_parallel_map import simple_parallel_map
-from .threaded_parallel_map import threaded_parallel_map
-from .worker_exception import WorkerException

great_ai/utilities/parallel_map/get_config.py

@@ -2,7 +2,7 @@ import os
 from math import ceil
 from typing import Callable, Iterable, Optional, Sequence, Union
 
-from ..logger import get_logger
+from ..logger.get_logger import get_logger
 from .parallel_map_configuration import ParallelMapConfiguration
 
 logger = get_logger("parallel_map")

great_ai/utilities/parallel_map/manage_communication.py

@@ -4,7 +4,7 @@ import traceback
 from typing import Dict, Iterable, List, TypeVar, Union
 
 from ..chunk import chunk
-from ..logger import get_logger
+from ..logger.get_logger import get_logger
 from .map_result import MapResult
 from .worker_exception import WorkerException
 

great_ai/utilities/parallel_map/parallel_map.py

@@ -83,6 +83,49 @@ def parallel_map(
     concurrency: Optional[int] = None,
     unordered: bool = False,
 ) -> Iterable[Optional[V]]:
+    """Execute a map operation on an iterable stream.
+
+    A custom parallel map operation supporting both synchronous and `async` map
+    functions. The `func` function is serialised with `dill`. Exceptions encountered in
+    the map function are sent to the host process where they are either raised (default)
+    or ignored.
+
+    The new processes are forked if the OS allows it, otherwise, new Python processes
+    are bootstrapped which can incur some startup cost. Each process processes a single
+    chunk at once.
+
+    Examples:
+        >>> import math
+        >>> list(parallel_map(math.sqrt, [9, 4, 1], concurrency=2))
+        [3.0, 2.0, 1.0]
+
+    Args:
+        func: The function that should be applied to each element of `input_values`.
+            It can `async`, in that case, a new event loop is started for each chunk.
+        input_values: An iterable of items that `func` is applied to.
+        chunk_size: Tune the number of items processed in each step. Larger numbers
+            result in smaller communication overhead but less parallelism at the start
+            and end. If `chunk_size` has a `__len__` property, the `chunk_size` is
+            calculated automatically if not given.
+        ignore_exceptions: Ignore chunks if `next()` raises an exception on
+            `input_values`. And return `None` if `func` raised an exception in a worker
+            process.
+        concurrency: Number of new processes to start. Shouldn't be too much more than
+            the number of physical cores.
+        unordered: Do not preserve the order of the elements, yield them as soon as they
+            have been processed. This decreases the latency caused by
+            difficult-to-process items.
+
+    Yields:
+        The next result obtained from applying `func` to each input value. May
+            contain `None`-s if `ignore_exceptions=True`. May have different order than
+            the input if `unordered=True`.
+
+    Raises:
+        WorkerException: If there was an error in the `func` function in a background
+            process and `ignore_exceptions=False`.
+    """
+
     config = get_config(
         function=func,
         input_values=input_values,

great_ai/utilities/parallel_map/parallel_map_configuration.py

@@ -2,7 +2,7 @@ from typing import Optional
 
 from pydantic import BaseModel
 
-from ..logger import get_logger
+from ..logger.get_logger import get_logger
 
 logger = get_logger("parallel_map")
 

great_ai/utilities/parallel_map/simple_parallel_map.py

@@ -15,6 +15,36 @@ def simple_parallel_map(
     chunk_size: Optional[int] = None,
     concurrency: Optional[int] = None,
 ) -> List[V]:
+    """Execute a map operation on an list mimicking the API of the built-in `map()`.
+
+    A thin-wrapper over [parallel_map][great_ai.utilities.parallel_map.parallel_map.parallel_map].
+    For more options, consult the documentation of
+    [parallel_map][great_ai.utilities.parallel_map.parallel_map.parallel_map].
+
+    Examples:
+        >>> import math
+        >>> list(simple_parallel_map(math.sqrt, [9, 4, 1]))
+        [3.0, 2.0, 1.0]
+
+    Args:
+        func: The function that should be applied to each element of `input_values`.
+            It can `async`, in that case, a new event loop is started for each chunk.
+        input_values: An iterable of items that `func` is applied to.
+        chunk_size: Tune the number of items processed in each step. Larger numbers
+            result in smaller communication overhead but less parallelism at the start
+            and end. If `chunk_size` has a `__len__` property, the `chunk_size` is
+            calculated automatically if not given.
+        concurrency: Number of new processes to start. Shouldn't be too much more than
+            the number of physical cores.
+
+    Returns:
+        An iterable of results obtained from applying `func` to each input value.
+
+    Raises:
+        WorkerException: If there was an error in the `func` function in a background
+            process.
+    """
+
     input_values = list(input_values)  # in case the input is mistakenly not a sequence
     return list(
         tqdm(

great_ai/utilities/parallel_map/threaded_parallel_map.py

@@ -81,6 +81,47 @@ def threaded_parallel_map(
     concurrency: Optional[int] = None,
     unordered: bool = False,
 ) -> Iterable[Optional[V]]:
+    """Execute a map operation on an iterable stream.
+
+    Similar to [parallel_map][great_ai.utilities.parallel_map.parallel_map.parallel_map]
+    but uses threads instead of processes. Hence, it is not helpful in CPU-bound
+    situations.
+
+    A custom parallel map operation supporting both synchronous and `async` map
+    functions. Exceptions encountered in the map function are sent to the host thread
+    where they are either raised (default) or ignored. Each process processes a single
+    chunk at once.
+
+    Examples:
+        >>> list(threaded_parallel_map(lambda x: x ** 2, [1, 2, 3]))
+        [1, 4, 9]
+
+    Args:
+        func: The function that should be applied to each element of `input_values`.
+            It can `async`, in that case, a new event loop is started for each chunk.
+        input_values: An iterable of items that `func` is applied to.
+        chunk_size: Tune the number of items processed in each step. Larger numbers
+            result in smaller communication overhead but less parallelism at the start
+            and end. If `chunk_size` has a `__len__` property, the `chunk_size` is
+            calculated automatically if not given.
+        ignore_exceptions: Ignore chunks if `next()` raises an exception on
+            `input_values`. And return `None` if `func` raised an exception in a worker
+            process.
+        concurrency: Number of new threads to start.
+        unordered: Do not preserve the order of the elements, yield them as soon as they
+            have been processed. This decreases the latency caused by
+            difficult-to-process items.
+
+    Yields:
+        The next result obtained from applying `func` to each input value. May
+            contain `None`-s if `ignore_exceptions=True`. May have different order than
+            the input if `unordered=True`.
+
+    Raises:
+        WorkerException: If there was an error in the `func` function in a background
+            thread and `ignore_exceptions=False`.
+    """
+
     config = get_config(
         function=func,
         input_values=input_values,