andras/great-ai
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases 4 Packages Wiki Activity Actions

Fix typing and minor issues

Browse source
This commit is contained in:
Andras Schmelczer 2022-07-07 14:12:44 +02:00
parent 2db2253578
commit 72ab627a34
54 changed files with 635 additions and 589 deletions
Download patch file Download diff file Expand all files Collapse all files

2
great_ai/utilities/chunk.py
View file

@ -3,7 +3,7 @@ from typing import Iterable, List, TypeVar
T = TypeVar("T")
def chunk(values: Iterable[T], chunk_size: int) -> Iterable[T]:
def chunk(values: Iterable[T], chunk_size: int) -> Iterable[List[T]]:
assert chunk_size >= 1
result: List[T] = []

16
great_ai/utilities/config_file/config_file.py
View file

@ -1,6 +1,6 @@
import os
from pathlib import Path
from typing import Dict, Iterable, Tuple, Union
from typing import Dict, ItemsView, Iterator, KeysView, Mapping, Union, ValuesView
from ..logger import get_logger
from .parse_error import ParseError
@ -11,7 +11,7 @@ ENVIRONMENT_VARIABLE_KEY_PREFIX = "ENV"
logger = get_logger("ConfigFile")
class ConfigFile:
class ConfigFile(Mapping[str, str]):
def __init__(self, path: Union[Path, str]) -> None:
if not isinstance(path, Path):
path = Path(path)
@ -24,7 +24,7 @@ class ConfigFile:
self._parse()
def _parse(self):
def _parse(self) -> None:
with open(self._path, encoding="utf-8") as f:
lines: str = f.read()
@ -72,20 +72,20 @@ class ConfigFile:
__getitem__ = __getattr__
def __iter__(self) -> Iterable[Tuple[str, str]]:
def __iter__(self) -> Iterator[str]:
return iter(self._key_values)
def __len__(self) -> int:
return len(self._key_values)
def keys(self):
def keys(self) -> KeysView[str]:
return self._key_values.keys()
def values(self):
def values(self) -> ValuesView[str]:
return self._key_values.values()
def items(self):
def items(self) -> ItemsView[str, str]:
return self._key_values.items()
def __repr__(self):
def __repr__(self) -> str:
return f"{type(self).__name__}(path={self._path}) {self._key_values}"

10
great_ai/utilities/evaluate_ranking/evaluate_ranking.py
View file

@ -1,5 +1,5 @@
from pathlib import Path
from typing import Dict, List, Optional, Union
from typing import Dict, List, Optional, TypeVar
import matplotlib
import matplotlib.pyplot as plt
@ -9,9 +9,11 @@ from sklearn.metrics import average_precision_score, precision_recall_curve
from ..unique import unique
from .draw_f1_iso_lines import draw_f1_iso_lines
T = TypeVar("T", str, float)
def evaluate_ranking(
expected: List[Union[str, float]],
expected: List[T],
actual_scores: List[float],
target_recall: float,
title: Optional[str] = "",
@ -20,7 +22,7 @@ def evaluate_ranking(
output_svg: Optional[Path] = None,
reverse_order: bool = False,
plot: bool = True,
) -> Dict[Union[str, float], float]:
) -> Dict[T, float]:
assert 0 <= target_recall <= 1
if plot and axes is None:
@ -39,7 +41,7 @@ def evaluate_ranking(
draw_f1_iso_lines(axes=ax)
results: Dict[Union[str, float], float] = {}
results: Dict[T, float] = {}
for i in range(len(classes) - 1):
binarized_expected = [
(v < classes[i]) if reverse_order else (v > classes[i])

14
great_ai/utilities/parallel_map/get_config.py
View file

@ -17,14 +17,15 @@ def get_config(
) -> ParallelMapConfiguration:
is_input_sequence = hasattr(input_values, "__len__")
input_length = len(input_values) if is_input_sequence else None # type: ignore
if concurrency is None:
concurrency = os.cpu_count() or 1
assert concurrency >= 1, "At least one mapper process has to be created"
if chunk_size is None:
if is_input_sequence:
chunk_size = max(1, ceil(len(input_values) / concurrency / 10))
if input_length is not None:
chunk_size = max(1, ceil(input_length / concurrency / 10))
else:
raise ValueError(
"The argument for `values` does not implement `__len__`, therefore, you must provide a `chunk_size`"
@ -32,19 +33,14 @@ def get_config(
assert chunk_size >= 1, "Chunks have to contain at least one element"
chunk_count: Optional[int] = None
if is_input_sequence:
chunk_count = ceil(len(input_values) / chunk_size)
if input_length is not None:
chunk_count = ceil(input_length / chunk_size)
if chunk_count < concurrency:
logger.warning(
f"Limiting concurrency to {chunk_count} because there are only {chunk_count} chunks"
)
concurrency = chunk_count
if concurrency == 1:
logger.warning("Running in series, there is no reason for parallelism")
input_length = len(input_values) if is_input_sequence else None
config = ParallelMapConfiguration(
concurrency=concurrency,
chunk_count=chunk_count,

32
great_ai/utilities/parallel_map/manage_communication.py
View file

@ -1,10 +1,11 @@
import multiprocessing as mp
import queue
import traceback
from typing import Dict, Iterable, TypeVar, Union
from typing import Dict, Iterable, List, TypeVar, Union
from ..chunk import chunk
from ..logger import get_logger
from .map_result import MapResult
from .worker_exception import WorkerException
logger = get_logger("parallel_map")
@ -27,6 +28,7 @@ def manage_communication(
next_output_index = 0
read_input_length = 0
is_iteration_over = False
while not is_iteration_over or next_output_index < read_input_length:
if not is_iteration_over:
try:
@ -36,30 +38,30 @@ def manage_communication(
except StopIteration:
is_iteration_over = True
except Exception as e:
if not ignore_exceptions:
raise
else:
if ignore_exceptions:
logger.error(
f"Exception {e} encountered in input, traceback:\n{traceback.format_exc()}"
)
else:
raise
try:
result_chunk = output_queue.get_nowait()
for index, value, exception in result_chunk:
if exception is not None:
e, tb = exception
if not ignore_exceptions:
raise WorkerException from e
else:
result_chunk: List[MapResult] = output_queue.get_nowait()
for r in result_chunk:
if r.exception is not None:
if ignore_exceptions:
logger.error(
f"Exception {e} encountered in worker, traceback:\n{tb}"
f"Exception {r.exception} encountered in worker, traceback:\n{r.worker_traceback}"
)
else:
raise WorkerException from r.exception
if unordered:
yield value
yield r.value
next_output_index += 1
else:
indexed_results[index] = value
indexed_results[r.order] = r.value
if not unordered:
while next_output_index in indexed_results:

36
great_ai/utilities/parallel_map/manage_serial.py
View file

@ -1,36 +0,0 @@
import traceback
from typing import Callable, Iterable, TypeVar
from ..logger import get_logger
from .worker_exception import WorkerException
logger = get_logger("parallel_map")
T = TypeVar("T")
V = TypeVar("V")
def manage_serial(
*,
function: Callable[[T], V],
input_values: Iterable[T],
ignore_exceptions: bool,
) -> Iterable[V]:
try:
for v in input_values:
try:
yield function(v)
except Exception as e:
if not ignore_exceptions:
raise WorkerException from e
else:
logger.error(
f"Exception {e} encountered in worker, traceback:\n{traceback.format_exc()}"
)
except Exception as e:
if not ignore_exceptions:
raise
else:
logger.error(
f"Exception {e} encountered in input, traceback:\n{traceback.format_exc()}"
)

8
great_ai/utilities/parallel_map/map_result.py Normal file
View file

@ -0,0 +1,8 @@
from typing import Any, NamedTuple, Optional
class MapResult(NamedTuple):
order: int
value: Any
exception: Optional[Exception] = None
worker_traceback: Optional[str] = None

63
great_ai/utilities/parallel_map/mapper_function.py
View file

@ -1,42 +1,71 @@
import asyncio
import inspect
import multiprocessing as mp
import queue
import threading
import traceback
from typing import Callable, Union
from multiprocessing.synchronize import Event
from typing import Any, Awaitable, Callable, List, TypeVar, Union, cast
import dill
from .map_result import MapResult
T = TypeVar("T")
V = TypeVar("V")
def mapper_function(
input_queue: Union[mp.Queue, queue.Queue],
output_queue: Union[mp.Queue, queue.Queue],
should_stop: Union[mp.Event, threading.Event],
map_function: Union[bytes, Callable],
should_stop: Union[Event, threading.Event],
func: Union[bytes, Callable[[T], V], Callable[[T], Awaitable[V]]],
) -> None:
try:
if isinstance(map_function, bytes):
map_function = dill.loads(map_function)
if isinstance(func, bytes):
func = cast(Callable[[T], V], dill.loads(func))
last_chunk = None
is_asynchronous = inspect.iscoroutinefunction(func)
last_chunk: List[MapResult] = []
while not should_stop.wait(0.1):
if last_chunk is None:
if not last_chunk:
try:
input_chunk = input_queue.get_nowait()
last_chunk = []
for i, v in input_chunk:
result, exception = None, None
try:
result = map_function(v)
except Exception as e:
exception = e, traceback.format_exc()
last_chunk.append((i, result, exception))
if is_asynchronous:
async def safe(i: int, value: T) -> Any:
try:
return MapResult(
i,
await cast(Callable[[T], Awaitable[V]], func)(
value
),
)
except Exception as e:
return MapResult(i, None, e, traceback.format_exc())
async def main() -> List[MapResult]:
return await asyncio.gather(
*[safe(i, v) for i, v in input_chunk]
)
last_chunk = asyncio.run(main())
else:
for i, value in input_chunk:
try:
last_chunk.append(MapResult(i, func(value)))
except Exception as e:
last_chunk.append(
MapResult(i, None, e, traceback.format_exc())
)
except queue.Empty:
pass
if last_chunk is not None:
if last_chunk:
try:
output_queue.put_nowait(last_chunk)
last_chunk = None
last_chunk = []
except queue.Full:
pass
except (KeyboardInterrupt, BrokenPipeError):

112
great_ai/utilities/parallel_map/parallel_map.py
View file

@ -1,11 +1,20 @@
import multiprocessing as mp
from typing import Callable, Iterable, Literal, Optional, Sequence, TypeVar, overload
from typing import (
Awaitable,
Callable,
Iterable,
Literal,
Optional,
Sequence,
TypeVar,
Union,
overload,
)
import dill
from .get_config import get_config
from .manage_communication import manage_communication
from .manage_serial import manage_serial
from .mapper_function import mapper_function
from .worker_exception import WorkerException
@ -15,79 +24,72 @@ V = TypeVar("V")
@overload
def parallel_map(
function: Callable[[T], V],
func: Callable[[T], Union[V, Awaitable[V]]],
input_values: Sequence[T],
*,
chunk_size: Optional[int],
concurrency: Optional[int],
unordered: Optional[bool],
ignore_exceptions: Optional[Literal[False]],
) -> Iterable[V]:
...
@overload
def parallel_map(
function: Callable[[T], V],
input_values: Iterable[T],
*,
chunk_size: int,
concurrency: Optional[int],
unordered: Optional[bool],
ignore_exceptions: Optional[Literal[False]],
) -> Iterable[V]:
...
@overload
def parallel_map(
function: Callable[[T], V],
input_values: Sequence[T],
*,
chunk_size: Optional[int],
concurrency: Optional[int],
unordered: Optional[bool],
ignore_exceptions: True,
ignore_exceptions: Literal[True],
chunk_size: Optional[int] = ...,
concurrency: Optional[int] = ...,
unordered: bool = ...,
) -> Iterable[Optional[V]]:
...
@overload
def parallel_map(
function: Callable[[T], V],
input_values: Iterable[T],
func: Callable[[T], Union[V, Awaitable[V]]],
input_values: Union[Iterable[T], Sequence[T]],
*,
chunk_size: int,
concurrency: Optional[int],
unordered: Optional[bool],
ignore_exceptions: True,
ignore_exceptions: Literal[True],
concurrency: Optional[int] = ...,
unordered: bool = ...,
) -> Iterable[Optional[V]]:
...
@overload
def parallel_map(
function,
input_values,
func: Callable[[T], Union[V, Awaitable[V]]],
input_values: Sequence[T],
*,
chunk_size=None,
concurrency=None,
unordered=False,
ignore_exceptions=False,
):
chunk_size: Optional[int] = ...,
ignore_exceptions: Literal[False] = ...,
concurrency: Optional[int] = ...,
unordered: bool = ...,
) -> Iterable[V]:
...
@overload
def parallel_map(
func: Callable[[T], Union[V, Awaitable[V]]],
input_values: Union[Iterable[T], Sequence[T]],
*,
chunk_size: int,
ignore_exceptions: Literal[False] = ...,
concurrency: Optional[int] = ...,
unordered: bool = ...,
) -> Iterable[V]:
...
def parallel_map(
func: Callable[[T], Union[V, Awaitable[V]]],
input_values: Union[Iterable[T], Sequence[T]],
*,
chunk_size: Optional[int] = None,
ignore_exceptions: bool = False,
concurrency: Optional[int] = None,
unordered: bool = False,
) -> Iterable[Optional[V]]:
config = get_config(
function=function,
function=func,
input_values=input_values,
chunk_size=chunk_size,
concurrency=concurrency,
)
if config.concurrency == 1:
yield from manage_serial(
function=function,
input_values=input_values,
ignore_exceptions=ignore_exceptions,
)
ctx = (
mp.get_context("fork")
if "fork" in mp.get_all_start_methods()
@ -99,7 +101,7 @@ def parallel_map(
output_queue = manager.Queue(config.concurrency * 2)
should_stop = ctx.Event()
serialized_map_function = dill.dumps(function, byref=True, recurse=False)
serialized_map_function = dill.dumps(func, byref=True, recurse=False)
processes = [
ctx.Process(
@ -110,7 +112,7 @@ def parallel_map(
input_queue=input_queue,
output_queue=output_queue,
should_stop=should_stop,
map_function=serialized_map_function,
func=serialized_map_function,
),
)
for i in range(config.concurrency)

114
great_ai/utilities/parallel_map/threaded_parallel_map.py
View file

@ -1,10 +1,19 @@
import queue
import threading
from typing import Callable, Iterable, Literal, Optional, Sequence, TypeVar, overload
from typing import (
Awaitable,
Callable,
Iterable,
Literal,
Optional,
Sequence,
TypeVar,
Union,
overload,
)
from .get_config import get_config
from .manage_communication import manage_communication
from .manage_serial import manage_serial
from .mapper_function import mapper_function
T = TypeVar("T")
@ -13,81 +22,74 @@ V = TypeVar("V")
@overload
def threaded_parallel_map(
function: Callable[[T], V],
func: Callable[[T], Union[V, Awaitable[V]]],
input_values: Sequence[T],
*,
chunk_size: Optional[int],
concurrency: Optional[int],
unordered: Optional[bool],
ignore_exceptions: Optional[Literal[False]],
) -> Iterable[V]:
...
@overload
def threaded_parallel_map(
function: Callable[[T], V],
input_values: Iterable[T],
*,
chunk_size: int,
concurrency: Optional[int],
unordered: Optional[bool],
ignore_exceptions: Optional[Literal[False]],
) -> Iterable[V]:
...
@overload
def threaded_parallel_map(
function: Callable[[T], V],
input_values: Sequence[T],
*,
chunk_size: Optional[int],
concurrency: Optional[int],
unordered: Optional[bool],
ignore_exceptions: True,
ignore_exceptions: Literal[True],
chunk_size: Optional[int] = ...,
concurrency: Optional[int] = ...,
unordered: bool = ...,
) -> Iterable[Optional[V]]:
...
@overload
def threaded_parallel_map(
function: Callable[[T], V],
input_values: Iterable[T],
func: Callable[[T], Union[V, Awaitable[V]]],
input_values: Union[Iterable[T], Sequence[T]],
*,
chunk_size: int,
concurrency: Optional[int],
unordered: Optional[bool],
ignore_exceptions: True,
ignore_exceptions: Literal[True],
concurrency: Optional[int] = ...,
unordered: bool = ...,
) -> Iterable[Optional[V]]:
...
@overload
def threaded_parallel_map(
function,
input_values,
func: Callable[[T], Union[V, Awaitable[V]]],
input_values: Sequence[T],
*,
chunk_size=None,
concurrency=None,
unordered=False,
ignore_exceptions=False,
):
chunk_size: Optional[int] = ...,
ignore_exceptions: Literal[False] = ...,
concurrency: Optional[int] = ...,
unordered: bool = ...,
) -> Iterable[V]:
...
@overload
def threaded_parallel_map(
func: Callable[[T], Union[V, Awaitable[V]]],
input_values: Union[Iterable[T], Sequence[T]],
*,
chunk_size: int,
ignore_exceptions: Literal[False] = ...,
concurrency: Optional[int] = ...,
unordered: bool = ...,
) -> Iterable[V]:
...
def threaded_parallel_map(
func: Callable[[T], Union[V, Awaitable[V]]],
input_values: Union[Iterable[T], Sequence[T]],
*,
chunk_size: Optional[int] = None,
ignore_exceptions: bool = False,
concurrency: Optional[int] = None,
unordered: bool = False,
) -> Iterable[Optional[V]]:
config = get_config(
function=function,
function=func,
input_values=input_values,
chunk_size=chunk_size,
concurrency=concurrency,
)
if config.concurrency == 1:
yield from manage_serial(
function=function,
input_values=input_values,
ignore_exceptions=ignore_exceptions,
)
input_queue = queue.Queue(config.concurrency * 2)
output_queue = queue.Queue(config.concurrency * 2)
input_queue: queue.Queue = queue.Queue(config.concurrency * 2)
output_queue: queue.Queue = queue.Queue(config.concurrency * 2)
should_stop = threading.Event()
threads = [
@ -99,7 +101,7 @@ def threaded_parallel_map(
input_queue=input_queue,
output_queue=output_queue,
should_stop=should_stop,
map_function=function,
func=func,
),
)
for i in range(config.concurrency)