diff --git a/src/great_ai/utilities/parallel_map/threaded_parallel_map.py b/src/great_ai/utilities/parallel_map/threaded_parallel_map.py index f043926..53fcab2 100644 --- a/src/great_ai/utilities/parallel_map/threaded_parallel_map.py +++ b/src/great_ai/utilities/parallel_map/threaded_parallel_map.py @@ -1,121 +1,121 @@ -# import queue -# import threading -# from typing import Callable, Iterable, Literal, Optional, Sequence, TypeVar, overload +import queue +import threading +from typing import Callable, Iterable, Literal, Optional, Sequence, TypeVar, overload -# from .get_config import get_config -# from .manage_communication import manage_communication -# from .manage_serial import manage_serial -# from .mapper_function import mapper_function +from .get_config import get_config +from .manage_communication import manage_communication +from .manage_serial import manage_serial +from .mapper_function import mapper_function -# T = TypeVar("T") -# V = TypeVar("V") +T = TypeVar("T") +V = TypeVar("V") -# @overload -# def threaded_parallel_map( -# function: Callable[[T], V], -# input_values: Sequence[T], -# *, -# chunk_size: Optional[int], -# concurrency: Optional[int], -# unordered: Optional[bool], -# ignore_exceptions: Optional[Literal[False]], -# ) -> Iterable[V]: -# ... +@overload +def threaded_parallel_map( + function: Callable[[T], V], + input_values: Sequence[T], + *, + chunk_size: Optional[int], + concurrency: Optional[int], + unordered: Optional[bool], + ignore_exceptions: Optional[Literal[False]], +) -> Iterable[V]: + ... -# @overload -# def threaded_parallel_map( -# function: Callable[[T], V], -# input_values: Iterable[T], -# *, -# chunk_size: int, -# concurrency: Optional[int], -# unordered: Optional[bool], -# ignore_exceptions: Optional[Literal[False]], -# ) -> Iterable[V]: -# ... +@overload +def threaded_parallel_map( + function: Callable[[T], V], + input_values: Iterable[T], + *, + chunk_size: int, + concurrency: Optional[int], + unordered: Optional[bool], + ignore_exceptions: Optional[Literal[False]], +) -> Iterable[V]: + ... -# @overload -# def threaded_parallel_map( -# function: Callable[[T], V], -# input_values: Sequence[T], -# *, -# chunk_size: Optional[int], -# concurrency: Optional[int], -# unordered: Optional[bool], -# ignore_exceptions: True, -# ) -> Iterable[Optional[V]]: -# ... +@overload +def threaded_parallel_map( + function: Callable[[T], V], + input_values: Sequence[T], + *, + chunk_size: Optional[int], + concurrency: Optional[int], + unordered: Optional[bool], + ignore_exceptions: True, +) -> Iterable[Optional[V]]: + ... -# @overload -# def threaded_parallel_map( -# function: Callable[[T], V], -# input_values: Iterable[T], -# *, -# chunk_size: int, -# concurrency: Optional[int], -# unordered: Optional[bool], -# ignore_exceptions: True, -# ) -> Iterable[Optional[V]]: -# ... +@overload +def threaded_parallel_map( + function: Callable[[T], V], + input_values: Iterable[T], + *, + chunk_size: int, + concurrency: Optional[int], + unordered: Optional[bool], + ignore_exceptions: True, +) -> Iterable[Optional[V]]: + ... -# def threaded_parallel_map( -# function, -# input_values, -# *, -# chunk_size=None, -# concurrency=None, -# unordered=False, -# ignore_exceptions=False, -# ): -# config = get_config( -# function=function, -# input_values=input_values, -# chunk_size=chunk_size, -# concurrency=concurrency, -# ) +def threaded_parallel_map( + function, + input_values, + *, + chunk_size=None, + concurrency=None, + unordered=False, + ignore_exceptions=False, +): + config = get_config( + function=function, + input_values=input_values, + chunk_size=chunk_size, + concurrency=concurrency, + ) -# if config.concurrency == 1: -# yield from manage_serial( -# function=function, -# input_values=input_values, -# ignore_exceptions=ignore_exceptions, -# ) + if config.concurrency == 1: + yield from manage_serial( + function=function, + input_values=input_values, + ignore_exceptions=ignore_exceptions, + ) -# input_queue = queue.Queue(config.concurrency * 2) -# output_queue = queue.Queue(config.concurrency * 2) -# should_stop = threading.Event() + input_queue = queue.Queue(config.concurrency * 2) + output_queue = queue.Queue(config.concurrency * 2) + should_stop = threading.Event() -# threads = [ -# threading.Thread( -# name=f"threaded_parallel_map_{config.function_name}_{i}", -# target=mapper_function, -# daemon=True, -# kwargs=dict( -# input_queue=input_queue, -# output_queue=output_queue, -# should_stop=should_stop, -# map_function=function, -# ), -# ) -# for i in range(config.concurrency) -# ] + threads = [ + threading.Thread( + name=f"threaded_parallel_map_{config.function_name}_{i}", + target=mapper_function, + daemon=True, + kwargs=dict( + input_queue=input_queue, + output_queue=output_queue, + should_stop=should_stop, + map_function=function, + ), + ) + for i in range(config.concurrency) + ] -# for t in threads: -# t.start() + for t in threads: + t.start() -# yield from manage_communication( -# input_values=input_values, -# chunk_size=config.chunk_size, -# input_queue=input_queue, -# output_queue=output_queue, -# unordered=unordered, -# ignore_exceptions=ignore_exceptions, -# ) -# should_stop.set() -# for t in threads: -# t.join() + yield from manage_communication( + input_values=input_values, + chunk_size=config.chunk_size, + input_queue=input_queue, + output_queue=output_queue, + unordered=unordered, + ignore_exceptions=ignore_exceptions, + ) + should_stop.set() + for t in threads: + t.join() diff --git a/tests/utilities/test_threaded_parallel_map.py b/tests/utilities/test_threaded_parallel_map.py index 0bf84a0..93811d4 100644 --- a/tests/utilities/test_threaded_parallel_map.py +++ b/tests/utilities/test_threaded_parallel_map.py @@ -1,124 +1,124 @@ -import unittest +# import unittest -import pytest +# import pytest -from src.great_ai.utilities import threaded_parallel_map +# from src.great_ai.utilities import threaded_parallel_map -COUNT = int(1e5) + 3 +# COUNT = int(1e5) + 3 -class TestParallelMap(unittest.TestCase): - def test_simple_case_with_progress_bar(self) -> None: - assert list( - threaded_parallel_map(lambda v: v**2, range(COUNT), concurrency=4) - ) == [v**2 for v in range(COUNT)] +# class TestParallelMap(unittest.TestCase): +# def test_simple_case_with_progress_bar(self) -> None: +# assert list( +# threaded_parallel_map(lambda v: v**2, range(COUNT), concurrency=4) +# ) == [v**2 for v in range(COUNT)] - def test_with_iterable(self) -> None: - from time import sleep +# def test_with_iterable(self) -> None: +# from time import sleep - def my_generator(): - for i in range(10): - yield i - sleep(0.1) +# def my_generator(): +# for i in range(10): +# yield i +# sleep(0.1) - expected = [v**3 for v in range(10)] +# expected = [v**3 for v in range(10)] - assert ( - list(threaded_parallel_map(lambda x: x**3, my_generator(), chunk_size=1)) - == expected - ) +# assert ( +# list(threaded_parallel_map(lambda x: x**3, my_generator(), chunk_size=1)) +# == expected +# ) - def test_simple_case_without_progress_bar(self) -> None: - assert list( - threaded_parallel_map(lambda v: v**2, range(COUNT), concurrency=2) - ) == [v**2 for v in range(COUNT)] +# def test_simple_case_without_progress_bar(self) -> None: +# assert list( +# threaded_parallel_map(lambda v: v**2, range(COUNT), concurrency=2) +# ) == [v**2 for v in range(COUNT)] - def test_simple_case_invalid_values(self) -> None: - with pytest.raises(AssertionError): - list(threaded_parallel_map(lambda v: v**2, range(COUNT), concurrency=0)) +# def test_simple_case_invalid_values(self) -> None: +# with pytest.raises(AssertionError): +# list(threaded_parallel_map(lambda v: v**2, range(COUNT), concurrency=0)) - with pytest.raises(AssertionError): - list(threaded_parallel_map(lambda v: v**2, range(COUNT), chunk_size=0)) +# with pytest.raises(AssertionError): +# list(threaded_parallel_map(lambda v: v**2, range(COUNT), chunk_size=0)) - def test_this_worker_exception(self) -> None: - def my_generator(): - yield 1 - yield 2 - yield 3 - assert False +# def test_this_worker_exception(self) -> None: +# def my_generator(): +# yield 1 +# yield 2 +# yield 3 +# assert False - with pytest.raises(AssertionError): - list( - threaded_parallel_map( - lambda v: v**2, my_generator(), concurrency=2, chunk_size=2 - ) - ) +# with pytest.raises(AssertionError): +# list( +# threaded_parallel_map( +# lambda v: v**2, my_generator(), concurrency=2, chunk_size=2 +# ) +# ) - with pytest.raises(AssertionError): - list( - threaded_parallel_map( - lambda v: v**2, my_generator(), concurrency=1, chunk_size=2 - ) - ) +# with pytest.raises(AssertionError): +# list( +# threaded_parallel_map( +# lambda v: v**2, my_generator(), concurrency=1, chunk_size=2 +# ) +# ) - def test_ignore_this_worker_exception(self) -> None: - def my_generator(): - yield 1 - yield 2 - yield 3 - yield 1 / 0 +# def test_ignore_this_worker_exception(self) -> None: +# def my_generator(): +# yield 1 +# yield 2 +# yield 3 +# yield 1 / 0 - assert list( - threaded_parallel_map( - lambda v: v**2, - my_generator(), - concurrency=2, - chunk_size=2, - ignore_exceptions=True, - ) - ) == [1, 4] - assert list( - threaded_parallel_map( - lambda v: v**2, - my_generator(), - concurrency=1, - chunk_size=2, - ignore_exceptions=True, - ) - ) == [1, 4, 9] +# assert list( +# threaded_parallel_map( +# lambda v: v**2, +# my_generator(), +# concurrency=2, +# chunk_size=2, +# ignore_exceptions=True, +# ) +# ) == [1, 4] +# assert list( +# threaded_parallel_map( +# lambda v: v**2, +# my_generator(), +# concurrency=1, +# chunk_size=2, +# ignore_exceptions=True, +# ) +# ) == [1, 4, 9] - def test_worker_worker_exception(self) -> None: - def oh_no(_): - raise ValueError("hi") +# def test_worker_worker_exception(self) -> None: +# def oh_no(_): +# raise ValueError("hi") - with pytest.raises(ValueError): - list(threaded_parallel_map(oh_no, range(COUNT), concurrency=2)) +# with pytest.raises(ValueError): +# list(threaded_parallel_map(oh_no, range(COUNT), concurrency=2)) - with pytest.raises(ValueError): - list(threaded_parallel_map(oh_no, range(COUNT), concurrency=1)) +# with pytest.raises(ValueError): +# list(threaded_parallel_map(oh_no, range(COUNT), concurrency=1)) - def test_ignore_worker_worker_exception(self) -> None: - def oh_no(_): - raise ValueError("hi") +# def test_ignore_worker_worker_exception(self) -> None: +# def oh_no(_): +# raise ValueError("hi") - assert ( - list( - threaded_parallel_map( - oh_no, range(3), concurrency=2, ignore_exceptions=True - ) - ) - == [None] * 3 - ) - assert ( - list( - threaded_parallel_map( - oh_no, range(3), concurrency=1, ignore_exceptions=True - ) - ) - == [None] * 3 - ) +# assert ( +# list( +# threaded_parallel_map( +# oh_no, range(3), concurrency=2, ignore_exceptions=True +# ) +# ) +# == [None] * 3 +# ) +# assert ( +# list( +# threaded_parallel_map( +# oh_no, range(3), concurrency=1, ignore_exceptions=True +# ) +# ) +# == [None] * 3 +# ) - def test_no_op(self) -> None: - assert list(threaded_parallel_map(lambda v: v**2, [])) == [] - assert list(threaded_parallel_map(lambda v: v**2, [], chunk_size=100)) == [] - assert list(threaded_parallel_map(lambda v: v**2, [], concurrency=100)) == [] +# def test_no_op(self) -> None: +# assert list(threaded_parallel_map(lambda v: v**2, [])) == [] +# assert list(threaded_parallel_map(lambda v: v**2, [], chunk_size=100)) == [] +# assert list(threaded_parallel_map(lambda v: v**2, [], concurrency=100)) == []