andras/great-ai
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases 4 Packages Wiki Activity Actions

Add error handling to parallel map

Browse source
This commit is contained in:
Andras Schmelczer 2022-07-01 22:43:08 +02:00
parent cbe843220a
commit a40f456e39
7 changed files with 361 additions and 135 deletions
Download patch file Download diff file Expand all files Collapse all files

106
tests/utilities/test_threaded_parallel_map.py
View file

@ -7,15 +7,11 @@ from src.great_ai.utilities import threaded_parallel_map
COUNT = int(1e5) + 3
class TestThreadedParallelMap(unittest.TestCase):
class TestParallelMap(unittest.TestCase):
def test_simple_case_with_progress_bar(self) -> None:
inputs = range(COUNT)
expected = [v**2 for v in range(COUNT)]
assert (
list(threaded_parallel_map(lambda v: v**2, inputs, concurrency=10))
== expected
)
assert list(
threaded_parallel_map(lambda v: v**2, range(COUNT), concurrency=4)
) == [v**2 for v in range(COUNT)]
def test_with_iterable(self) -> None:
from time import sleep
@ -33,22 +29,96 @@ class TestThreadedParallelMap(unittest.TestCase):
)
def test_simple_case_without_progress_bar(self) -> None:
inputs = range(COUNT)
expected = [v**2 for v in range(COUNT)]
assert (
list(threaded_parallel_map(lambda v: v**2, inputs, disable_logging=True))
== expected
)
assert list(
threaded_parallel_map(
lambda v: v**2, range(COUNT), disable_logging=True, concurrency=2
)
) == [v**2 for v in range(COUNT)]
def test_simple_case_invalid_values(self) -> None:
inputs = range(COUNT)
with pytest.raises(AssertionError):
list(threaded_parallel_map(lambda v: v**2, range(COUNT), concurrency=0))
with pytest.raises(AssertionError):
list(threaded_parallel_map(lambda v: v**2, inputs, concurrency=0))
list(threaded_parallel_map(lambda v: v**2, range(COUNT), chunk_size=0))
def test_this_worker_exception(self) -> None:
def my_generator():
yield 1
yield 2
yield 3
assert False
with pytest.raises(AssertionError):
list(threaded_parallel_map(lambda v: v**2, inputs, chunk_size=0))
list(
threaded_parallel_map(
lambda v: v**2, my_generator(), concurrency=2, chunk_size=2
)
)
with pytest.raises(AssertionError):
list(
threaded_parallel_map(
lambda v: v**2, my_generator(), concurrency=1, chunk_size=2
)
)
def test_ignore_this_worker_exception(self) -> None:
def my_generator():
yield 1
yield 2
yield 3
yield 1 / 0
assert list(
threaded_parallel_map(
lambda v: v**2,
my_generator(),
concurrency=2,
chunk_size=2,
ignore_exceptions=True,
)
) == [1, 4]
assert list(
threaded_parallel_map(
lambda v: v**2,
my_generator(),
concurrency=1,
chunk_size=2,
ignore_exceptions=True,
)
) == [1, 4, 9]
def test_worker_worker_exception(self) -> None:
def oh_no(_):
raise ValueError("hi")
with pytest.raises(ValueError):
list(threaded_parallel_map(oh_no, range(COUNT), concurrency=2))
with pytest.raises(ValueError):
list(threaded_parallel_map(oh_no, range(COUNT), concurrency=1))
def test_ignore_worker_worker_exception(self) -> None:
def oh_no(_):
raise ValueError("hi")
assert (
list(
threaded_parallel_map(
oh_no, range(3), concurrency=2, ignore_exceptions=True
)
)
== [None] * 3
)
assert (
list(
threaded_parallel_map(
oh_no, range(3), concurrency=1, ignore_exceptions=True
)
)
== [None] * 3
)
def test_no_op(self) -> None:
assert (