Try debugging stuck tests
This commit is contained in:
parent
e85edf4222
commit
05fe47e037
2 changed files with 205 additions and 205 deletions
|
|
@ -1,121 +1,121 @@
|
|||
# import queue
|
||||
# import threading
|
||||
# from typing import Callable, Iterable, Literal, Optional, Sequence, TypeVar, overload
|
||||
import queue
|
||||
import threading
|
||||
from typing import Callable, Iterable, Literal, Optional, Sequence, TypeVar, overload
|
||||
|
||||
# from .get_config import get_config
|
||||
# from .manage_communication import manage_communication
|
||||
# from .manage_serial import manage_serial
|
||||
# from .mapper_function import mapper_function
|
||||
from .get_config import get_config
|
||||
from .manage_communication import manage_communication
|
||||
from .manage_serial import manage_serial
|
||||
from .mapper_function import mapper_function
|
||||
|
||||
# T = TypeVar("T")
|
||||
# V = TypeVar("V")
|
||||
T = TypeVar("T")
|
||||
V = TypeVar("V")
|
||||
|
||||
|
||||
# @overload
|
||||
# def threaded_parallel_map(
|
||||
# function: Callable[[T], V],
|
||||
# input_values: Sequence[T],
|
||||
# *,
|
||||
# chunk_size: Optional[int],
|
||||
# concurrency: Optional[int],
|
||||
# unordered: Optional[bool],
|
||||
# ignore_exceptions: Optional[Literal[False]],
|
||||
# ) -> Iterable[V]:
|
||||
# ...
|
||||
@overload
|
||||
def threaded_parallel_map(
|
||||
function: Callable[[T], V],
|
||||
input_values: Sequence[T],
|
||||
*,
|
||||
chunk_size: Optional[int],
|
||||
concurrency: Optional[int],
|
||||
unordered: Optional[bool],
|
||||
ignore_exceptions: Optional[Literal[False]],
|
||||
) -> Iterable[V]:
|
||||
...
|
||||
|
||||
|
||||
# @overload
|
||||
# def threaded_parallel_map(
|
||||
# function: Callable[[T], V],
|
||||
# input_values: Iterable[T],
|
||||
# *,
|
||||
# chunk_size: int,
|
||||
# concurrency: Optional[int],
|
||||
# unordered: Optional[bool],
|
||||
# ignore_exceptions: Optional[Literal[False]],
|
||||
# ) -> Iterable[V]:
|
||||
# ...
|
||||
@overload
|
||||
def threaded_parallel_map(
|
||||
function: Callable[[T], V],
|
||||
input_values: Iterable[T],
|
||||
*,
|
||||
chunk_size: int,
|
||||
concurrency: Optional[int],
|
||||
unordered: Optional[bool],
|
||||
ignore_exceptions: Optional[Literal[False]],
|
||||
) -> Iterable[V]:
|
||||
...
|
||||
|
||||
|
||||
# @overload
|
||||
# def threaded_parallel_map(
|
||||
# function: Callable[[T], V],
|
||||
# input_values: Sequence[T],
|
||||
# *,
|
||||
# chunk_size: Optional[int],
|
||||
# concurrency: Optional[int],
|
||||
# unordered: Optional[bool],
|
||||
# ignore_exceptions: True,
|
||||
# ) -> Iterable[Optional[V]]:
|
||||
# ...
|
||||
@overload
|
||||
def threaded_parallel_map(
|
||||
function: Callable[[T], V],
|
||||
input_values: Sequence[T],
|
||||
*,
|
||||
chunk_size: Optional[int],
|
||||
concurrency: Optional[int],
|
||||
unordered: Optional[bool],
|
||||
ignore_exceptions: True,
|
||||
) -> Iterable[Optional[V]]:
|
||||
...
|
||||
|
||||
|
||||
# @overload
|
||||
# def threaded_parallel_map(
|
||||
# function: Callable[[T], V],
|
||||
# input_values: Iterable[T],
|
||||
# *,
|
||||
# chunk_size: int,
|
||||
# concurrency: Optional[int],
|
||||
# unordered: Optional[bool],
|
||||
# ignore_exceptions: True,
|
||||
# ) -> Iterable[Optional[V]]:
|
||||
# ...
|
||||
@overload
|
||||
def threaded_parallel_map(
|
||||
function: Callable[[T], V],
|
||||
input_values: Iterable[T],
|
||||
*,
|
||||
chunk_size: int,
|
||||
concurrency: Optional[int],
|
||||
unordered: Optional[bool],
|
||||
ignore_exceptions: True,
|
||||
) -> Iterable[Optional[V]]:
|
||||
...
|
||||
|
||||
|
||||
# def threaded_parallel_map(
|
||||
# function,
|
||||
# input_values,
|
||||
# *,
|
||||
# chunk_size=None,
|
||||
# concurrency=None,
|
||||
# unordered=False,
|
||||
# ignore_exceptions=False,
|
||||
# ):
|
||||
# config = get_config(
|
||||
# function=function,
|
||||
# input_values=input_values,
|
||||
# chunk_size=chunk_size,
|
||||
# concurrency=concurrency,
|
||||
# )
|
||||
def threaded_parallel_map(
|
||||
function,
|
||||
input_values,
|
||||
*,
|
||||
chunk_size=None,
|
||||
concurrency=None,
|
||||
unordered=False,
|
||||
ignore_exceptions=False,
|
||||
):
|
||||
config = get_config(
|
||||
function=function,
|
||||
input_values=input_values,
|
||||
chunk_size=chunk_size,
|
||||
concurrency=concurrency,
|
||||
)
|
||||
|
||||
# if config.concurrency == 1:
|
||||
# yield from manage_serial(
|
||||
# function=function,
|
||||
# input_values=input_values,
|
||||
# ignore_exceptions=ignore_exceptions,
|
||||
# )
|
||||
if config.concurrency == 1:
|
||||
yield from manage_serial(
|
||||
function=function,
|
||||
input_values=input_values,
|
||||
ignore_exceptions=ignore_exceptions,
|
||||
)
|
||||
|
||||
# input_queue = queue.Queue(config.concurrency * 2)
|
||||
# output_queue = queue.Queue(config.concurrency * 2)
|
||||
# should_stop = threading.Event()
|
||||
input_queue = queue.Queue(config.concurrency * 2)
|
||||
output_queue = queue.Queue(config.concurrency * 2)
|
||||
should_stop = threading.Event()
|
||||
|
||||
# threads = [
|
||||
# threading.Thread(
|
||||
# name=f"threaded_parallel_map_{config.function_name}_{i}",
|
||||
# target=mapper_function,
|
||||
# daemon=True,
|
||||
# kwargs=dict(
|
||||
# input_queue=input_queue,
|
||||
# output_queue=output_queue,
|
||||
# should_stop=should_stop,
|
||||
# map_function=function,
|
||||
# ),
|
||||
# )
|
||||
# for i in range(config.concurrency)
|
||||
# ]
|
||||
threads = [
|
||||
threading.Thread(
|
||||
name=f"threaded_parallel_map_{config.function_name}_{i}",
|
||||
target=mapper_function,
|
||||
daemon=True,
|
||||
kwargs=dict(
|
||||
input_queue=input_queue,
|
||||
output_queue=output_queue,
|
||||
should_stop=should_stop,
|
||||
map_function=function,
|
||||
),
|
||||
)
|
||||
for i in range(config.concurrency)
|
||||
]
|
||||
|
||||
# for t in threads:
|
||||
# t.start()
|
||||
for t in threads:
|
||||
t.start()
|
||||
|
||||
# yield from manage_communication(
|
||||
# input_values=input_values,
|
||||
# chunk_size=config.chunk_size,
|
||||
# input_queue=input_queue,
|
||||
# output_queue=output_queue,
|
||||
# unordered=unordered,
|
||||
# ignore_exceptions=ignore_exceptions,
|
||||
# )
|
||||
# should_stop.set()
|
||||
# for t in threads:
|
||||
# t.join()
|
||||
yield from manage_communication(
|
||||
input_values=input_values,
|
||||
chunk_size=config.chunk_size,
|
||||
input_queue=input_queue,
|
||||
output_queue=output_queue,
|
||||
unordered=unordered,
|
||||
ignore_exceptions=ignore_exceptions,
|
||||
)
|
||||
should_stop.set()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
|
|
|||
|
|
@ -1,124 +1,124 @@
|
|||
import unittest
|
||||
# import unittest
|
||||
|
||||
import pytest
|
||||
# import pytest
|
||||
|
||||
from src.great_ai.utilities import threaded_parallel_map
|
||||
# from src.great_ai.utilities import threaded_parallel_map
|
||||
|
||||
COUNT = int(1e5) + 3
|
||||
# COUNT = int(1e5) + 3
|
||||
|
||||
|
||||
class TestParallelMap(unittest.TestCase):
|
||||
def test_simple_case_with_progress_bar(self) -> None:
|
||||
assert list(
|
||||
threaded_parallel_map(lambda v: v**2, range(COUNT), concurrency=4)
|
||||
) == [v**2 for v in range(COUNT)]
|
||||
# class TestParallelMap(unittest.TestCase):
|
||||
# def test_simple_case_with_progress_bar(self) -> None:
|
||||
# assert list(
|
||||
# threaded_parallel_map(lambda v: v**2, range(COUNT), concurrency=4)
|
||||
# ) == [v**2 for v in range(COUNT)]
|
||||
|
||||
def test_with_iterable(self) -> None:
|
||||
from time import sleep
|
||||
# def test_with_iterable(self) -> None:
|
||||
# from time import sleep
|
||||
|
||||
def my_generator():
|
||||
for i in range(10):
|
||||
yield i
|
||||
sleep(0.1)
|
||||
# def my_generator():
|
||||
# for i in range(10):
|
||||
# yield i
|
||||
# sleep(0.1)
|
||||
|
||||
expected = [v**3 for v in range(10)]
|
||||
# expected = [v**3 for v in range(10)]
|
||||
|
||||
assert (
|
||||
list(threaded_parallel_map(lambda x: x**3, my_generator(), chunk_size=1))
|
||||
== expected
|
||||
)
|
||||
# assert (
|
||||
# list(threaded_parallel_map(lambda x: x**3, my_generator(), chunk_size=1))
|
||||
# == expected
|
||||
# )
|
||||
|
||||
def test_simple_case_without_progress_bar(self) -> None:
|
||||
assert list(
|
||||
threaded_parallel_map(lambda v: v**2, range(COUNT), concurrency=2)
|
||||
) == [v**2 for v in range(COUNT)]
|
||||
# def test_simple_case_without_progress_bar(self) -> None:
|
||||
# assert list(
|
||||
# threaded_parallel_map(lambda v: v**2, range(COUNT), concurrency=2)
|
||||
# ) == [v**2 for v in range(COUNT)]
|
||||
|
||||
def test_simple_case_invalid_values(self) -> None:
|
||||
with pytest.raises(AssertionError):
|
||||
list(threaded_parallel_map(lambda v: v**2, range(COUNT), concurrency=0))
|
||||
# def test_simple_case_invalid_values(self) -> None:
|
||||
# with pytest.raises(AssertionError):
|
||||
# list(threaded_parallel_map(lambda v: v**2, range(COUNT), concurrency=0))
|
||||
|
||||
with pytest.raises(AssertionError):
|
||||
list(threaded_parallel_map(lambda v: v**2, range(COUNT), chunk_size=0))
|
||||
# with pytest.raises(AssertionError):
|
||||
# list(threaded_parallel_map(lambda v: v**2, range(COUNT), chunk_size=0))
|
||||
|
||||
def test_this_worker_exception(self) -> None:
|
||||
def my_generator():
|
||||
yield 1
|
||||
yield 2
|
||||
yield 3
|
||||
assert False
|
||||
# def test_this_worker_exception(self) -> None:
|
||||
# def my_generator():
|
||||
# yield 1
|
||||
# yield 2
|
||||
# yield 3
|
||||
# assert False
|
||||
|
||||
with pytest.raises(AssertionError):
|
||||
list(
|
||||
threaded_parallel_map(
|
||||
lambda v: v**2, my_generator(), concurrency=2, chunk_size=2
|
||||
)
|
||||
)
|
||||
# with pytest.raises(AssertionError):
|
||||
# list(
|
||||
# threaded_parallel_map(
|
||||
# lambda v: v**2, my_generator(), concurrency=2, chunk_size=2
|
||||
# )
|
||||
# )
|
||||
|
||||
with pytest.raises(AssertionError):
|
||||
list(
|
||||
threaded_parallel_map(
|
||||
lambda v: v**2, my_generator(), concurrency=1, chunk_size=2
|
||||
)
|
||||
)
|
||||
# with pytest.raises(AssertionError):
|
||||
# list(
|
||||
# threaded_parallel_map(
|
||||
# lambda v: v**2, my_generator(), concurrency=1, chunk_size=2
|
||||
# )
|
||||
# )
|
||||
|
||||
def test_ignore_this_worker_exception(self) -> None:
|
||||
def my_generator():
|
||||
yield 1
|
||||
yield 2
|
||||
yield 3
|
||||
yield 1 / 0
|
||||
# def test_ignore_this_worker_exception(self) -> None:
|
||||
# def my_generator():
|
||||
# yield 1
|
||||
# yield 2
|
||||
# yield 3
|
||||
# yield 1 / 0
|
||||
|
||||
assert list(
|
||||
threaded_parallel_map(
|
||||
lambda v: v**2,
|
||||
my_generator(),
|
||||
concurrency=2,
|
||||
chunk_size=2,
|
||||
ignore_exceptions=True,
|
||||
)
|
||||
) == [1, 4]
|
||||
assert list(
|
||||
threaded_parallel_map(
|
||||
lambda v: v**2,
|
||||
my_generator(),
|
||||
concurrency=1,
|
||||
chunk_size=2,
|
||||
ignore_exceptions=True,
|
||||
)
|
||||
) == [1, 4, 9]
|
||||
# assert list(
|
||||
# threaded_parallel_map(
|
||||
# lambda v: v**2,
|
||||
# my_generator(),
|
||||
# concurrency=2,
|
||||
# chunk_size=2,
|
||||
# ignore_exceptions=True,
|
||||
# )
|
||||
# ) == [1, 4]
|
||||
# assert list(
|
||||
# threaded_parallel_map(
|
||||
# lambda v: v**2,
|
||||
# my_generator(),
|
||||
# concurrency=1,
|
||||
# chunk_size=2,
|
||||
# ignore_exceptions=True,
|
||||
# )
|
||||
# ) == [1, 4, 9]
|
||||
|
||||
def test_worker_worker_exception(self) -> None:
|
||||
def oh_no(_):
|
||||
raise ValueError("hi")
|
||||
# def test_worker_worker_exception(self) -> None:
|
||||
# def oh_no(_):
|
||||
# raise ValueError("hi")
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
list(threaded_parallel_map(oh_no, range(COUNT), concurrency=2))
|
||||
# with pytest.raises(ValueError):
|
||||
# list(threaded_parallel_map(oh_no, range(COUNT), concurrency=2))
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
list(threaded_parallel_map(oh_no, range(COUNT), concurrency=1))
|
||||
# with pytest.raises(ValueError):
|
||||
# list(threaded_parallel_map(oh_no, range(COUNT), concurrency=1))
|
||||
|
||||
def test_ignore_worker_worker_exception(self) -> None:
|
||||
def oh_no(_):
|
||||
raise ValueError("hi")
|
||||
# def test_ignore_worker_worker_exception(self) -> None:
|
||||
# def oh_no(_):
|
||||
# raise ValueError("hi")
|
||||
|
||||
assert (
|
||||
list(
|
||||
threaded_parallel_map(
|
||||
oh_no, range(3), concurrency=2, ignore_exceptions=True
|
||||
)
|
||||
)
|
||||
== [None] * 3
|
||||
)
|
||||
assert (
|
||||
list(
|
||||
threaded_parallel_map(
|
||||
oh_no, range(3), concurrency=1, ignore_exceptions=True
|
||||
)
|
||||
)
|
||||
== [None] * 3
|
||||
)
|
||||
# assert (
|
||||
# list(
|
||||
# threaded_parallel_map(
|
||||
# oh_no, range(3), concurrency=2, ignore_exceptions=True
|
||||
# )
|
||||
# )
|
||||
# == [None] * 3
|
||||
# )
|
||||
# assert (
|
||||
# list(
|
||||
# threaded_parallel_map(
|
||||
# oh_no, range(3), concurrency=1, ignore_exceptions=True
|
||||
# )
|
||||
# )
|
||||
# == [None] * 3
|
||||
# )
|
||||
|
||||
def test_no_op(self) -> None:
|
||||
assert list(threaded_parallel_map(lambda v: v**2, [])) == []
|
||||
assert list(threaded_parallel_map(lambda v: v**2, [], chunk_size=100)) == []
|
||||
assert list(threaded_parallel_map(lambda v: v**2, [], concurrency=100)) == []
|
||||
# def test_no_op(self) -> None:
|
||||
# assert list(threaded_parallel_map(lambda v: v**2, [])) == []
|
||||
# assert list(threaded_parallel_map(lambda v: v**2, [], chunk_size=100)) == []
|
||||
# assert list(threaded_parallel_map(lambda v: v**2, [], concurrency=100)) == []
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue