From bbe0b978b598c50ec003382ff755c263ed4f8ea6 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 3 Jul 2022 13:32:42 +0200 Subject: [PATCH] Try debugging hanging --- tests/utilities/test_parallel_map.py | 186 ++++++++-------- tests/utilities/test_threaded_parallel_map.py | 202 +++++++++--------- 2 files changed, 194 insertions(+), 194 deletions(-) diff --git a/tests/utilities/test_parallel_map.py b/tests/utilities/test_parallel_map.py index 07a14c8..8bbc71c 100644 --- a/tests/utilities/test_parallel_map.py +++ b/tests/utilities/test_parallel_map.py @@ -1,116 +1,116 @@ -import unittest +# import unittest -import pytest +# import pytest -from src.great_ai.utilities import parallel_map +# from src.great_ai.utilities import parallel_map -COUNT = int(1e5) + 3 +# COUNT = int(1e5) + 3 -class TestParallelMap(unittest.TestCase): - def test_simple_case_with_progress_bar(self) -> None: - assert list(parallel_map(lambda v: v**2, range(COUNT), concurrency=4)) == [ - v**2 for v in range(COUNT) - ] +# class TestParallelMap(unittest.TestCase): +# def test_simple_case_with_progress_bar(self) -> None: +# assert list(parallel_map(lambda v: v**2, range(COUNT), concurrency=4)) == [ +# v**2 for v in range(COUNT) +# ] - def test_with_iterable(self) -> None: - from time import sleep +# def test_with_iterable(self) -> None: +# from time import sleep - def my_generator(): - for i in range(10): - yield i - sleep(0.1) +# def my_generator(): +# for i in range(10): +# yield i +# sleep(0.1) - expected = [v**3 for v in range(10)] +# expected = [v**3 for v in range(10)] - assert ( - list(parallel_map(lambda x: x**3, my_generator(), chunk_size=1)) - == expected - ) +# assert ( +# list(parallel_map(lambda x: x**3, my_generator(), chunk_size=1)) +# == expected +# ) - def test_simple_case_without_progress_bar(self) -> None: - assert list(parallel_map(lambda v: v**2, range(COUNT), concurrency=2)) == [ - v**2 for v in range(COUNT) - ] +# def test_simple_case_without_progress_bar(self) -> None: +# assert list(parallel_map(lambda v: v**2, range(COUNT), concurrency=2)) == [ +# v**2 for v in range(COUNT) +# ] - def test_simple_case_invalid_values(self) -> None: - with pytest.raises(AssertionError): - list(parallel_map(lambda v: v**2, range(COUNT), concurrency=0)) +# def test_simple_case_invalid_values(self) -> None: +# with pytest.raises(AssertionError): +# list(parallel_map(lambda v: v**2, range(COUNT), concurrency=0)) - with pytest.raises(AssertionError): - list(parallel_map(lambda v: v**2, range(COUNT), chunk_size=0)) +# with pytest.raises(AssertionError): +# list(parallel_map(lambda v: v**2, range(COUNT), chunk_size=0)) - def test_this_process_exception(self) -> None: - def my_generator(): - yield 1 - yield 2 - yield 3 - assert False +# def test_this_process_exception(self) -> None: +# def my_generator(): +# yield 1 +# yield 2 +# yield 3 +# assert False - with pytest.raises(AssertionError): - list( - parallel_map( - lambda v: v**2, my_generator(), concurrency=2, chunk_size=2 - ) - ) +# with pytest.raises(AssertionError): +# list( +# parallel_map( +# lambda v: v**2, my_generator(), concurrency=2, chunk_size=2 +# ) +# ) - with pytest.raises(AssertionError): - list( - parallel_map( - lambda v: v**2, my_generator(), concurrency=1, chunk_size=2 - ) - ) +# with pytest.raises(AssertionError): +# list( +# parallel_map( +# lambda v: v**2, my_generator(), concurrency=1, chunk_size=2 +# ) +# ) - def test_ignore_this_process_exception(self) -> None: - def my_generator(): - yield 1 - yield 2 - yield 3 - yield 1 / 0 +# def test_ignore_this_process_exception(self) -> None: +# def my_generator(): +# yield 1 +# yield 2 +# yield 3 +# yield 1 / 0 - assert list( - parallel_map( - lambda v: v**2, - my_generator(), - concurrency=2, - chunk_size=2, - ignore_exceptions=True, - ) - ) == [1, 4] - assert list( - parallel_map( - lambda v: v**2, - my_generator(), - concurrency=1, - chunk_size=2, - ignore_exceptions=True, - ) - ) == [1, 4, 9] +# assert list( +# parallel_map( +# lambda v: v**2, +# my_generator(), +# concurrency=2, +# chunk_size=2, +# ignore_exceptions=True, +# ) +# ) == [1, 4] +# assert list( +# parallel_map( +# lambda v: v**2, +# my_generator(), +# concurrency=1, +# chunk_size=2, +# ignore_exceptions=True, +# ) +# ) == [1, 4, 9] - def test_worker_process_exception(self) -> None: - def oh_no(_): - raise ValueError("hi") +# def test_worker_process_exception(self) -> None: +# def oh_no(_): +# raise ValueError("hi") - with pytest.raises(ValueError): - list(parallel_map(oh_no, range(COUNT), concurrency=2)) +# with pytest.raises(ValueError): +# list(parallel_map(oh_no, range(COUNT), concurrency=2)) - with pytest.raises(ValueError): - list(parallel_map(oh_no, range(COUNT), concurrency=1)) +# with pytest.raises(ValueError): +# list(parallel_map(oh_no, range(COUNT), concurrency=1)) - def test_ignore_worker_process_exception(self) -> None: - def oh_no(_): - raise ValueError("hi") +# def test_ignore_worker_process_exception(self) -> None: +# def oh_no(_): +# raise ValueError("hi") - assert ( - list(parallel_map(oh_no, range(3), concurrency=2, ignore_exceptions=True)) - == [None] * 3 - ) - assert ( - list(parallel_map(oh_no, range(3), concurrency=1, ignore_exceptions=True)) - == [None] * 3 - ) +# assert ( +# list(parallel_map(oh_no, range(3), concurrency=2, ignore_exceptions=True)) +# == [None] * 3 +# ) +# assert ( +# list(parallel_map(oh_no, range(3), concurrency=1, ignore_exceptions=True)) +# == [None] * 3 +# ) - def test_no_op(self) -> None: - assert list(parallel_map(lambda v: v**2, [])) == [] - assert list(parallel_map(lambda v: v**2, [], chunk_size=100)) == [] - assert list(parallel_map(lambda v: v**2, [], concurrency=100)) == [] +# def test_no_op(self) -> None: +# assert list(parallel_map(lambda v: v**2, [])) == [] +# assert list(parallel_map(lambda v: v**2, [], chunk_size=100)) == [] +# assert list(parallel_map(lambda v: v**2, [], concurrency=100)) == [] diff --git a/tests/utilities/test_threaded_parallel_map.py b/tests/utilities/test_threaded_parallel_map.py index 93811d4..0bf84a0 100644 --- a/tests/utilities/test_threaded_parallel_map.py +++ b/tests/utilities/test_threaded_parallel_map.py @@ -1,124 +1,124 @@ -# import unittest +import unittest -# import pytest +import pytest -# from src.great_ai.utilities import threaded_parallel_map +from src.great_ai.utilities import threaded_parallel_map -# COUNT = int(1e5) + 3 +COUNT = int(1e5) + 3 -# class TestParallelMap(unittest.TestCase): -# def test_simple_case_with_progress_bar(self) -> None: -# assert list( -# threaded_parallel_map(lambda v: v**2, range(COUNT), concurrency=4) -# ) == [v**2 for v in range(COUNT)] +class TestParallelMap(unittest.TestCase): + def test_simple_case_with_progress_bar(self) -> None: + assert list( + threaded_parallel_map(lambda v: v**2, range(COUNT), concurrency=4) + ) == [v**2 for v in range(COUNT)] -# def test_with_iterable(self) -> None: -# from time import sleep + def test_with_iterable(self) -> None: + from time import sleep -# def my_generator(): -# for i in range(10): -# yield i -# sleep(0.1) + def my_generator(): + for i in range(10): + yield i + sleep(0.1) -# expected = [v**3 for v in range(10)] + expected = [v**3 for v in range(10)] -# assert ( -# list(threaded_parallel_map(lambda x: x**3, my_generator(), chunk_size=1)) -# == expected -# ) + assert ( + list(threaded_parallel_map(lambda x: x**3, my_generator(), chunk_size=1)) + == expected + ) -# def test_simple_case_without_progress_bar(self) -> None: -# assert list( -# threaded_parallel_map(lambda v: v**2, range(COUNT), concurrency=2) -# ) == [v**2 for v in range(COUNT)] + def test_simple_case_without_progress_bar(self) -> None: + assert list( + threaded_parallel_map(lambda v: v**2, range(COUNT), concurrency=2) + ) == [v**2 for v in range(COUNT)] -# def test_simple_case_invalid_values(self) -> None: -# with pytest.raises(AssertionError): -# list(threaded_parallel_map(lambda v: v**2, range(COUNT), concurrency=0)) + def test_simple_case_invalid_values(self) -> None: + with pytest.raises(AssertionError): + list(threaded_parallel_map(lambda v: v**2, range(COUNT), concurrency=0)) -# with pytest.raises(AssertionError): -# list(threaded_parallel_map(lambda v: v**2, range(COUNT), chunk_size=0)) + with pytest.raises(AssertionError): + list(threaded_parallel_map(lambda v: v**2, range(COUNT), chunk_size=0)) -# def test_this_worker_exception(self) -> None: -# def my_generator(): -# yield 1 -# yield 2 -# yield 3 -# assert False + def test_this_worker_exception(self) -> None: + def my_generator(): + yield 1 + yield 2 + yield 3 + assert False -# with pytest.raises(AssertionError): -# list( -# threaded_parallel_map( -# lambda v: v**2, my_generator(), concurrency=2, chunk_size=2 -# ) -# ) + with pytest.raises(AssertionError): + list( + threaded_parallel_map( + lambda v: v**2, my_generator(), concurrency=2, chunk_size=2 + ) + ) -# with pytest.raises(AssertionError): -# list( -# threaded_parallel_map( -# lambda v: v**2, my_generator(), concurrency=1, chunk_size=2 -# ) -# ) + with pytest.raises(AssertionError): + list( + threaded_parallel_map( + lambda v: v**2, my_generator(), concurrency=1, chunk_size=2 + ) + ) -# def test_ignore_this_worker_exception(self) -> None: -# def my_generator(): -# yield 1 -# yield 2 -# yield 3 -# yield 1 / 0 + def test_ignore_this_worker_exception(self) -> None: + def my_generator(): + yield 1 + yield 2 + yield 3 + yield 1 / 0 -# assert list( -# threaded_parallel_map( -# lambda v: v**2, -# my_generator(), -# concurrency=2, -# chunk_size=2, -# ignore_exceptions=True, -# ) -# ) == [1, 4] -# assert list( -# threaded_parallel_map( -# lambda v: v**2, -# my_generator(), -# concurrency=1, -# chunk_size=2, -# ignore_exceptions=True, -# ) -# ) == [1, 4, 9] + assert list( + threaded_parallel_map( + lambda v: v**2, + my_generator(), + concurrency=2, + chunk_size=2, + ignore_exceptions=True, + ) + ) == [1, 4] + assert list( + threaded_parallel_map( + lambda v: v**2, + my_generator(), + concurrency=1, + chunk_size=2, + ignore_exceptions=True, + ) + ) == [1, 4, 9] -# def test_worker_worker_exception(self) -> None: -# def oh_no(_): -# raise ValueError("hi") + def test_worker_worker_exception(self) -> None: + def oh_no(_): + raise ValueError("hi") -# with pytest.raises(ValueError): -# list(threaded_parallel_map(oh_no, range(COUNT), concurrency=2)) + with pytest.raises(ValueError): + list(threaded_parallel_map(oh_no, range(COUNT), concurrency=2)) -# with pytest.raises(ValueError): -# list(threaded_parallel_map(oh_no, range(COUNT), concurrency=1)) + with pytest.raises(ValueError): + list(threaded_parallel_map(oh_no, range(COUNT), concurrency=1)) -# def test_ignore_worker_worker_exception(self) -> None: -# def oh_no(_): -# raise ValueError("hi") + def test_ignore_worker_worker_exception(self) -> None: + def oh_no(_): + raise ValueError("hi") -# assert ( -# list( -# threaded_parallel_map( -# oh_no, range(3), concurrency=2, ignore_exceptions=True -# ) -# ) -# == [None] * 3 -# ) -# assert ( -# list( -# threaded_parallel_map( -# oh_no, range(3), concurrency=1, ignore_exceptions=True -# ) -# ) -# == [None] * 3 -# ) + assert ( + list( + threaded_parallel_map( + oh_no, range(3), concurrency=2, ignore_exceptions=True + ) + ) + == [None] * 3 + ) + assert ( + list( + threaded_parallel_map( + oh_no, range(3), concurrency=1, ignore_exceptions=True + ) + ) + == [None] * 3 + ) -# def test_no_op(self) -> None: -# assert list(threaded_parallel_map(lambda v: v**2, [])) == [] -# assert list(threaded_parallel_map(lambda v: v**2, [], chunk_size=100)) == [] -# assert list(threaded_parallel_map(lambda v: v**2, [], concurrency=100)) == [] + def test_no_op(self) -> None: + assert list(threaded_parallel_map(lambda v: v**2, [])) == [] + assert list(threaded_parallel_map(lambda v: v**2, [], chunk_size=100)) == [] + assert list(threaded_parallel_map(lambda v: v**2, [], concurrency=100)) == []