andras/great-ai
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases 4 Packages Wiki Activity Actions

Try fixing zombie processes

Browse source
This commit is contained in:
Andras Schmelczer 2022-07-03 13:54:15 +02:00
parent 63c109e616
commit 4733706eb1
No known key found for this signature in database
GPG key ID: 0EA1BC97D0AB076E
2 changed files with 27 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

6
src/great_ai/utilities/parallel_map/parallel_map.py
View file

@ -123,13 +123,13 @@ def parallel_map(
ignore_exceptions=ignore_exceptions,
)
should_stop.set()
for p in processes:
p.join()
p.close()
except:
for p in processes:
p.terminate()
raise
finally:
for p in processes:
p.join() # terminated processes have to be joined else they remain zombies
p.close()
input_queue.close()
output_queue.close()

48
tests/utilities/test_parallel_map.py
View file

@ -61,31 +61,31 @@ class TestParallelMap(unittest.TestCase):
)
)
# def test_ignore_this_process_exception(self) -> None:
# def my_generator():
# yield 1
# yield 2
# yield 3
# yield 1 / 0
def test_ignore_this_process_exception(self) -> None:
def my_generator():
yield 1
yield 2
yield 3
yield 1 / 0
# assert list(
# parallel_map(
# lambda v: v**2,
# my_generator(),
# concurrency=2,
# chunk_size=2,
# ignore_exceptions=True,
# )
# ) == [1, 4]
# assert list(
# parallel_map(
# lambda v: v**2,
# my_generator(),
# concurrency=1,
# chunk_size=2,
# ignore_exceptions=True,
# )
# ) == [1, 4, 9]
assert list(
parallel_map(
lambda v: v**2,
my_generator(),
concurrency=2,
chunk_size=2,
ignore_exceptions=True,
)
) == [1, 4]
assert list(
parallel_map(
lambda v: v**2,
my_generator(),
concurrency=1,
chunk_size=2,
ignore_exceptions=True,
)
) == [1, 4, 9]
def test_worker_process_exception(self) -> None:
def oh_no(_):