andras/great-ai
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases 4 Packages Wiki Activity Actions

Try fixing zombie processes

Browse source
This commit is contained in:
Andras Schmelczer 2022-07-03 13:54:15 +02:00
parent 63c109e616
commit 4733706eb1
No known key found for this signature in database
GPG key ID: 0EA1BC97D0AB076E
2 changed files with 27 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

6
src/great_ai/utilities/parallel_map/parallel_map.py
View file

@ -123,13 +123,13 @@ def parallel_map(
ignore_exceptions=ignore_exceptions, ignore_exceptions=ignore_exceptions,
) )
should_stop.set() should_stop.set()
for p in processes:
p.join()
p.close()
except: except:
for p in processes: for p in processes:
p.terminate() p.terminate()
raise raise
finally: finally:
for p in processes:
p.join() # terminated processes have to be joined else they remain zombies
p.close()
input_queue.close() input_queue.close()
output_queue.close() output_queue.close()

48
tests/utilities/test_parallel_map.py
View file

@ -61,31 +61,31 @@ class TestParallelMap(unittest.TestCase):
) )
) )
# def test_ignore_this_process_exception(self) -> None: def test_ignore_this_process_exception(self) -> None:
# def my_generator(): def my_generator():
# yield 1 yield 1
# yield 2 yield 2
# yield 3 yield 3
# yield 1 / 0 yield 1 / 0
# assert list( assert list(
# parallel_map( parallel_map(
# lambda v: v**2, lambda v: v**2,
# my_generator(), my_generator(),
# concurrency=2, concurrency=2,
# chunk_size=2, chunk_size=2,
# ignore_exceptions=True, ignore_exceptions=True,
# ) )
# ) == [1, 4] ) == [1, 4]
# assert list( assert list(
# parallel_map( parallel_map(
# lambda v: v**2, lambda v: v**2,
# my_generator(), my_generator(),
# concurrency=1, concurrency=1,
# chunk_size=2, chunk_size=2,
# ignore_exceptions=True, ignore_exceptions=True,
# ) )
# ) == [1, 4, 9] ) == [1, 4, 9]
def test_worker_process_exception(self) -> None: def test_worker_process_exception(self) -> None:
def oh_no(_): def oh_no(_):