andras/great-ai
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases 4 Packages Wiki Activity Actions

Add threaded parallel_map

Browse source
This commit is contained in:
Andras Schmelczer 2022-06-30 17:22:16 +02:00
parent 154cc52fa6
commit 0ff0b49a79
9 changed files with 225 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

8
src/great_ai/utilities/parallel_map/mapper_function.py
View file

@ -1,13 +1,15 @@
import multiprocessing as mp
import queue
import threading
from typing import Union
import dill
def mapper_function(
input_queue: mp.Queue,
output_queue: mp.Queue,
should_stop: mp.Event,
input_queue: Union[mp.Queue, queue.Queue],
output_queue: Union[mp.Queue, queue.Queue],
should_stop: Union[mp.Event, threading.Event],
serialized_map_function: bytes,
):
map_function = dill.loads(serialized_map_function)