andras/great-ai
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases 4 Packages Wiki Activity Actions

Improve serialization

Browse source
This commit is contained in:
Andras Schmelczer 2022-07-03 12:25:56 +02:00
parent 7d774f37be
commit c76342534a
8 changed files with 16 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

7
src/great_ai/utilities/parallel_map/mapper_function.py
View file

@ -3,7 +3,7 @@ import queue
import threading
import traceback
from time import sleep
from typing import Union
from typing import Callable, Union
import dill
@ -12,10 +12,11 @@ def mapper_function(
input_queue: Union[mp.Queue, queue.Queue],
output_queue: Union[mp.Queue, queue.Queue],
should_stop: Union[mp.Event, threading.Event],
serialized_map_function: bytes,
map_function: Union[bytes, Callable],
) -> None:
try:
map_function = dill.loads(serialized_map_function)
if isinstance(map_function, bytes):
map_function = dill.loads(map_function)
last_chunk = None
while not should_stop.is_set():