Add simpler parallel_map
This commit is contained in:
parent
7db1503a45
commit
286ede5fd3
3 changed files with 37 additions and 1 deletions
|
|
@ -1,3 +1,4 @@
|
|||
from .parallel_map import parallel_map
|
||||
from .simple_parallel_map import simple_parallel_map
|
||||
from .threaded_parallel_map import threaded_parallel_map
|
||||
from .worker_exception import WorkerException
|
||||
|
|
|
|||
30
great_ai/utilities/parallel_map/simple_parallel_map.py
Normal file
30
great_ai/utilities/parallel_map/simple_parallel_map.py
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
from typing import Awaitable, Callable, List, Optional, Sequence, TypeVar, Union
|
||||
|
||||
from tqdm.cli import tqdm
|
||||
|
||||
from .parallel_map import parallel_map
|
||||
|
||||
T = TypeVar("T")
|
||||
V = TypeVar("V")
|
||||
|
||||
|
||||
def simple_parallel_map(
|
||||
func: Callable[[T], Union[V, Awaitable[V]]],
|
||||
input_values: Sequence[T],
|
||||
*,
|
||||
chunk_size: Optional[int] = None,
|
||||
concurrency: Optional[int] = None,
|
||||
) -> List[V]:
|
||||
input_values = list(input_values) # in case the input is mistakenly not a sequence
|
||||
return list(
|
||||
tqdm(
|
||||
parallel_map(
|
||||
func=func,
|
||||
input_values=input_values,
|
||||
chunk_size=chunk_size,
|
||||
concurrency=concurrency,
|
||||
),
|
||||
total=len(input_values),
|
||||
dynamic_ncols=True,
|
||||
)
|
||||
)
|
||||
Loading…
Add table
Add a link
Reference in a new issue