andras/great-ai
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases 4 Packages Wiki Activity Actions

Fix typing and minor issues

Browse source
This commit is contained in:
Andras Schmelczer 2022-07-07 14:12:44 +02:00
parent 2db2253578
commit 72ab627a34
54 changed files with 635 additions and 589 deletions
Download patch file Download diff file Expand all files Collapse all files

14
great_ai/utilities/parallel_map/get_config.py
View file

@ -17,14 +17,15 @@ def get_config(
) -> ParallelMapConfiguration:
is_input_sequence = hasattr(input_values, "__len__")
input_length = len(input_values) if is_input_sequence else None # type: ignore
if concurrency is None:
concurrency = os.cpu_count() or 1
assert concurrency >= 1, "At least one mapper process has to be created"
if chunk_size is None:
if is_input_sequence:
chunk_size = max(1, ceil(len(input_values) / concurrency / 10))
if input_length is not None:
chunk_size = max(1, ceil(input_length / concurrency / 10))
else:
raise ValueError(
"The argument for `values` does not implement `__len__`, therefore, you must provide a `chunk_size`"
@ -32,19 +33,14 @@ def get_config(
assert chunk_size >= 1, "Chunks have to contain at least one element"
chunk_count: Optional[int] = None
if is_input_sequence:
chunk_count = ceil(len(input_values) / chunk_size)
if input_length is not None:
chunk_count = ceil(input_length / chunk_size)
if chunk_count < concurrency:
logger.warning(
f"Limiting concurrency to {chunk_count} because there are only {chunk_count} chunks"
)
concurrency = chunk_count
if concurrency == 1:
logger.warning("Running in series, there is no reason for parallelism")
input_length = len(input_values) if is_input_sequence else None
config = ParallelMapConfiguration(
concurrency=concurrency,
chunk_count=chunk_count,