andras/great-ai
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases 4 Packages Wiki Activity Actions

Add better logging

Browse source
This commit is contained in:
Andras Schmelczer 2022-05-27 11:12:48 +02:00
parent 5b02a0a94d
commit 2c516cf861
1 changed files with 10 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

13
great_ai/src/great_ai/utilities/parallel_map.py
View file

@ -5,9 +5,9 @@ import multiprocess as mp
import psutil import psutil
from tqdm.auto import tqdm from tqdm.auto import tqdm
from .logger import create_logger from .logger import get_logger
logger = create_logger("parallel_map") logger = get_logger("parallel_map")
def parallel_map( def parallel_map(
@ -25,8 +25,15 @@ def parallel_map(
if not chunk_size: if not chunk_size:
chunk_size = max(1, ceil(len(values) / concurrency / 10)) chunk_size = max(1, ceil(len(values) / concurrency / 10))
chunk_count = ceil(len(values) / chunk_size)
if chunk_count < concurrency:
logger.warning(
f"Limiting concurrency to {chunk_count} because there are only {chunk_count} chunks"
)
concurrency = chunk_count
logger.info( logger.info(
f"Starting parallel map, concurrency: {concurrency}, chunk size: {chunk_size}" f"Starting parallel map (concurrency: {concurrency}, chunk size: {chunk_size})"
) )
if concurrency == 1 or len(values) <= chunk_size: if concurrency == 1 or len(values) <= chunk_size: