From 2c516cf8611cb40eeb0df1eb4be60fc50e373a42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A1s=20Schmelczer?= Date: Fri, 27 May 2022 11:12:48 +0200 Subject: [PATCH] Add better logging --- great_ai/src/great_ai/utilities/parallel_map.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/great_ai/src/great_ai/utilities/parallel_map.py b/great_ai/src/great_ai/utilities/parallel_map.py index 4b5d6f4..b9aa438 100644 --- a/great_ai/src/great_ai/utilities/parallel_map.py +++ b/great_ai/src/great_ai/utilities/parallel_map.py @@ -5,9 +5,9 @@ import multiprocess as mp import psutil from tqdm.auto import tqdm -from .logger import create_logger +from .logger import get_logger -logger = create_logger("parallel_map") +logger = get_logger("parallel_map") def parallel_map( @@ -25,8 +25,15 @@ def parallel_map( if not chunk_size: chunk_size = max(1, ceil(len(values) / concurrency / 10)) + chunk_count = ceil(len(values) / chunk_size) + if chunk_count < concurrency: + logger.warning( + f"Limiting concurrency to {chunk_count} because there are only {chunk_count} chunks" + ) + concurrency = chunk_count + logger.info( - f"Starting parallel map, concurrency: {concurrency}, chunk size: {chunk_size}" + f"Starting parallel map (concurrency: {concurrency}, chunk size: {chunk_size})" ) if concurrency == 1 or len(values) <= chunk_size: