diff --git a/great_ai/src/great_ai/utilities/__init__.py b/great_ai/src/great_ai/utilities/__init__.py index e69de29..1e9de78 100644 --- a/great_ai/src/great_ai/utilities/__init__.py +++ b/great_ai/src/great_ai/utilities/__init__.py @@ -0,0 +1,12 @@ +from .clean import clean +from .evaluate_ranking import evaluate_ranking +from .get_sentences import get_sentences +from .language import english_name_of_language, is_english, predict_language +from .lemmatize_text import lemmatize_text +from .lemmatize_token import lemmatize_token +from .match_names import match_names +from .matcher import fast_tokenize, filter_sentences, normalize +from .nlp import nlp +from .parallel_map import parallel_map +from .publication_tei import PublicationTEI +from .unique import unique diff --git a/great_ai/src/great_ai/utilities/parallel_map.py b/great_ai/src/great_ai/utilities/parallel_map.py index b9aa438..108bdcc 100644 --- a/great_ai/src/great_ai/utilities/parallel_map.py +++ b/great_ai/src/great_ai/utilities/parallel_map.py @@ -14,9 +14,12 @@ def parallel_map( function: Callable[[Any], Any], values: Iterable[Any], chunk_size: Optional[int] = None, - concurrency: int = psutil.cpu_count(), + concurrency: Optional[int] = None, disable_progress: bool = False, ) -> List[Any]: + if concurrency is None: + concurrency = psutil.cpu_count() + assert concurrency > 0 assert chunk_size is None or chunk_size > 0