Refactor and reformat

great_ai/src/great_ai/large_file/__main__.py

@@ -4,7 +4,7 @@ from argparse import Namespace
 from pathlib import Path
 from typing import Mapping, Type
 
-from great_ai.utilities.logger import get_logger
+from great_ai.utilities import get_logger
 
 from .large_file import LargeFile, LargeFileLocal, LargeFileMongo, LargeFileS3
 from .parse_arguments import parse_arguments

great_ai/src/great_ai/large_file/large_file/large_file.py

@@ -1,4 +1,3 @@
-import configparser
 import os
 import shutil
 import tempfile
@@ -7,7 +6,7 @@ from pathlib import Path
 from types import TracebackType
 from typing import IO, Any, List, Optional, Type, Union, cast
 
-from great_ai.utilities.logger import get_logger
+from great_ai.utilities import ConfigFile, get_logger
 
 from ..helper import human_readable_to_byte
 from ..models import DataInstance
@@ -86,16 +85,7 @@ class LargeFile(ABC):
         cls,
         secrets_path: Union[Path, str],
     ) -> None:
-        if isinstance(secrets_path, str):
+        cls.configure_credentials(**ConfigFile(secrets_path))
-            secrets_path = Path(secrets_path)
-
-        if not secrets_path.exists():
-            raise FileNotFoundError(secrets_path.resolve())
-
-        credentials = configparser.ConfigParser()
-        credentials.read(secrets_path)
-        credentials.default_section
-        cls.configure_credentials(**credentials[credentials.default_section])
 
     @classmethod
     def configure_credentials(

great_ai/src/great_ai/large_file/large_file/large_file_local.py

@@ -1,7 +1,7 @@
 from pathlib import Path
 from typing import Any, List, Optional
 
-from great_ai.utilities.logger import get_logger
+from great_ai.utilities import get_logger
 
 from ..models import DataInstance
 from .large_file import LargeFile

great_ai/src/great_ai/large_file/large_file/large_file_mongo.py

@@ -6,7 +6,7 @@ from typing import Any, List, Mapping
 from gridfs import DEFAULT_CHUNK_SIZE, Database, GridFSBucket
 from pymongo import MongoClient
 
-from great_ai.utilities.logger import get_logger
+from great_ai.utilities import get_logger
 
 from ..helper import DownloadProgressBar, UploadProgressBar
 from ..models import DataInstance

great_ai/src/great_ai/large_file/large_file/large_file_s3.py

@@ -4,7 +4,7 @@ from typing import Any, List, Mapping, Optional
 
 import boto3
 
-from great_ai.utilities.logger import get_logger
+from great_ai.utilities import get_logger
 
 from ..helper import DownloadProgressBar, UploadProgressBar
 from ..models import DataInstance

great_ai/src/great_ai/utilities/__init__.py

@@ -1,13 +1,14 @@
 from .clean import clean
+from .config_file import ConfigFile
 from .evaluate_ranking import evaluate_ranking
 from .get_sentences import get_sentences
 from .language import english_name_of_language, is_english, predict_language
 from .lemmatize_text import lemmatize_text
 from .lemmatize_token import lemmatize_token
+from .logger import get_logger
 from .match_names import match_names
 from .matcher import fast_tokenize, filter_sentences, normalize
 from .nlp import nlp
 from .parallel_map import parallel_map
 from .publication_tei import PublicationTEI
 from .unique import unique
-from .config_file import ConfigFile

great_ai/src/great_ai/utilities/config_file/__init__.py

@@ -1,2 +1,2 @@
 from .config_file import ConfigFile
-from .parse_error import ParseError
+from .parse_error import ParseError

great_ai/src/great_ai/utilities/config_file/config_file.py

@@ -1,52 +1,87 @@
-from pathlib import Path
 import os
-from typing import Dict, Union
+from pathlib import Path
+from typing import Dict, Iterable, Tuple, Union
 
 from ..logger import get_logger
 from .parse_error import ParseError
 from .pattern import pattern
 
-ENVIRONMENT_VARIABLE_KEY_PREFIX = 'ENV'
+ENVIRONMENT_VARIABLE_KEY_PREFIX = "ENV"
+
+logger = get_logger("ConfigFile")
 
-logger = get_logger('ConfigFile')
 
 class ConfigFile:
-    def __init__(self, path: Union[Path, str], ignore_missing_environment_variables:bool=False) -> None:
+    def __init__(
+        self, path: Union[Path, str], ignore_missing_environment_variables: bool = False
+    ) -> None:
         if not isinstance(path, Path):
             path = Path(path)
+
+        if not path.exists():
+            raise FileNotFoundError(path.absolute())
+
         self._path = path
-        self._ignore_missing_environment_variables = ignore_missing_environment_variables
+        self._ignore_missing_environment_variables = (
+            ignore_missing_environment_variables
+        )
         self._key_values: Dict[str, str] = {}
 
         self._parse()
 
     def _parse(self):
-        with open(self._path, encoding='utf-8') as f:
+        with open(self._path, encoding="utf-8") as f:
             lines: str = f.read()
 
         matches = pattern.findall(lines)
         for key, *values in matches:
             if key in self._key_values:
-                raise KeyError(f'Key `{key}` has been already defined and its value is `{value}`')
+                raise KeyError(
+                    f"Key `{key}` has been already defined and its value is `{self._key_values[key]}`"
+                )
 
             try:
                 value = next(v for v in values if v)
             except StopIteration:
-                raise ParseError(f'Cannot parse config file ({self._path.absolute()}), error at key `{key}`')
+                raise ParseError(
+                    f"Cannot parse config file ({self._path.absolute()}), error at key `{key}`"
+                )
 
-            if value.startswith(f'{ENVIRONMENT_VARIABLE_KEY_PREFIX}:'):
+            if value.startswith(f"{ENVIRONMENT_VARIABLE_KEY_PREFIX}:"):
-                _, value = value.split(':')
+                _, value = value.split(":")
                 if value not in os.environ:
                     issue = f'The value of `{key}` contains the "{ENVIRONMENT_VARIABLE_KEY_PREFIX}` prefix but `{value}` is not defined as an environment variable'
                     if self._ignore_missing_environment_variables:
-                        logger.warning(f'{issue}, defaulting to `None`')
+                        logger.warning(f"{issue}, defaulting to `None`")
                     else:
                         raise KeyError(issue)
                 value = os.environ[value]
-            
+
             self._key_values[key] = value
 
     def __getattr__(self, key: str) -> str:
         if key in self._key_values:
             return self._key_values[key]
-        raise KeyError(f'Key `{key}` is not found in configuration file ({self._path.absolute()})')
+        raise KeyError(
+            f"Key `{key}` is not found in configuration file ({self._path.absolute()})"
+        )
+
+    __getitem__ = __getattr__
+
+    def __iter__(self) -> Iterable[Tuple[str, str]]:
+        return iter(self._key_values)
+
+    def __len__(self) -> int:
+        return len(self._key_values)
+
+    def keys(self):
+        return self._key_values.keys()
+
+    def values(self):
+        return self._key_values.values()
+
+    def items(self):
+        return self._key_values.items()
+
+    def __repr__(self):
+        return f"{type(self).__name__}(\n  path={self._path},\n  ignore_missing_environment_variables={self._ignore_missing_environment_variables}\n) {{{self._key_values}}}"

great_ai/src/great_ai/utilities/config_file/parse_error.py

@@ -1,3 +1,2 @@
-
 class ParseError(Exception):
     pass

great_ai/src/great_ai/utilities/config_file/pattern.py

@@ -1,7 +1,7 @@
 import re
 
-
+pattern = re.compile(
-pattern = re.compile(r'''
+    r"""
     \s*             # leading whitespace is allowed
     (\w+?)          # then comes the key
     \s*=\s*         # the key and value are separated by an equal sign
@@ -13,4 +13,6 @@ pattern = re.compile(r'''
     )
     \s*(?:\#.*)?        # comments can be added with the `#` symbol
     (?:\n|$)            # a key-value pairs are separated by new lines
-''', flags=re.UNICODE | re.VERBOSE)
+""",
+    flags=re.UNICODE | re.VERBOSE,
+)

great_ai/tests/utilities/data/good.conf

@@ -1,6 +1,6 @@
 
 zeroth_key="test"
-first_key=test
+first_key= András 
 second_key = test 2 # comment is ignored
 third_key= "test= 2=="
 

great_ai/tests/utilities/data/parsed.py

@@ -1,4 +1,4 @@
-from great_ai.utilities.publication_tei.models import (
+from great_ai.utilities.publication_tei import (
     Affiliation,
     Author,
     Bookmark,

great_ai/tests/utilities/test_clean.py

@@ -1,6 +1,6 @@
 import unittest
 
-from src.great_ai.utilities.clean import clean
+from src.great_ai.utilities import clean
 
 
 class TestClean(unittest.TestCase):

great_ai/tests/utilities/test_config_file.py

@@ -1,7 +1,9 @@
 import unittest
 from pathlib import Path
+
 import pytest
-from src.great_ai.utilities.config_file import ConfigFile
+
+from src.great_ai.utilities import ConfigFile
 
 DATA_PATH = Path(__file__).parent.resolve() / "data"
 
@@ -9,38 +11,60 @@ DATA_PATH = Path(__file__).parent.resolve() / "data"
 class TestConfigFile(unittest.TestCase):
     def test_simple(self) -> None:
         c = ConfigFile(DATA_PATH / "good.conf")
-        assert c.zeroth_key == 'test'
+        assert c.zeroth_key == "test"
-        assert c.first_key == 'test'
+        assert c.first_key == "András"
-        assert c.second_key == 'test 2'
+        assert c.second_key == "test 2"
-        assert c.third_key == 'test= 2=='
+        assert c.third_key == "test= 2=="
-        assert c.fourth_key == '''
+        assert (
+            c.fourth_key
+            == """
 this#
 is
 multiline
-'''
+"""
-        assert c.whitespace == 'hardly  matters'
+        )
-        
+        assert c.whitespace == "hardly  matters"
+
+    def test_simple_dict(self) -> None:
+        c = ConfigFile(DATA_PATH / "good.conf")
+        assert c["zeroth_key"] == "test"
+        assert c["first_key"] == "András"
+        assert c["second_key"] == "test 2"
+        assert c["third_key"] == "test= 2=="
+        assert (
+            c["fourth_key"]
+            == """
+this#
+is
+multiline
+"""
+        )
+        assert c["whitespace"] == "hardly  matters"
 
     def test_string_path(self) -> None:
         c = ConfigFile(str(DATA_PATH / "good.conf"))
-        assert c.zeroth_key == 'test'
+        assert c.zeroth_key == "test"
-        assert c.first_key == 'test'
+        assert c.first_key == "András"
-        assert c.second_key == 'test 2'
+        assert c.second_key == "test 2"
-        assert c.third_key == 'test= 2=='
+        assert c.third_key == "test= 2=="
-        assert c.fourth_key == '''
+        assert (
+            c.fourth_key
+            == """
 this#
 is
 multiline
-'''
+"""
-        assert c.whitespace == 'hardly  matters'
+        )
+        assert c.whitespace == "hardly  matters"
 
     def test_env(self) -> None:
         import os
-        os.environ['alma'] = '12'
+
+        os.environ["alma"] = "12"
         c = ConfigFile(DATA_PATH / "env.conf")
-        del os.environ['alma']
+        del os.environ["alma"]
-        assert c.first_key == 'test'
+        assert c.first_key == "test"
-        assert c.second_key == '12'
+        assert c.second_key == "12"
 
     def test_env_not_exists(self) -> None:
         with pytest.raises(KeyError):
@@ -48,9 +72,10 @@ multiline
 
     def test_env_not_exists_but_ignored(self) -> None:
         with pytest.raises(KeyError):
-            ConfigFile(DATA_PATH / "env.conf", ignore_missing_environment_variables=True)
+            ConfigFile(
+                DATA_PATH / "env.conf", ignore_missing_environment_variables=True
+            )
 
     def test_duplicate_key(self) -> None:
         with pytest.raises(KeyError):
             ConfigFile(DATA_PATH / "bad.conf")
-       

great_ai/tests/utilities/test_evaluate_ranking.py

@@ -5,7 +5,7 @@ import matplotlib
 
 matplotlib.use("Agg")  # don't show a window for each test
 
-from src.great_ai.utilities.evaluate_ranking import evaluate_ranking
+from src.great_ai.utilities import evaluate_ranking
 
 
 class TestEvaluateRanking(unittest.TestCase):

great_ai/tests/utilities/test_get_sentences.py

@@ -1,6 +1,6 @@
 import unittest
 
-from src.great_ai.utilities.get_sentences import get_sentences
+from src.great_ai.utilities import get_sentences
 
 
 class TestGetSentences(unittest.TestCase):

great_ai/tests/utilities/test_language.py

@@ -1,6 +1,6 @@
 import unittest
 
-from src.great_ai.utilities.language import (
+from src.great_ai.utilities import (
     english_name_of_language,
     is_english,
     predict_language,

great_ai/tests/utilities/test_lemmatize_text.py

@@ -1,6 +1,6 @@
 import unittest
 
-from src.great_ai.utilities.lemmatize_text import lemmatize_text
+from src.great_ai.utilities import lemmatize_text
 
 
 class TestLemmatizeText(unittest.TestCase):

great_ai/tests/utilities/test_lemmatize_token.py

@@ -1,7 +1,6 @@
 import unittest
 
-from src.great_ai.utilities.lemmatize_text import lemmatize_token
+from src.great_ai.utilities import lemmatize_token, nlp
-from src.great_ai.utilities.nlp import nlp
 
 
 class TestLemmatizeToken(unittest.TestCase):

great_ai/tests/utilities/test_match_names.py

@@ -1,6 +1,6 @@
 import unittest
 
-from src.great_ai.utilities.match_names.match_names import match_names
+from src.great_ai.utilities import match_names
 
 
 class TestMatchNames(unittest.TestCase):

great_ai/tests/utilities/test_parallel_map.py

@@ -1,6 +1,6 @@
 import unittest
 
-from src.great_ai.utilities.parallel_map import parallel_map
+from src.great_ai.utilities import parallel_map
 
 COUNT = int(1e5) + 3
 

great_ai/tests/utilities/test_publication_tei.py

@@ -1,7 +1,7 @@
 import unittest
 from pathlib import Path
 
-from src.great_ai.utilities.publication_tei import PublicationTEI
+from src.great_ai.utilities import PublicationTEI
 
 from .data.parsed import (
     abstract,

great_ai/tests/utilities/test_unique.py

@@ -1,6 +1,6 @@
 import unittest
 
-from src.great_ai.utilities.unique import unique
+from src.great_ai.utilities import unique
 
 original = [
     ("a", 1),