From 750ba7b0d46b7781f0cf5b895aab1a59b29baed1 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 25 Jun 2022 11:06:04 +0200 Subject: [PATCH] Refactor and reformat --- great_ai/src/great_ai/large_file/__main__.py | 2 +- .../large_file/large_file/large_file.py | 14 +--- .../large_file/large_file/large_file_local.py | 2 +- .../large_file/large_file/large_file_mongo.py | 2 +- .../large_file/large_file/large_file_s3.py | 2 +- great_ai/src/great_ai/utilities/__init__.py | 3 +- .../utilities/config_file/__init__.py | 2 +- .../utilities/config_file/config_file.py | 63 +++++++++++++---- .../utilities/config_file/parse_error.py | 1 - .../great_ai/utilities/config_file/pattern.py | 8 ++- great_ai/tests/utilities/data/good.conf | 2 +- great_ai/tests/utilities/data/parsed.py | 2 +- great_ai/tests/utilities/test_clean.py | 2 +- great_ai/tests/utilities/test_config_file.py | 69 +++++++++++++------ .../tests/utilities/test_evaluate_ranking.py | 2 +- .../tests/utilities/test_get_sentences.py | 2 +- great_ai/tests/utilities/test_language.py | 2 +- .../tests/utilities/test_lemmatize_text.py | 2 +- .../tests/utilities/test_lemmatize_token.py | 3 +- great_ai/tests/utilities/test_match_names.py | 2 +- great_ai/tests/utilities/test_parallel_map.py | 2 +- .../tests/utilities/test_publication_tei.py | 2 +- great_ai/tests/utilities/test_unique.py | 2 +- 23 files changed, 122 insertions(+), 71 deletions(-) diff --git a/great_ai/src/great_ai/large_file/__main__.py b/great_ai/src/great_ai/large_file/__main__.py index 6079d9d..c443b4f 100644 --- a/great_ai/src/great_ai/large_file/__main__.py +++ b/great_ai/src/great_ai/large_file/__main__.py @@ -4,7 +4,7 @@ from argparse import Namespace from pathlib import Path from typing import Mapping, Type -from great_ai.utilities.logger import get_logger +from great_ai.utilities import get_logger from .large_file import LargeFile, LargeFileLocal, LargeFileMongo, LargeFileS3 from .parse_arguments import parse_arguments diff --git a/great_ai/src/great_ai/large_file/large_file/large_file.py b/great_ai/src/great_ai/large_file/large_file/large_file.py index ce96ca8..09fd350 100644 --- a/great_ai/src/great_ai/large_file/large_file/large_file.py +++ b/great_ai/src/great_ai/large_file/large_file/large_file.py @@ -1,4 +1,3 @@ -import configparser import os import shutil import tempfile @@ -7,7 +6,7 @@ from pathlib import Path from types import TracebackType from typing import IO, Any, List, Optional, Type, Union, cast -from great_ai.utilities.logger import get_logger +from great_ai.utilities import ConfigFile, get_logger from ..helper import human_readable_to_byte from ..models import DataInstance @@ -86,16 +85,7 @@ class LargeFile(ABC): cls, secrets_path: Union[Path, str], ) -> None: - if isinstance(secrets_path, str): - secrets_path = Path(secrets_path) - - if not secrets_path.exists(): - raise FileNotFoundError(secrets_path.resolve()) - - credentials = configparser.ConfigParser() - credentials.read(secrets_path) - credentials.default_section - cls.configure_credentials(**credentials[credentials.default_section]) + cls.configure_credentials(**ConfigFile(secrets_path)) @classmethod def configure_credentials( diff --git a/great_ai/src/great_ai/large_file/large_file/large_file_local.py b/great_ai/src/great_ai/large_file/large_file/large_file_local.py index ed39560..33480ae 100644 --- a/great_ai/src/great_ai/large_file/large_file/large_file_local.py +++ b/great_ai/src/great_ai/large_file/large_file/large_file_local.py @@ -1,7 +1,7 @@ from pathlib import Path from typing import Any, List, Optional -from great_ai.utilities.logger import get_logger +from great_ai.utilities import get_logger from ..models import DataInstance from .large_file import LargeFile diff --git a/great_ai/src/great_ai/large_file/large_file/large_file_mongo.py b/great_ai/src/great_ai/large_file/large_file/large_file_mongo.py index f80c4ef..4cf2118 100644 --- a/great_ai/src/great_ai/large_file/large_file/large_file_mongo.py +++ b/great_ai/src/great_ai/large_file/large_file/large_file_mongo.py @@ -6,7 +6,7 @@ from typing import Any, List, Mapping from gridfs import DEFAULT_CHUNK_SIZE, Database, GridFSBucket from pymongo import MongoClient -from great_ai.utilities.logger import get_logger +from great_ai.utilities import get_logger from ..helper import DownloadProgressBar, UploadProgressBar from ..models import DataInstance diff --git a/great_ai/src/great_ai/large_file/large_file/large_file_s3.py b/great_ai/src/great_ai/large_file/large_file/large_file_s3.py index 35a02b7..25d8be9 100644 --- a/great_ai/src/great_ai/large_file/large_file/large_file_s3.py +++ b/great_ai/src/great_ai/large_file/large_file/large_file_s3.py @@ -4,7 +4,7 @@ from typing import Any, List, Mapping, Optional import boto3 -from great_ai.utilities.logger import get_logger +from great_ai.utilities import get_logger from ..helper import DownloadProgressBar, UploadProgressBar from ..models import DataInstance diff --git a/great_ai/src/great_ai/utilities/__init__.py b/great_ai/src/great_ai/utilities/__init__.py index 1eb71bc..7da0959 100644 --- a/great_ai/src/great_ai/utilities/__init__.py +++ b/great_ai/src/great_ai/utilities/__init__.py @@ -1,13 +1,14 @@ from .clean import clean +from .config_file import ConfigFile from .evaluate_ranking import evaluate_ranking from .get_sentences import get_sentences from .language import english_name_of_language, is_english, predict_language from .lemmatize_text import lemmatize_text from .lemmatize_token import lemmatize_token +from .logger import get_logger from .match_names import match_names from .matcher import fast_tokenize, filter_sentences, normalize from .nlp import nlp from .parallel_map import parallel_map from .publication_tei import PublicationTEI from .unique import unique -from .config_file import ConfigFile diff --git a/great_ai/src/great_ai/utilities/config_file/__init__.py b/great_ai/src/great_ai/utilities/config_file/__init__.py index d749ca0..6eabc46 100644 --- a/great_ai/src/great_ai/utilities/config_file/__init__.py +++ b/great_ai/src/great_ai/utilities/config_file/__init__.py @@ -1,2 +1,2 @@ from .config_file import ConfigFile -from .parse_error import ParseError \ No newline at end of file +from .parse_error import ParseError diff --git a/great_ai/src/great_ai/utilities/config_file/config_file.py b/great_ai/src/great_ai/utilities/config_file/config_file.py index e7a0af2..c6be097 100644 --- a/great_ai/src/great_ai/utilities/config_file/config_file.py +++ b/great_ai/src/great_ai/utilities/config_file/config_file.py @@ -1,52 +1,87 @@ -from pathlib import Path import os -from typing import Dict, Union +from pathlib import Path +from typing import Dict, Iterable, Tuple, Union from ..logger import get_logger from .parse_error import ParseError from .pattern import pattern -ENVIRONMENT_VARIABLE_KEY_PREFIX = 'ENV' +ENVIRONMENT_VARIABLE_KEY_PREFIX = "ENV" + +logger = get_logger("ConfigFile") -logger = get_logger('ConfigFile') class ConfigFile: - def __init__(self, path: Union[Path, str], ignore_missing_environment_variables:bool=False) -> None: + def __init__( + self, path: Union[Path, str], ignore_missing_environment_variables: bool = False + ) -> None: if not isinstance(path, Path): path = Path(path) + + if not path.exists(): + raise FileNotFoundError(path.absolute()) + self._path = path - self._ignore_missing_environment_variables = ignore_missing_environment_variables + self._ignore_missing_environment_variables = ( + ignore_missing_environment_variables + ) self._key_values: Dict[str, str] = {} self._parse() def _parse(self): - with open(self._path, encoding='utf-8') as f: + with open(self._path, encoding="utf-8") as f: lines: str = f.read() matches = pattern.findall(lines) for key, *values in matches: if key in self._key_values: - raise KeyError(f'Key `{key}` has been already defined and its value is `{value}`') + raise KeyError( + f"Key `{key}` has been already defined and its value is `{self._key_values[key]}`" + ) try: value = next(v for v in values if v) except StopIteration: - raise ParseError(f'Cannot parse config file ({self._path.absolute()}), error at key `{key}`') + raise ParseError( + f"Cannot parse config file ({self._path.absolute()}), error at key `{key}`" + ) - if value.startswith(f'{ENVIRONMENT_VARIABLE_KEY_PREFIX}:'): - _, value = value.split(':') + if value.startswith(f"{ENVIRONMENT_VARIABLE_KEY_PREFIX}:"): + _, value = value.split(":") if value not in os.environ: issue = f'The value of `{key}` contains the "{ENVIRONMENT_VARIABLE_KEY_PREFIX}` prefix but `{value}` is not defined as an environment variable' if self._ignore_missing_environment_variables: - logger.warning(f'{issue}, defaulting to `None`') + logger.warning(f"{issue}, defaulting to `None`") else: raise KeyError(issue) value = os.environ[value] - + self._key_values[key] = value def __getattr__(self, key: str) -> str: if key in self._key_values: return self._key_values[key] - raise KeyError(f'Key `{key}` is not found in configuration file ({self._path.absolute()})') + raise KeyError( + f"Key `{key}` is not found in configuration file ({self._path.absolute()})" + ) + + __getitem__ = __getattr__ + + def __iter__(self) -> Iterable[Tuple[str, str]]: + return iter(self._key_values) + + def __len__(self) -> int: + return len(self._key_values) + + def keys(self): + return self._key_values.keys() + + def values(self): + return self._key_values.values() + + def items(self): + return self._key_values.items() + + def __repr__(self): + return f"{type(self).__name__}(\n path={self._path},\n ignore_missing_environment_variables={self._ignore_missing_environment_variables}\n) {{{self._key_values}}}" diff --git a/great_ai/src/great_ai/utilities/config_file/parse_error.py b/great_ai/src/great_ai/utilities/config_file/parse_error.py index 1564ad7..831cca9 100644 --- a/great_ai/src/great_ai/utilities/config_file/parse_error.py +++ b/great_ai/src/great_ai/utilities/config_file/parse_error.py @@ -1,3 +1,2 @@ - class ParseError(Exception): pass diff --git a/great_ai/src/great_ai/utilities/config_file/pattern.py b/great_ai/src/great_ai/utilities/config_file/pattern.py index d05c9c3..f31f09d 100644 --- a/great_ai/src/great_ai/utilities/config_file/pattern.py +++ b/great_ai/src/great_ai/utilities/config_file/pattern.py @@ -1,7 +1,7 @@ import re - -pattern = re.compile(r''' +pattern = re.compile( + r""" \s* # leading whitespace is allowed (\w+?) # then comes the key \s*=\s* # the key and value are separated by an equal sign @@ -13,4 +13,6 @@ pattern = re.compile(r''' ) \s*(?:\#.*)? # comments can be added with the `#` symbol (?:\n|$) # a key-value pairs are separated by new lines -''', flags=re.UNICODE | re.VERBOSE) +""", + flags=re.UNICODE | re.VERBOSE, +) diff --git a/great_ai/tests/utilities/data/good.conf b/great_ai/tests/utilities/data/good.conf index e5e4c2c..ccca226 100644 --- a/great_ai/tests/utilities/data/good.conf +++ b/great_ai/tests/utilities/data/good.conf @@ -1,6 +1,6 @@ zeroth_key="test" -first_key=test +first_key= András second_key = test 2 # comment is ignored third_key= "test= 2==" diff --git a/great_ai/tests/utilities/data/parsed.py b/great_ai/tests/utilities/data/parsed.py index b861406..2b46fb0 100644 --- a/great_ai/tests/utilities/data/parsed.py +++ b/great_ai/tests/utilities/data/parsed.py @@ -1,4 +1,4 @@ -from great_ai.utilities.publication_tei.models import ( +from great_ai.utilities.publication_tei import ( Affiliation, Author, Bookmark, diff --git a/great_ai/tests/utilities/test_clean.py b/great_ai/tests/utilities/test_clean.py index 69355d6..500cada 100644 --- a/great_ai/tests/utilities/test_clean.py +++ b/great_ai/tests/utilities/test_clean.py @@ -1,6 +1,6 @@ import unittest -from src.great_ai.utilities.clean import clean +from src.great_ai.utilities import clean class TestClean(unittest.TestCase): diff --git a/great_ai/tests/utilities/test_config_file.py b/great_ai/tests/utilities/test_config_file.py index 2ea1733..aed18e3 100644 --- a/great_ai/tests/utilities/test_config_file.py +++ b/great_ai/tests/utilities/test_config_file.py @@ -1,7 +1,9 @@ import unittest from pathlib import Path + import pytest -from src.great_ai.utilities.config_file import ConfigFile + +from src.great_ai.utilities import ConfigFile DATA_PATH = Path(__file__).parent.resolve() / "data" @@ -9,38 +11,60 @@ DATA_PATH = Path(__file__).parent.resolve() / "data" class TestConfigFile(unittest.TestCase): def test_simple(self) -> None: c = ConfigFile(DATA_PATH / "good.conf") - assert c.zeroth_key == 'test' - assert c.first_key == 'test' - assert c.second_key == 'test 2' - assert c.third_key == 'test= 2==' - assert c.fourth_key == ''' + assert c.zeroth_key == "test" + assert c.first_key == "András" + assert c.second_key == "test 2" + assert c.third_key == "test= 2==" + assert ( + c.fourth_key + == """ this# is multiline -''' - assert c.whitespace == 'hardly matters' - +""" + ) + assert c.whitespace == "hardly matters" + + def test_simple_dict(self) -> None: + c = ConfigFile(DATA_PATH / "good.conf") + assert c["zeroth_key"] == "test" + assert c["first_key"] == "András" + assert c["second_key"] == "test 2" + assert c["third_key"] == "test= 2==" + assert ( + c["fourth_key"] + == """ +this# +is +multiline +""" + ) + assert c["whitespace"] == "hardly matters" def test_string_path(self) -> None: c = ConfigFile(str(DATA_PATH / "good.conf")) - assert c.zeroth_key == 'test' - assert c.first_key == 'test' - assert c.second_key == 'test 2' - assert c.third_key == 'test= 2==' - assert c.fourth_key == ''' + assert c.zeroth_key == "test" + assert c.first_key == "András" + assert c.second_key == "test 2" + assert c.third_key == "test= 2==" + assert ( + c.fourth_key + == """ this# is multiline -''' - assert c.whitespace == 'hardly matters' +""" + ) + assert c.whitespace == "hardly matters" def test_env(self) -> None: import os - os.environ['alma'] = '12' + + os.environ["alma"] = "12" c = ConfigFile(DATA_PATH / "env.conf") - del os.environ['alma'] - assert c.first_key == 'test' - assert c.second_key == '12' + del os.environ["alma"] + assert c.first_key == "test" + assert c.second_key == "12" def test_env_not_exists(self) -> None: with pytest.raises(KeyError): @@ -48,9 +72,10 @@ multiline def test_env_not_exists_but_ignored(self) -> None: with pytest.raises(KeyError): - ConfigFile(DATA_PATH / "env.conf", ignore_missing_environment_variables=True) + ConfigFile( + DATA_PATH / "env.conf", ignore_missing_environment_variables=True + ) def test_duplicate_key(self) -> None: with pytest.raises(KeyError): ConfigFile(DATA_PATH / "bad.conf") - \ No newline at end of file diff --git a/great_ai/tests/utilities/test_evaluate_ranking.py b/great_ai/tests/utilities/test_evaluate_ranking.py index d3ca6bf..60d8226 100644 --- a/great_ai/tests/utilities/test_evaluate_ranking.py +++ b/great_ai/tests/utilities/test_evaluate_ranking.py @@ -5,7 +5,7 @@ import matplotlib matplotlib.use("Agg") # don't show a window for each test -from src.great_ai.utilities.evaluate_ranking import evaluate_ranking +from src.great_ai.utilities import evaluate_ranking class TestEvaluateRanking(unittest.TestCase): diff --git a/great_ai/tests/utilities/test_get_sentences.py b/great_ai/tests/utilities/test_get_sentences.py index ed6e4c4..da2f772 100644 --- a/great_ai/tests/utilities/test_get_sentences.py +++ b/great_ai/tests/utilities/test_get_sentences.py @@ -1,6 +1,6 @@ import unittest -from src.great_ai.utilities.get_sentences import get_sentences +from src.great_ai.utilities import get_sentences class TestGetSentences(unittest.TestCase): diff --git a/great_ai/tests/utilities/test_language.py b/great_ai/tests/utilities/test_language.py index cbba2d9..aa3d986 100644 --- a/great_ai/tests/utilities/test_language.py +++ b/great_ai/tests/utilities/test_language.py @@ -1,6 +1,6 @@ import unittest -from src.great_ai.utilities.language import ( +from src.great_ai.utilities import ( english_name_of_language, is_english, predict_language, diff --git a/great_ai/tests/utilities/test_lemmatize_text.py b/great_ai/tests/utilities/test_lemmatize_text.py index 094dfac..315cf85 100644 --- a/great_ai/tests/utilities/test_lemmatize_text.py +++ b/great_ai/tests/utilities/test_lemmatize_text.py @@ -1,6 +1,6 @@ import unittest -from src.great_ai.utilities.lemmatize_text import lemmatize_text +from src.great_ai.utilities import lemmatize_text class TestLemmatizeText(unittest.TestCase): diff --git a/great_ai/tests/utilities/test_lemmatize_token.py b/great_ai/tests/utilities/test_lemmatize_token.py index a828ea5..308e612 100644 --- a/great_ai/tests/utilities/test_lemmatize_token.py +++ b/great_ai/tests/utilities/test_lemmatize_token.py @@ -1,7 +1,6 @@ import unittest -from src.great_ai.utilities.lemmatize_text import lemmatize_token -from src.great_ai.utilities.nlp import nlp +from src.great_ai.utilities import lemmatize_token, nlp class TestLemmatizeToken(unittest.TestCase): diff --git a/great_ai/tests/utilities/test_match_names.py b/great_ai/tests/utilities/test_match_names.py index b4e5a4e..5ee8d39 100644 --- a/great_ai/tests/utilities/test_match_names.py +++ b/great_ai/tests/utilities/test_match_names.py @@ -1,6 +1,6 @@ import unittest -from src.great_ai.utilities.match_names.match_names import match_names +from src.great_ai.utilities import match_names class TestMatchNames(unittest.TestCase): diff --git a/great_ai/tests/utilities/test_parallel_map.py b/great_ai/tests/utilities/test_parallel_map.py index c24f721..0a363df 100644 --- a/great_ai/tests/utilities/test_parallel_map.py +++ b/great_ai/tests/utilities/test_parallel_map.py @@ -1,6 +1,6 @@ import unittest -from src.great_ai.utilities.parallel_map import parallel_map +from src.great_ai.utilities import parallel_map COUNT = int(1e5) + 3 diff --git a/great_ai/tests/utilities/test_publication_tei.py b/great_ai/tests/utilities/test_publication_tei.py index 75593ac..35c84c6 100644 --- a/great_ai/tests/utilities/test_publication_tei.py +++ b/great_ai/tests/utilities/test_publication_tei.py @@ -1,7 +1,7 @@ import unittest from pathlib import Path -from src.great_ai.utilities.publication_tei import PublicationTEI +from src.great_ai.utilities import PublicationTEI from .data.parsed import ( abstract, diff --git a/great_ai/tests/utilities/test_unique.py b/great_ai/tests/utilities/test_unique.py index 42e1708..e2204db 100644 --- a/great_ai/tests/utilities/test_unique.py +++ b/great_ai/tests/utilities/test_unique.py @@ -1,6 +1,6 @@ import unittest -from src.great_ai.utilities.unique import unique +from src.great_ai.utilities import unique original = [ ("a", 1),