From 9b8b288ba600ae501b6646948a46968c473e9459 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 25 Jun 2022 10:38:39 +0200 Subject: [PATCH] Add ConfigFile --- great_ai/src/great_ai/utilities/__init__.py | 1 + .../utilities/config_file/__init__.py | 2 + .../utilities/config_file/config_file.py | 52 +++++++++++++++++ .../utilities/config_file/parse_error.py | 3 + .../great_ai/utilities/config_file/pattern.py | 16 ++++++ great_ai/tests/utilities/data/bad.conf | 3 + great_ai/tests/utilities/data/env.conf | 5 ++ great_ai/tests/utilities/data/good.conf | 15 +++++ great_ai/tests/utilities/test_config_file.py | 56 +++++++++++++++++++ 9 files changed, 153 insertions(+) create mode 100644 great_ai/src/great_ai/utilities/config_file/__init__.py create mode 100644 great_ai/src/great_ai/utilities/config_file/config_file.py create mode 100644 great_ai/src/great_ai/utilities/config_file/parse_error.py create mode 100644 great_ai/src/great_ai/utilities/config_file/pattern.py create mode 100644 great_ai/tests/utilities/data/bad.conf create mode 100644 great_ai/tests/utilities/data/env.conf create mode 100644 great_ai/tests/utilities/data/good.conf create mode 100644 great_ai/tests/utilities/test_config_file.py diff --git a/great_ai/src/great_ai/utilities/__init__.py b/great_ai/src/great_ai/utilities/__init__.py index 1e9de78..1eb71bc 100644 --- a/great_ai/src/great_ai/utilities/__init__.py +++ b/great_ai/src/great_ai/utilities/__init__.py @@ -10,3 +10,4 @@ from .nlp import nlp from .parallel_map import parallel_map from .publication_tei import PublicationTEI from .unique import unique +from .config_file import ConfigFile diff --git a/great_ai/src/great_ai/utilities/config_file/__init__.py b/great_ai/src/great_ai/utilities/config_file/__init__.py new file mode 100644 index 0000000..d749ca0 --- /dev/null +++ b/great_ai/src/great_ai/utilities/config_file/__init__.py @@ -0,0 +1,2 @@ +from .config_file import ConfigFile +from .parse_error import ParseError \ No newline at end of file diff --git a/great_ai/src/great_ai/utilities/config_file/config_file.py b/great_ai/src/great_ai/utilities/config_file/config_file.py new file mode 100644 index 0000000..e7a0af2 --- /dev/null +++ b/great_ai/src/great_ai/utilities/config_file/config_file.py @@ -0,0 +1,52 @@ +from pathlib import Path +import os +from typing import Dict, Union + +from ..logger import get_logger +from .parse_error import ParseError +from .pattern import pattern + +ENVIRONMENT_VARIABLE_KEY_PREFIX = 'ENV' + +logger = get_logger('ConfigFile') + +class ConfigFile: + def __init__(self, path: Union[Path, str], ignore_missing_environment_variables:bool=False) -> None: + if not isinstance(path, Path): + path = Path(path) + self._path = path + self._ignore_missing_environment_variables = ignore_missing_environment_variables + self._key_values: Dict[str, str] = {} + + self._parse() + + def _parse(self): + with open(self._path, encoding='utf-8') as f: + lines: str = f.read() + + matches = pattern.findall(lines) + for key, *values in matches: + if key in self._key_values: + raise KeyError(f'Key `{key}` has been already defined and its value is `{value}`') + + try: + value = next(v for v in values if v) + except StopIteration: + raise ParseError(f'Cannot parse config file ({self._path.absolute()}), error at key `{key}`') + + if value.startswith(f'{ENVIRONMENT_VARIABLE_KEY_PREFIX}:'): + _, value = value.split(':') + if value not in os.environ: + issue = f'The value of `{key}` contains the "{ENVIRONMENT_VARIABLE_KEY_PREFIX}` prefix but `{value}` is not defined as an environment variable' + if self._ignore_missing_environment_variables: + logger.warning(f'{issue}, defaulting to `None`') + else: + raise KeyError(issue) + value = os.environ[value] + + self._key_values[key] = value + + def __getattr__(self, key: str) -> str: + if key in self._key_values: + return self._key_values[key] + raise KeyError(f'Key `{key}` is not found in configuration file ({self._path.absolute()})') diff --git a/great_ai/src/great_ai/utilities/config_file/parse_error.py b/great_ai/src/great_ai/utilities/config_file/parse_error.py new file mode 100644 index 0000000..1564ad7 --- /dev/null +++ b/great_ai/src/great_ai/utilities/config_file/parse_error.py @@ -0,0 +1,3 @@ + +class ParseError(Exception): + pass diff --git a/great_ai/src/great_ai/utilities/config_file/pattern.py b/great_ai/src/great_ai/utilities/config_file/pattern.py new file mode 100644 index 0000000..d05c9c3 --- /dev/null +++ b/great_ai/src/great_ai/utilities/config_file/pattern.py @@ -0,0 +1,16 @@ +import re + + +pattern = re.compile(r''' + \s* # leading whitespace is allowed + (\w+?) # then comes the key + \s*=\s* # the key and value are separated by an equal sign + (?: # then comes the value + "([^"]*)" # the value can be surrounded by quotes: "value" + | '([^']*)' # the value can be surrounded by quotes: 'value' + | `([^`]*)` # the value can be surrounded by quotes: `value` + | ([^#\n]*?) # or it is bare, in that case, the trailing whitespace is ignored + ) + \s*(?:\#.*)? # comments can be added with the `#` symbol + (?:\n|$) # a key-value pairs are separated by new lines +''', flags=re.UNICODE | re.VERBOSE) diff --git a/great_ai/tests/utilities/data/bad.conf b/great_ai/tests/utilities/data/bad.conf new file mode 100644 index 0000000..1d45da0 --- /dev/null +++ b/great_ai/tests/utilities/data/bad.conf @@ -0,0 +1,3 @@ + +zeroth_key="test" +zeroth_key="test" diff --git a/great_ai/tests/utilities/data/env.conf b/great_ai/tests/utilities/data/env.conf new file mode 100644 index 0000000..ca78f66 --- /dev/null +++ b/great_ai/tests/utilities/data/env.conf @@ -0,0 +1,5 @@ + +first_key=test + +second_key="ENV:alma" +third_key = ENV:alma diff --git a/great_ai/tests/utilities/data/good.conf b/great_ai/tests/utilities/data/good.conf new file mode 100644 index 0000000..e5e4c2c --- /dev/null +++ b/great_ai/tests/utilities/data/good.conf @@ -0,0 +1,15 @@ + +zeroth_key="test" +first_key=test +second_key = test 2 # comment is ignored +third_key= "test= 2==" + +fourth_key = " +this# +is +multiline +" # cannot write comments inside multiline + + whitespace = hardly matters + + diff --git a/great_ai/tests/utilities/test_config_file.py b/great_ai/tests/utilities/test_config_file.py new file mode 100644 index 0000000..2ea1733 --- /dev/null +++ b/great_ai/tests/utilities/test_config_file.py @@ -0,0 +1,56 @@ +import unittest +from pathlib import Path +import pytest +from src.great_ai.utilities.config_file import ConfigFile + +DATA_PATH = Path(__file__).parent.resolve() / "data" + + +class TestConfigFile(unittest.TestCase): + def test_simple(self) -> None: + c = ConfigFile(DATA_PATH / "good.conf") + assert c.zeroth_key == 'test' + assert c.first_key == 'test' + assert c.second_key == 'test 2' + assert c.third_key == 'test= 2==' + assert c.fourth_key == ''' +this# +is +multiline +''' + assert c.whitespace == 'hardly matters' + + + def test_string_path(self) -> None: + c = ConfigFile(str(DATA_PATH / "good.conf")) + assert c.zeroth_key == 'test' + assert c.first_key == 'test' + assert c.second_key == 'test 2' + assert c.third_key == 'test= 2==' + assert c.fourth_key == ''' +this# +is +multiline +''' + assert c.whitespace == 'hardly matters' + + def test_env(self) -> None: + import os + os.environ['alma'] = '12' + c = ConfigFile(DATA_PATH / "env.conf") + del os.environ['alma'] + assert c.first_key == 'test' + assert c.second_key == '12' + + def test_env_not_exists(self) -> None: + with pytest.raises(KeyError): + ConfigFile(DATA_PATH / "env.conf") + + def test_env_not_exists_but_ignored(self) -> None: + with pytest.raises(KeyError): + ConfigFile(DATA_PATH / "env.conf", ignore_missing_environment_variables=True) + + def test_duplicate_key(self) -> None: + with pytest.raises(KeyError): + ConfigFile(DATA_PATH / "bad.conf") + \ No newline at end of file