Add ConfigFile

great_ai/src/great_ai/utilities/__init__.py

@@ -10,3 +10,4 @@ from .nlp import nlp
 from .parallel_map import parallel_map
 from .publication_tei import PublicationTEI
 from .unique import unique
+from .config_file import ConfigFile

great_ai/src/great_ai/utilities/config_file/__init__.py

@@ -0,0 +1,2 @@
+from .config_file import ConfigFile
+from .parse_error import ParseError

great_ai/src/great_ai/utilities/config_file/config_file.py

@@ -0,0 +1,52 @@
+from pathlib import Path
+import os
+from typing import Dict, Union
+
+from ..logger import get_logger
+from .parse_error import ParseError
+from .pattern import pattern
+
+ENVIRONMENT_VARIABLE_KEY_PREFIX = 'ENV'
+
+logger = get_logger('ConfigFile')
+
+class ConfigFile:
+    def __init__(self, path: Union[Path, str], ignore_missing_environment_variables:bool=False) -> None:
+        if not isinstance(path, Path):
+            path = Path(path)
+        self._path = path
+        self._ignore_missing_environment_variables = ignore_missing_environment_variables
+        self._key_values: Dict[str, str] = {}
+
+        self._parse()
+
+    def _parse(self):
+        with open(self._path, encoding='utf-8') as f:
+            lines: str = f.read()
+
+        matches = pattern.findall(lines)
+        for key, *values in matches:
+            if key in self._key_values:
+                raise KeyError(f'Key `{key}` has been already defined and its value is `{value}`')
+
+            try:
+                value = next(v for v in values if v)
+            except StopIteration:
+                raise ParseError(f'Cannot parse config file ({self._path.absolute()}), error at key `{key}`')
+
+            if value.startswith(f'{ENVIRONMENT_VARIABLE_KEY_PREFIX}:'):
+                _, value = value.split(':')
+                if value not in os.environ:
+                    issue = f'The value of `{key}` contains the "{ENVIRONMENT_VARIABLE_KEY_PREFIX}` prefix but `{value}` is not defined as an environment variable'
+                    if self._ignore_missing_environment_variables:
+                        logger.warning(f'{issue}, defaulting to `None`')
+                    else:
+                        raise KeyError(issue)
+                value = os.environ[value]
+            
+            self._key_values[key] = value
+
+    def __getattr__(self, key: str) -> str:
+        if key in self._key_values:
+            return self._key_values[key]
+        raise KeyError(f'Key `{key}` is not found in configuration file ({self._path.absolute()})')

great_ai/src/great_ai/utilities/config_file/parse_error.py

@@ -0,0 +1,3 @@
+
+class ParseError(Exception):
+    pass

great_ai/src/great_ai/utilities/config_file/pattern.py

@@ -0,0 +1,16 @@
+import re
+
+
+pattern = re.compile(r'''
+    \s*             # leading whitespace is allowed
+    (\w+?)          # then comes the key
+    \s*=\s*         # the key and value are separated by an equal sign
+    (?:             # then comes the value
+         "([^"]*)"     # the value can be surrounded by quotes: "value"
+       | '([^']*)'     # the value can be surrounded by quotes: 'value'
+       | `([^`]*)`     # the value can be surrounded by quotes: `value`
+       | ([^#\n]*?)    # or it is bare, in that case, the trailing whitespace is ignored
+    )
+    \s*(?:\#.*)?        # comments can be added with the `#` symbol
+    (?:\n|$)            # a key-value pairs are separated by new lines
+''', flags=re.UNICODE | re.VERBOSE)

great_ai/tests/utilities/data/bad.conf

@@ -0,0 +1,3 @@
+
+zeroth_key="test"
+zeroth_key="test"

great_ai/tests/utilities/data/env.conf

@@ -0,0 +1,5 @@
+
+first_key=test
+
+second_key="ENV:alma"
+third_key = ENV:alma 

great_ai/tests/utilities/data/good.conf

@@ -0,0 +1,15 @@
+
+zeroth_key="test"
+first_key=test
+second_key = test 2 # comment is ignored
+third_key= "test= 2=="
+
+fourth_key = "
+this#
+is
+multiline
+" # cannot write comments inside multiline
+
+ whitespace =  hardly  matters 
+
+

great_ai/tests/utilities/test_config_file.py

@@ -0,0 +1,56 @@
+import unittest
+from pathlib import Path
+import pytest
+from src.great_ai.utilities.config_file import ConfigFile
+
+DATA_PATH = Path(__file__).parent.resolve() / "data"
+
+
+class TestConfigFile(unittest.TestCase):
+    def test_simple(self) -> None:
+        c = ConfigFile(DATA_PATH / "good.conf")
+        assert c.zeroth_key == 'test'
+        assert c.first_key == 'test'
+        assert c.second_key == 'test 2'
+        assert c.third_key == 'test= 2=='
+        assert c.fourth_key == '''
+this#
+is
+multiline
+'''
+        assert c.whitespace == 'hardly  matters'
+        
+
+    def test_string_path(self) -> None:
+        c = ConfigFile(str(DATA_PATH / "good.conf"))
+        assert c.zeroth_key == 'test'
+        assert c.first_key == 'test'
+        assert c.second_key == 'test 2'
+        assert c.third_key == 'test= 2=='
+        assert c.fourth_key == '''
+this#
+is
+multiline
+'''
+        assert c.whitespace == 'hardly  matters'
+
+    def test_env(self) -> None:
+        import os
+        os.environ['alma'] = '12'
+        c = ConfigFile(DATA_PATH / "env.conf")
+        del os.environ['alma']
+        assert c.first_key == 'test'
+        assert c.second_key == '12'
+
+    def test_env_not_exists(self) -> None:
+        with pytest.raises(KeyError):
+            ConfigFile(DATA_PATH / "env.conf")
+
+    def test_env_not_exists_but_ignored(self) -> None:
+        with pytest.raises(KeyError):
+            ConfigFile(DATA_PATH / "env.conf", ignore_missing_environment_variables=True)
+
+    def test_duplicate_key(self) -> None:
+        with pytest.raises(KeyError):
+            ConfigFile(DATA_PATH / "bad.conf")
+