Integrate new large_files

great_ai/open_s3.md

@@ -15,9 +15,9 @@ pip install open-large
 ### Simple example
 
 ```python
-from open_s3 import LargeFile
+from large_file import LargeFileS3
 
-LargeFile.configure_credentials({
+LargeFileS3.configure_credentials({
     "aws_region_name": "your_region_like_eu-west-2",
     "aws_access_key_id": "YOUR_ACCESS_KEY_ID",
     "aws_secret_access_key": "YOUR_VERY_SECRET_ACCESS_KEY",
@@ -25,13 +25,13 @@ LargeFile.configure_credentials({
 })
 
 # Creates a new version and deletes the older version leaving the 3 most recently used intact
-with LargeFile("test.txt", "w", keep_last_n=3) as f:
+with LargeFileS3("test.txt", "w", keep_last_n=3) as f:
     for i in range(100000):
         f.write('test\n')
 
 # By default the latest version is returned
 # but an optional `version` keyword argument can be provided as well
-with LargeFile("test.txt", "r") as f:
+with LargeFileS3("test.txt", "r") as f:
     print(f.readlines()[0])
 ```
 
@@ -91,13 +91,13 @@ endpoint_url = this is optional, for backblaze, use this: https://s3.us-west-002
 #### Print the expected options
 
 ```sh
-python3 -m open_s3 --help
+python3 -m large_file --help
 ```
 
 #### Upload some files
 
 ```sh
-python3 -m open_s3 --secrets secrets.ini --push my_first_file.json folder/my_second_file my_folder
+python3 -m large_file --backend s3 --secrets secrets.ini --push my_first_file.json folder/my_second_file my_folder
 ```
 
 > Only the filename is used as the S3 name, the rest of the path is ignored.
@@ -107,7 +107,7 @@ python3 -m open_s3 --secrets secrets.ini --push my_first_file.json folder/my_sec
 This can be useful when building a Docker image for example. This way, the files can already reside inside the container and need not be downloaded later.
 
 ```sh
-python3 -m open_s3 --secrets ~/.aws/credentials --cache my_first_file.json:3 my_second_file my_folder:0
+python3 -m large_file --backend s3 -secrets ~/.aws/credentials --cache my_first_file.json:3 my_second_file my_folder:0
 ```
 
 > Versions may be specified by using `:`-s.
@@ -115,5 +115,5 @@ python3 -m open_s3 --secrets ~/.aws/credentials --cache my_first_file.json:3 my_
 #### Delete remote files
 
 ```sh
-python3 -m open_s3 --secrets ~/.aws/credentials --delete my_first_file.json
+python3 -m large_file --backend s3 --secrets ~/.aws/credentials --delete my_first_file.json
 ```

great_ai/setup.cfg

@@ -44,6 +44,7 @@ install_requires =
     dash >= 2.4.0
     uvicorn[standard] >= 0.17.0
     watchdog >= 2.1.0
+    pymongo >= 4.0.0
 
 [options.package_data]
 * = *.json, *.yaml, *.yml, *.css

great_ai/src/great_ai/great_ai/constants.py

@@ -1,4 +1,14 @@
+from pathlib import Path
+
+from great_ai.large_file import LargeFileLocal, LargeFileMongo, LargeFileS3
+
 ENV_VAR_KEY = "ENVIRONMENT"
 PRODUCTION_KEY = "production"
 DEFAULT_TRACING_DB_FILENAME = "tracing_database.json"
 METRICS_PATH = "/metrics"
+
+DEFAULT_LARGE_FILE_CONFIG_PATHS = {
+    LargeFileLocal: None,
+    LargeFileMongo: Path("mongodb.ini"),
+    LargeFileS3: Path("s3.ini"),
+}

great_ai/src/great_ai/great_ai/context/configure.py

@@ -2,24 +2,29 @@ import os
 import random
 from logging import DEBUG, Logger
 from pathlib import Path
-from typing import Optional
+from typing import Optional, Type
 
 import great_ai.great_ai.context.context as context
-from great_ai.open_s3 import LargeFile
+from great_ai.large_file import LargeFile, LargeFileLocal
 from great_ai.utilities.logger import get_logger
 
-from ..constants import DEFAULT_TRACING_DB_FILENAME, ENV_VAR_KEY, PRODUCTION_KEY
-from ..persistence import ParallelTinyDbDriver, PersistenceDriver
+from ..constants import (
+    DEFAULT_LARGE_FILE_CONFIG_PATHS,
+    DEFAULT_TRACING_DB_FILENAME,
+    ENV_VAR_KEY,
+    PRODUCTION_KEY,
+)
+from ..tracing.parallel_tinydb_driver import ParallelTinyDbDriver, TracingDatabase
 
 
 def configure(
     version: str = "0.0.1",
     log_level: int = DEBUG,
-    s3_config: Path = Path("s3.ini"),
     seed: int = 42,
-    persistence_driver: PersistenceDriver = ParallelTinyDbDriver(
+    tracing_database: TracingDatabase = ParallelTinyDbDriver(
         Path(DEFAULT_TRACING_DB_FILENAME)
     ),
+    large_file_implementation: Type[LargeFile] = LargeFileLocal,
     should_log_exception_stack: Optional[bool] = None,
     prediction_cache_size: int = 512,
 ) -> None:
@@ -28,21 +33,22 @@ def configure(
     if context._context is not None:
         logger.warn(
             "Configuration has been already initialised, overwriting.\n"
-            + 'Make sure to call "configure()" before importing your application code.'
+            + "Make sure to call `configure()` before importing your application code."
         )
 
     is_production = _is_in_production_mode(logger=logger)
-    _initialize_large_file(s3_config, logger=logger)
+    _initialize_large_file(large_file_implementation, logger=logger)
     _set_seed(seed)
 
-    if not persistence_driver.is_threadsafe:
+    if not tracing_database.is_threadsafe:
         logger.warning(
-            f"The selected persistence driver ({type(persistence_driver).__name__}) is not threadsafe"
+            f"The selected persistence driver ({type(tracing_database).__name__}) is not threadsafe"
         )
 
     context._context = context.Context(
         version=version,
-        persistence=persistence_driver,
+        tracing_database=tracing_database,
+        large_file_implementation=large_file_implementation,
         is_production=is_production,
         logger=logger,
         should_log_exception_stack=not is_production
@@ -77,12 +83,23 @@ def _is_in_production_mode(logger: Optional[Logger]) -> bool:
     return is_production
 
 
-def _initialize_large_file(s3_config: Path, logger: Logger) -> None:
-    if s3_config.exists():
-        LargeFile.configure_credentials_from_file(s3_config)
+def _initialize_large_file(large_file: Type[LargeFile], logger: Logger) -> None:
+    path = DEFAULT_LARGE_FILE_CONFIG_PATHS[large_file]
+    if path is None:
+        return
+
+    if large_file.initialized:
+        logger.warning(
+            f"{large_file.__name__} has been already configured: skipping initialisation"
+        )
+        return
+
+    if path.exists():
+        large_file.configure_credentials_from_file(path)
+        logger.info(f"{large_file.__name__} initialised with config ({path.resolve()})")
     else:
         logger.warning(
-            f"Provided S3 config ({s3_config.resolve()}) not found, skipping LargeFile initialisation"
+            f"Default {large_file.__name__} config ({path.resolve()}) not found, skipping {large_file.__name__} initialisation"
         )
 
 

great_ai/src/great_ai/great_ai/context/context.py

@@ -1,14 +1,17 @@
 from logging import Logger
-from typing import Optional
+from typing import Any, Dict, Optional, Type
 
 from pydantic import BaseModel
 
-from ..persistence import PersistenceDriver
+from great_ai.large_file.large_file.large_file import LargeFile
+
+from ..tracing.tracing_database import TracingDatabase
 
 
 class Context(BaseModel):
     version: str
-    persistence: PersistenceDriver
+    tracing_database: TracingDatabase
+    large_file_implementation: Type[LargeFile]
     is_production: bool
     logger: Logger
     should_log_exception_stack: bool
@@ -17,5 +20,15 @@ class Context(BaseModel):
     class Config:
         arbitrary_types_allowed = True
 
+    def to_flat_dict(self) -> Dict[str, Any]:
+        return {
+            "version": self.version,
+            "tracing_database": type(self.tracing_database).__name__,
+            "large_file_implementation": self.large_file_implementation.__name__,
+            "is_production": self.is_production,
+            "logger": type(self.logger).__name__,
+            "should_log_exception_stack": self.should_log_exception_stack,
+        }
+
 
 _context: Optional[Context] = None