From 6acfb6819e478b9924c2fb17f8b40c20c0c41940 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A1s=20Schmelczer?= Date: Fri, 3 Jun 2022 19:26:55 +0200 Subject: [PATCH] Integrate new large_files --- great_ai/open_s3.md | 16 +++---- great_ai/setup.cfg | 1 + great_ai/src/great_ai/great_ai/constants.py | 10 ++++ .../great_ai/great_ai/context/configure.py | 47 +++++++++++++------ .../src/great_ai/great_ai/context/context.py | 19 ++++++-- 5 files changed, 67 insertions(+), 26 deletions(-) diff --git a/great_ai/open_s3.md b/great_ai/open_s3.md index 8263b5b..49acb9e 100644 --- a/great_ai/open_s3.md +++ b/great_ai/open_s3.md @@ -15,9 +15,9 @@ pip install open-large ### Simple example ```python -from open_s3 import LargeFile +from large_file import LargeFileS3 -LargeFile.configure_credentials({ +LargeFileS3.configure_credentials({ "aws_region_name": "your_region_like_eu-west-2", "aws_access_key_id": "YOUR_ACCESS_KEY_ID", "aws_secret_access_key": "YOUR_VERY_SECRET_ACCESS_KEY", @@ -25,13 +25,13 @@ LargeFile.configure_credentials({ }) # Creates a new version and deletes the older version leaving the 3 most recently used intact -with LargeFile("test.txt", "w", keep_last_n=3) as f: +with LargeFileS3("test.txt", "w", keep_last_n=3) as f: for i in range(100000): f.write('test\n') # By default the latest version is returned # but an optional `version` keyword argument can be provided as well -with LargeFile("test.txt", "r") as f: +with LargeFileS3("test.txt", "r") as f: print(f.readlines()[0]) ``` @@ -91,13 +91,13 @@ endpoint_url = this is optional, for backblaze, use this: https://s3.us-west-002 #### Print the expected options ```sh -python3 -m open_s3 --help +python3 -m large_file --help ``` #### Upload some files ```sh -python3 -m open_s3 --secrets secrets.ini --push my_first_file.json folder/my_second_file my_folder +python3 -m large_file --backend s3 --secrets secrets.ini --push my_first_file.json folder/my_second_file my_folder ``` > Only the filename is used as the S3 name, the rest of the path is ignored. @@ -107,7 +107,7 @@ python3 -m open_s3 --secrets secrets.ini --push my_first_file.json folder/my_sec This can be useful when building a Docker image for example. This way, the files can already reside inside the container and need not be downloaded later. ```sh -python3 -m open_s3 --secrets ~/.aws/credentials --cache my_first_file.json:3 my_second_file my_folder:0 +python3 -m large_file --backend s3 -secrets ~/.aws/credentials --cache my_first_file.json:3 my_second_file my_folder:0 ``` > Versions may be specified by using `:`-s. @@ -115,5 +115,5 @@ python3 -m open_s3 --secrets ~/.aws/credentials --cache my_first_file.json:3 my_ #### Delete remote files ```sh -python3 -m open_s3 --secrets ~/.aws/credentials --delete my_first_file.json +python3 -m large_file --backend s3 --secrets ~/.aws/credentials --delete my_first_file.json ``` diff --git a/great_ai/setup.cfg b/great_ai/setup.cfg index 1a071a4..1ca9e8f 100644 --- a/great_ai/setup.cfg +++ b/great_ai/setup.cfg @@ -44,6 +44,7 @@ install_requires = dash >= 2.4.0 uvicorn[standard] >= 0.17.0 watchdog >= 2.1.0 + pymongo >= 4.0.0 [options.package_data] * = *.json, *.yaml, *.yml, *.css diff --git a/great_ai/src/great_ai/great_ai/constants.py b/great_ai/src/great_ai/great_ai/constants.py index 4b0c623..adca62b 100644 --- a/great_ai/src/great_ai/great_ai/constants.py +++ b/great_ai/src/great_ai/great_ai/constants.py @@ -1,4 +1,14 @@ +from pathlib import Path + +from great_ai.large_file import LargeFileLocal, LargeFileMongo, LargeFileS3 + ENV_VAR_KEY = "ENVIRONMENT" PRODUCTION_KEY = "production" DEFAULT_TRACING_DB_FILENAME = "tracing_database.json" METRICS_PATH = "/metrics" + +DEFAULT_LARGE_FILE_CONFIG_PATHS = { + LargeFileLocal: None, + LargeFileMongo: Path("mongodb.ini"), + LargeFileS3: Path("s3.ini"), +} diff --git a/great_ai/src/great_ai/great_ai/context/configure.py b/great_ai/src/great_ai/great_ai/context/configure.py index 9ac946d..66fd999 100644 --- a/great_ai/src/great_ai/great_ai/context/configure.py +++ b/great_ai/src/great_ai/great_ai/context/configure.py @@ -2,24 +2,29 @@ import os import random from logging import DEBUG, Logger from pathlib import Path -from typing import Optional +from typing import Optional, Type import great_ai.great_ai.context.context as context -from great_ai.open_s3 import LargeFile +from great_ai.large_file import LargeFile, LargeFileLocal from great_ai.utilities.logger import get_logger -from ..constants import DEFAULT_TRACING_DB_FILENAME, ENV_VAR_KEY, PRODUCTION_KEY -from ..persistence import ParallelTinyDbDriver, PersistenceDriver +from ..constants import ( + DEFAULT_LARGE_FILE_CONFIG_PATHS, + DEFAULT_TRACING_DB_FILENAME, + ENV_VAR_KEY, + PRODUCTION_KEY, +) +from ..tracing.parallel_tinydb_driver import ParallelTinyDbDriver, TracingDatabase def configure( version: str = "0.0.1", log_level: int = DEBUG, - s3_config: Path = Path("s3.ini"), seed: int = 42, - persistence_driver: PersistenceDriver = ParallelTinyDbDriver( + tracing_database: TracingDatabase = ParallelTinyDbDriver( Path(DEFAULT_TRACING_DB_FILENAME) ), + large_file_implementation: Type[LargeFile] = LargeFileLocal, should_log_exception_stack: Optional[bool] = None, prediction_cache_size: int = 512, ) -> None: @@ -28,21 +33,22 @@ def configure( if context._context is not None: logger.warn( "Configuration has been already initialised, overwriting.\n" - + 'Make sure to call "configure()" before importing your application code.' + + "Make sure to call `configure()` before importing your application code." ) is_production = _is_in_production_mode(logger=logger) - _initialize_large_file(s3_config, logger=logger) + _initialize_large_file(large_file_implementation, logger=logger) _set_seed(seed) - if not persistence_driver.is_threadsafe: + if not tracing_database.is_threadsafe: logger.warning( - f"The selected persistence driver ({type(persistence_driver).__name__}) is not threadsafe" + f"The selected persistence driver ({type(tracing_database).__name__}) is not threadsafe" ) context._context = context.Context( version=version, - persistence=persistence_driver, + tracing_database=tracing_database, + large_file_implementation=large_file_implementation, is_production=is_production, logger=logger, should_log_exception_stack=not is_production @@ -77,12 +83,23 @@ def _is_in_production_mode(logger: Optional[Logger]) -> bool: return is_production -def _initialize_large_file(s3_config: Path, logger: Logger) -> None: - if s3_config.exists(): - LargeFile.configure_credentials_from_file(s3_config) +def _initialize_large_file(large_file: Type[LargeFile], logger: Logger) -> None: + path = DEFAULT_LARGE_FILE_CONFIG_PATHS[large_file] + if path is None: + return + + if large_file.initialized: + logger.warning( + f"{large_file.__name__} has been already configured: skipping initialisation" + ) + return + + if path.exists(): + large_file.configure_credentials_from_file(path) + logger.info(f"{large_file.__name__} initialised with config ({path.resolve()})") else: logger.warning( - f"Provided S3 config ({s3_config.resolve()}) not found, skipping LargeFile initialisation" + f"Default {large_file.__name__} config ({path.resolve()}) not found, skipping {large_file.__name__} initialisation" ) diff --git a/great_ai/src/great_ai/great_ai/context/context.py b/great_ai/src/great_ai/great_ai/context/context.py index e3998a0..8fd3726 100644 --- a/great_ai/src/great_ai/great_ai/context/context.py +++ b/great_ai/src/great_ai/great_ai/context/context.py @@ -1,14 +1,17 @@ from logging import Logger -from typing import Optional +from typing import Any, Dict, Optional, Type from pydantic import BaseModel -from ..persistence import PersistenceDriver +from great_ai.large_file.large_file.large_file import LargeFile + +from ..tracing.tracing_database import TracingDatabase class Context(BaseModel): version: str - persistence: PersistenceDriver + tracing_database: TracingDatabase + large_file_implementation: Type[LargeFile] is_production: bool logger: Logger should_log_exception_stack: bool @@ -17,5 +20,15 @@ class Context(BaseModel): class Config: arbitrary_types_allowed = True + def to_flat_dict(self) -> Dict[str, Any]: + return { + "version": self.version, + "tracing_database": type(self.tracing_database).__name__, + "large_file_implementation": self.large_file_implementation.__name__, + "is_production": self.is_production, + "logger": type(self.logger).__name__, + "should_log_exception_stack": self.should_log_exception_stack, + } + _context: Optional[Context] = None