Move files

src/__main__.py

@@ -14,12 +14,13 @@ from uvicorn.supervisors.basereload import BaseReload
 from watchdog.events import FileSystemEvent, PatternMatchingEventHandler
 from watchdog.observers import Observer
 
+from utilities import get_logger
+
 from .great_ai.constants import SERVER_NAME
 from .great_ai.context import _is_in_production_mode
 from .great_ai.deploy import GreatAI
 from .great_ai.exceptions import ArgumentValidationError, MissingArgumentError
 from .parse_arguments import parse_arguments
-from .utilities import get_logger
 
 logger = get_logger(SERVER_NAME)
 

src/great_ai/__init__.py

@@ -1,3 +1,23 @@
-from .great_ai import *
+from .context import configure
-from .large_file import *
+from .deploy import GreatAI
-from .utilities import *
+from .exceptions import (
+    ArgumentValidationError,
+    MissingArgumentError,
+    WrongDecoratorOrderError,
+)
+from .models import save_model, use_model
+from .output_views import (
+    ClassificationOutput,
+    MultiLabelClassificationOutput,
+    RegressionOutput,
+)
+from .parameters import log_metric, parameter
+from .persistence import MongodbDriver, ParallelTinyDbDriver, TracingDatabaseDriver
+from .remote import (
+    HttpClient,
+    RemoteCallError,
+    call_remote_great_ai,
+    call_remote_great_ai_async,
+)
+from .tracing import add_ground_truth, delete_ground_truth, query_ground_truth
+from .views import Trace

src/great_ai/constants.py

@@ -1,4 +1,5 @@
-from ..large_file import LargeFileMongo, LargeFileS3
+from large_file import LargeFileMongo, LargeFileS3
+
 from .persistence.mongodb_driver import MongodbDriver
 
 ENV_VAR_KEY = "ENVIRONMENT"

src/great_ai/context.py

@@ -6,8 +6,9 @@ from typing import Any, Dict, Optional, Type, cast
 
 from pydantic import BaseModel
 
-from ..large_file import LargeFile, LargeFileLocal
+from large_file import LargeFileBase, LargeFileLocal
-from ..utilities import get_logger
+from utilities import get_logger
+
 from .constants import (
     DEFAULT_LARGE_FILE_CONFIG_PATHS,
     DEFAULT_TRACING_DATABASE_CONFIG_PATHS,
@@ -21,7 +22,7 @@ from .persistence import ParallelTinyDbDriver, TracingDatabaseDriver
 
 class Context(BaseModel):
     tracing_database: TracingDatabaseDriver
-    large_file_implementation: Type[LargeFile]
+    large_file_implementation: Type[LargeFileBase]
     is_production: bool
     logger: Logger
     should_log_exception_stack: bool
@@ -57,7 +58,7 @@ def configure(
     log_level: int = DEBUG,
     seed: int = 42,
     tracing_database: Optional[Type[TracingDatabaseDriver]] = None,
-    large_file_implementation: Optional[Type[LargeFile]] = None,
+    large_file_implementation: Optional[Type[LargeFileBase]] = None,
     should_log_exception_stack: Optional[bool] = None,
     prediction_cache_size: int = 512,
     disable_se4ml_banner: bool = False,
@@ -160,8 +161,8 @@ def _initialize_tracing_database(
 
 
 def _initialize_large_file(
-    selected: Optional[Type[LargeFile]], logger: Logger
+    selected: Optional[Type[LargeFileBase]], logger: Logger
-) -> Type[LargeFile]:
+) -> Type[LargeFileBase]:
     for large_file, paths in DEFAULT_LARGE_FILE_CONFIG_PATHS.items():
         if selected is None or selected == large_file:
             if large_file.initialized:

src/great_ai/deploy/great_ai.py

@@ -16,7 +16,8 @@ from typing import (
 from fastapi import APIRouter, FastAPI, status
 from pydantic import BaseModel, create_model
 
-from ...utilities import parallel_map
+from utilities import parallel_map
+
 from ..constants import DASHBOARD_PATH
 from ..context import get_context
 from ..helper import (

src/great_ai/deploy/routes/dashboard/create_dash_app.py

@@ -8,7 +8,8 @@ from dash import Dash, dcc, html
 from dash.dependencies import Input, Output
 from flask import Flask
 
-from .....utilities import unique
+from utilities import unique
+
 from ....constants import DASHBOARD_PATH, ONLINE_TAG_NAME
 from ....context import get_context
 from ....helper import freeze, snake_case_to_text, text_to_hex_color

src/great_ai/great_ai/__init__.py

@@ -1,23 +0,0 @@
-from .context import configure
-from .deploy import GreatAI
-from .exceptions import (
-    ArgumentValidationError,
-    MissingArgumentError,
-    WrongDecoratorOrderError,
-)
-from .models import save_model, use_model
-from .output_views import (
-    ClassificationOutput,
-    MultiLabelClassificationOutput,
-    RegressionOutput,
-)
-from .parameters import log_metric, parameter
-from .persistence import MongodbDriver, ParallelTinyDbDriver, TracingDatabaseDriver
-from .remote import (
-    HttpClient,
-    RemoteCallError,
-    call_remote_great_ai,
-    call_remote_great_ai_async,
-)
-from .tracing import add_ground_truth, delete_ground_truth, query_ground_truth
-from .views import Trace

src/great_ai/large_file/__init__.py

@@ -1 +0,0 @@
-from .large_file import LargeFile, LargeFileLocal, LargeFileMongo, LargeFileS3

src/great_ai/persistence/tracing_database_driver.py

@@ -3,7 +3,8 @@ from datetime import datetime
 from pathlib import Path
 from typing import List, Optional, Sequence, Tuple, Union
 
-from ...utilities import ConfigFile
+from utilities import ConfigFile
+
 from ..views import Filter, SortBy, Trace
 
 

src/great_ai/remote/call_remote_great_ai.py

@@ -3,7 +3,8 @@ from typing import Any, Mapping, Optional, Type, TypeVar
 
 from pydantic import BaseModel
 
-from ...utilities import get_logger
+from utilities import get_logger
+
 from ..views import Trace
 from .call_remote_great_ai_async import call_remote_great_ai_async
 

src/great_ai/utilities/unique.py

@@ -1,16 +0,0 @@
-from typing import Any, Callable, Iterable, List
-
-
-def unique(
-    values: Iterable[Any], *, key: Callable[[Any], Any] = lambda v: v
-) -> List[Any]:
-    """Only keep first occurrences and maintain order"""
-
-    key_values = {}
-    for v in values:
-        k = key(v)
-        if k not in key_values:
-            # dicts maintain insertion order: https://mail.python.org/pipermail/python-dev/2017-December/151283.html
-            key_values[k] = v
-
-    return list(key_values.values())

src/large_file/__init__.py

@@ -0,0 +1 @@
+from .large_file import LargeFileBase, LargeFileLocal, LargeFileMongo, LargeFileS3

src/large_file/__main__.py

@@ -4,8 +4,9 @@ from argparse import Namespace
 from pathlib import Path
 from typing import Mapping, Type
 
-from ..utilities import get_logger
+from large_file import LargeFileBase, LargeFileLocal, LargeFileMongo, LargeFileS3
-from .large_file import LargeFile, LargeFileLocal, LargeFileMongo, LargeFileS3
+from utilities import get_logger
+
 from .parse_arguments import parse_arguments
 
 logger = get_logger("large_file")
@@ -37,8 +38,8 @@ def main() -> None:
             large_file(f).delete()
 
 
-def get_class(args: Namespace) -> Type[LargeFile]:
+def get_class(args: Namespace) -> Type[LargeFileBase]:
-    factory: Mapping[str, Type[LargeFile]] = {
+    factory: Mapping[str, Type[LargeFileBase]] = {
         "s3": LargeFileS3,
         "local": LargeFileLocal,
         "mongodb": LargeFileMongo,

src/large_file/large_file/__init__.py

@@ -1,4 +1,4 @@
-from .large_file import LargeFile
+from .large_file_base import LargeFileBase
 from .large_file_local import LargeFileLocal
 from .large_file_mongo import LargeFileMongo
 from .large_file_s3 import LargeFileS3

src/large_file/large_file/large_file_base.py

@@ -7,7 +7,8 @@ from pathlib import Path
 from types import TracebackType
 from typing import IO, Any, List, Optional, Type, Union, cast
 
-from ...utilities import ConfigFile, get_logger
+from utilities import ConfigFile, get_logger
+
 from ..helper import human_readable_to_byte
 from ..models import DataInstance
 
@@ -19,7 +20,7 @@ COMPRESSION_ALGORITHM = "gztar"
 ARCHIVE_EXTENSION = ".tar.gz"
 
 
-class LargeFile(ABC):
+class LargeFileBase(ABC):
     """
     Store large files remotely. Use local cache for speed up.
 
@@ -75,7 +76,7 @@ class LargeFile(ABC):
         self._errors = errors
         self._newline = newline
 
-        LargeFile.cache_path.mkdir(parents=True, exist_ok=True)
+        LargeFileBase.cache_path.mkdir(parents=True, exist_ok=True)
 
         self._find_instances()
         self._check_mode_and_set_version()

src/large_file/large_file/large_file_local.py

@@ -1,14 +1,15 @@
 from pathlib import Path
 from typing import Any, List, Optional
 
-from ...utilities import get_logger
+from utilities import get_logger
+
 from ..models import DataInstance
-from .large_file import LargeFile
+from .large_file_base import LargeFileBase
 
 logger = get_logger("large_file")
 
 
-class LargeFileLocal(LargeFile):
+class LargeFileLocal(LargeFileBase):
     def __init__(
         self,
         name: str,

src/large_file/large_file/large_file_mongo.py

@@ -6,10 +6,11 @@ from typing import Any, List, Mapping
 from gridfs import DEFAULT_CHUNK_SIZE, Database, GridFSBucket
 from pymongo import MongoClient
 
-from ...utilities import get_logger
+from utilities import get_logger
+
 from ..helper import DownloadProgressBar, UploadProgressBar
 from ..models import DataInstance
-from .large_file import LargeFile
+from .large_file_base import LargeFileBase
 
 logger = get_logger("large_file")
 
@@ -17,7 +18,7 @@ logger = get_logger("large_file")
 MONGO_NAME_VERSION_SEPARATOR = "_"
 
 
-class LargeFileMongo(LargeFile):
+class LargeFileMongo(LargeFileBase):
     mongo_connection_string = None
     mongo_database = None
 

src/large_file/large_file/large_file_s3.py

@@ -4,10 +4,11 @@ from typing import Any, List, Mapping, Optional
 
 import boto3
 
-from ...utilities import get_logger
+from utilities import get_logger
+
 from ..helper import DownloadProgressBar, UploadProgressBar
 from ..models import DataInstance
-from .large_file import LargeFile
+from .large_file_base import LargeFileBase
 
 logger = get_logger("large_file")
 
@@ -15,7 +16,7 @@ logger = get_logger("large_file")
 S3_NAME_VERSION_SEPARATOR = "/"
 
 
-class LargeFileS3(LargeFile):
+class LargeFileS3(LargeFileBase):
     """
     Store large files in S3. Use local cache for speed up.