Document and reformat

great_ai/__init__.py

@@ -14,9 +14,14 @@ from .models.save_model import save_model
 from .models.use_model import use_model
 from .parameters.log_metric import log_metric
 from .parameters.parameter import parameter
-from .persistence import MongodbDriver, ParallelTinyDbDriver, TracingDatabaseDriver
+from .persistence.mongodb_driver import MongoDbDriver
-from .remote import call_remote_great_ai, call_remote_great_ai_async
+from .persistence.parallel_tinydb_driver import ParallelTinyDbDriver
-from .tracing import add_ground_truth, delete_ground_truth, query_ground_truth
+from .persistence.tracing_database_driver import TracingDatabaseDriver
+from .remote.call_remote_great_ai import call_remote_great_ai
+from .remote.call_remote_great_ai_async import call_remote_great_ai_async
+from .tracing.add_ground_truth import add_ground_truth
+from .tracing.delete_ground_truth import delete_ground_truth
+from .tracing.query_ground_truth import query_ground_truth
 from .views import (
     ClassificationOutput,
     MultiLabelClassificationOutput,

great_ai/constants.py

@@ -1,5 +1,5 @@
 from .large_file import LargeFileMongo, LargeFileS3
-from .persistence.mongodb_driver import MongodbDriver
+from .persistence.mongodb_driver import MongoDbDriver
 
 ENV_VAR_KEY = "ENVIRONMENT"
 PRODUCTION_KEY = "production"
@@ -7,7 +7,7 @@ DASHBOARD_PATH = "/dashboard"
 
 MONGO_CONFIG_PATHS = ["mongodb.ini", "mongo.ini", "mongo_db.ini", "mongo-db.ini"]
 DEFAULT_TRACING_DATABASE_CONFIG_PATHS = {
-    MongodbDriver: MONGO_CONFIG_PATHS,
+    MongoDbDriver: MONGO_CONFIG_PATHS,
 }
 
 DEFAULT_LARGE_FILE_CONFIG_PATHS = {

great_ai/context.py

@@ -17,7 +17,8 @@ from .constants import (
     SE4ML_WEBSITE,
 )
 from .large_file import LargeFileBase, LargeFileLocal
-from .persistence import ParallelTinyDbDriver, TracingDatabaseDriver
+from .persistence.parallel_tinydb_driver import ParallelTinyDbDriver
+from .persistence.tracing_database_driver import TracingDatabaseDriver
 from .utilities import get_logger
 from .views import RouteConfig
 
@@ -67,9 +68,39 @@ def configure(
     should_log_exception_stack: Optional[bool] = None,
     prediction_cache_size: int = 512,
     disable_se4ml_banner: bool = False,
-    dashboard_table_size: int = 20,
+    dashboard_table_size: int = 50,
     route_config: RouteConfig = RouteConfig(),
 ) -> None:
+    """Set the global configuration used by the great-ai library.
+
+    You must call `configure` before calling (or decorating with) any other great-ai
+    function.
+
+    If `tracing_database_factory` or `large_file_implementation` is not specified, their
+    default value is determined based on which TracingDatabase and LargeFile has been
+    configured (e.g.: LargeFileS3.configure_credentials_from_file('s3.ini')), or whether
+    there is any file named s3.ini or mongo.ini in the working directory.
+
+    Examples:
+        >>> configure(prediction_cache_size=0)
+
+    Arguments:
+        version: The version of your application (using SemVer is recommended).
+        log_level: Set the default logging level of `logging`.
+        seed: Set seed of `random` (and `numpy` if installed) for reproducibility.
+        tracing_database_factory: Specify a different TracingDatabaseDriver than the one
+            already configured.
+        large_file_implementation: Specify a different LargeFile than the one already
+            configured.
+        should_log_exception_stack: Log the traces of unhandled exceptions.
+        prediction_cache_size: Size of the LRU cache applied over the prediction
+            functions.
+        disable_se4ml_banner: Turn off the warning about the importance of SE4ML best-
+            practices.
+        dashboard_table_size: Number of rows to display in the dashboard's table.
+        route_config: Enable or disable specific HTTP API endpoints.
+    """
+
     global _context
     logger = get_logger("great_ai", level=log_level)
 
@@ -89,7 +120,10 @@ def configure(
     tracing_database = tracing_database_factory()
 
     if not tracing_database.is_production_ready:
-        message = f"The selected tracing database ({tracing_database_factory.__name__}) is not recommended for production"
+        message = f"""The selected tracing database ({
+            tracing_database_factory.__name__
+        }) is not recommended for production"""
+
         if is_production:
             logger.error(message)
         else:
@@ -117,7 +151,8 @@ def configure(
 
     if not is_production and not disable_se4ml_banner:
         logger.warning(
-            "You still need to check whether you follow all best practices before trusting your deployment."
+            "You still need to check whether you follow all best practices before "
+            "trusting your deployment."
         )
         logger.warning(f"> Find out more at {SE4ML_WEBSITE}")
 
@@ -128,7 +163,8 @@ def _is_in_production_mode(logger: Optional[Logger]) -> bool:
     if environment is None:
         if logger:
             logger.warning(
-                f"Environment variable {ENV_VAR_KEY} is not set, defaulting to development mode ‼️"
+                f"Environment variable {ENV_VAR_KEY} is not set, "
+                "defaulting to development mode ‼️"
             )
         is_production = False
     else:
@@ -136,8 +172,8 @@ def _is_in_production_mode(logger: Optional[Logger]) -> bool:
         if logger:
             if not is_production:
                 logger.info(
-                    f"Value of {ENV_VAR_KEY} is `{environment}` which is not equal to `{PRODUCTION_KEY}`"
+                    f"Value of {ENV_VAR_KEY} is `{environment}` which is not equal to"
-                    + " defaulting to development mode ‼️"
+                    + f"`{PRODUCTION_KEY}` defaulting to development mode ‼️"
                 )
             else:
                 logger.info("Running in production mode ✅")
@@ -152,13 +188,16 @@ def _initialize_tracing_database(
         if selected is None or selected == tracing_driver:
             if tracing_driver.initialized:
                 logger.info(
-                    f"{tracing_driver.__name__} has been already configured: skipping initialisation"
+                    f"{tracing_driver.__name__} has been already configured: "
+                    "skipping initialisation"
                 )
                 return tracing_driver
             for p in paths:
                 if Path(p).exists():
                     logger.info(
-                        f"Found credentials file ({Path(p).absolute()}), initialising {tracing_driver.__name__}"
+                        f"""Found credentials file ({Path(p).absolute()}), initialising {
+                            tracing_driver.__name__
+                        }"""
                     )
                     tracing_driver.configure_credentials_from_file(p)
                     return tracing_driver
@@ -181,7 +220,9 @@ def _initialize_large_file(
             for p in paths:
                 if Path(p).exists():
                     logger.info(
-                        f"Found credentials file ({Path(p).absolute()}), initialising {large_file.__name__}"
+                        f"""Found credentials file ({Path(p).absolute()}), initialising {
+                            large_file.__name__
+                        }"""
                     )
                     large_file.configure_credentials_from_file(p)
                     return large_file

great_ai/deploy/great_ai.py

@@ -42,8 +42,22 @@ V = TypeVar("V")
 
 
 class GreatAI(Generic[T, V]):
-    __name__: str
+    """Wrapper for a prediction function providing the implementation of SE4ML best-practices.
-    __doc__: str
+
+    Provides caching (with argument freezing), a TracingContext during execution, the
+    scaffolding of HTTP endpoints using FastAPI and a dashboard using Dash.
+
+    Supports wrapping async and synchronous functions while also maintaining correct
+    typing.
+
+    Attributes:
+        app: FastAPI instance wrapping the scaffolded endpoints and the Dash app.
+        version: SemVer derived from the app's version and the model names and versions
+            registered through use_model.
+    """
+
+    __name__: str  # help for MyPy
+    __doc__: str  # help for MyPy
 
     def __init__(
         self,
@@ -102,14 +116,15 @@ class GreatAI(Generic[T, V]):
         """Decorate a function by wrapping it in a GreatAI instance.
 
         The function can be typed, synchronous or async. If it has
-        unwrapped parameters (parameters not affected by a @parameter
+        unwrapped parameters (parameters not affected by a
-        or @use_model decorator), those will be automatically wrapped.
+        [@parameter][great_ai.parameter] or [@use_model][great_ai.use_model] decorator),
+        those will be automatically wrapped.
 
         The return value is replaced by a Trace (or Awaitable[Trace]),
         while the original return value is available under the `.output`
         property.
 
-        For configuration options, see great_ai.configure.
+        For configuration options, see [great_ai.configure][].
 
         Examples:
             >>> @GreatAI.create
@@ -173,6 +188,23 @@ class GreatAI(Generic[T, V]):
         unpack_arguments: bool = False,
         do_not_persist_traces: bool = False,
     ) -> List[Trace[V]]:
+        """Map the wrapped function over a list of input_values (`batch`).
+
+        A wrapper over [parallel_map][great_ai.utilities.parallel_map]
+        providing type-safety and a progressbar through tqdm.
+
+        Args:
+            batch: A list of arguments for the original (wrapped) function. If the
+                function expects multiple arguments, provide a list of tuples and set
+                `unpack_arguments=True`.
+            concurrency: Number of processes to start. Don't set it too much higher than
+                the number of available CPU cores.
+            unpack_arguments: Expect a list of tuples and unpack the tuples before
+                giving them to the wrapped function.
+            do_not_persist_traces: Don't save the traces in the database. Useful for
+                evaluations run part of the CI.
+        """
+
         wrapped_function = self._wrapped_func
 
         def inner(value: Any) -> T:

great_ai/large_file/__init__.py

@@ -1 +1,4 @@
-from .large_file import LargeFileBase, LargeFileLocal, LargeFileMongo, LargeFileS3
+from .large_file.large_file_base import LargeFileBase
+from .large_file.large_file_local import LargeFileLocal
+from .large_file.large_file_mongo import LargeFileMongo
+from .large_file.large_file_s3 import LargeFileS3

great_ai/large_file/__main__.py

@@ -5,7 +5,7 @@ from pathlib import Path
 from typing import Mapping, Type
 
 from ..utilities import get_logger
-from .large_file import LargeFileBase, LargeFileLocal, LargeFileMongo, LargeFileS3
+from . import LargeFileBase, LargeFileLocal, LargeFileMongo, LargeFileS3
 from .parse_arguments import parse_arguments
 
 logger = get_logger("large_file")

great_ai/large_file/large_file/__init__.py

@@ -1,4 +0,0 @@
-from .large_file_base import LargeFileBase
-from .large_file_local import LargeFileLocal
-from .large_file_mongo import LargeFileMongo
-from .large_file_s3 import LargeFileS3

great_ai/persistence/__init__.py

@@ -1,3 +0,0 @@
-from .mongodb_driver import MongodbDriver
-from .parallel_tinydb_driver import ParallelTinyDbDriver
-from .tracing_database_driver import TracingDatabaseDriver

great_ai/persistence/mongodb_driver.py

@@ -18,7 +18,16 @@ operator_mapping = {
 }
 
 
-class MongodbDriver(TracingDatabaseDriver):
+class MongoDbDriver(TracingDatabaseDriver):
+    """TracingDatabaseDriver implementation using MongoDB as a backend.
+
+    A production-ready database driver suitable for efficiently handling semi-structured
+    data.
+
+    Checkout [MongoDB Atlas](https://www.mongodb.com/cloud/atlas/register) for a hosted
+    MongoDB solution.
+    """
+
     is_production_ready = True
 
     mongo_connection_string: str
@@ -28,7 +37,8 @@ class MongodbDriver(TracingDatabaseDriver):
         super().__init__()
         if self.mongo_connection_string is None or self.mongo_database is None:
             raise ValueError(
-                "Please configure the MongoDB access options by calling MongodbDriver.configure_credentials"
+                "Please configure the MongoDB access options by calling "
+                "MongoDbDriver.configure_credentials"
             )
 
         with MongoClient[Any](self.mongo_connection_string) as client:
@@ -44,6 +54,14 @@ class MongodbDriver(TracingDatabaseDriver):
         mongo_database: str,
         **_: Any,
     ) -> None:
+        """Configure the connection details for MongoDB.
+
+        Args:
+            mongo_connection_string: For example:
+                'mongodb://my_user:my_pass@localhost:27017'
+            mongo_database: Name of the database to use. If doesn't exist, it is
+                created and initialised.
+        """
         cls.mongo_connection_string = mongo_connection_string
         cls.mongo_database = mongo_database
         super().configure_credentials()

great_ai/persistence/parallel_tinydb_driver.py

@@ -17,6 +17,14 @@ operator_mapping = {"=": "eq", "!=": "ne", "<": "lt", "<=": "le", ">": "gt", ">=
 
 
 class ParallelTinyDbDriver(TracingDatabaseDriver):
+    """TracingDatabaseDriver with TinyDB as a backend.
+
+    Saves the database as a JSON into a single file. Highly inefficient on inserting,
+    not advised for production use.
+
+    A multiprocessing lock protects the database file to avoid parallelisation issues.
+    """
+
     is_production_ready = False
     path_to_db = Path(DEFAULT_TRACING_DB_FILENAME)
 

great_ai/persistence/tracing_database_driver.py

@@ -8,6 +8,8 @@ from ..views import Filter, SortBy, Trace
 
 
 class TracingDatabaseDriver(ABC):
+    """Interface expected from a database to be used for storing traces."""
+
     is_production_ready: bool
     initialized: bool = False
 

great_ai/remote/__init__.py

@@ -1,2 +0,0 @@
-from .call_remote_great_ai import call_remote_great_ai
-from .call_remote_great_ai_async import call_remote_great_ai_async

great_ai/tracing/__init__.py

@@ -1,4 +1 @@
-from .add_ground_truth import add_ground_truth
-from .delete_ground_truth import delete_ground_truth
-from .query_ground_truth import query_ground_truth
 from .tracing_context import TracingContext

great_ai/utilities/config_file/__init__.py

@@ -1,2 +0,0 @@
-from .config_file import ConfigFile
-from .parse_error import ParseError

great_ai/views/trace.py

@@ -82,4 +82,6 @@ class Trace(Generic[T], HashableBaseModel):
         }
 
     def __repr__(self) -> str:
-        return f"Trace[{type(self.output).__name__}]({pformat(self.dict(), indent=2, compact=True)})"
+        return f"""Trace[{type(self.output).__name__}]({
+            pformat(self.dict(), indent=2, compact=True).replace('{ ', '{', 1)
+        })"""