Move files

2022-06-25 14:01:14 +02:00 · 2022-06-25 14:01:14 +02:00 · 8b004ec46d
commit 8b004ec46d
parent 79ddc5c7df
233 changed files with 117 additions and 2394 deletions
--- a/src/init.py
+++ b/src/init.py
--- a/src/great_ai/init.py
+++ b/src/great_ai/init.py
@ -0,0 +1,3 @@
+from .great_ai import *
+from .large_file import *
+from .utilities import *
--- a/src/great_ai/main.py
+++ b/src/great_ai/main.py
@ -0,0 +1,194 @@
+#!/usr/bin/env python3
+
+import logging
+import re
+import time
+from importlib import import_module, reload
+from pathlib import Path
+from typing import Optional
+
+import uvicorn
+from uvicorn.config import LOGGING_CONFIG, Config
+from uvicorn.subprocess import get_subprocess
+from uvicorn.supervisors.basereload import BaseReload
+from watchdog.events import FileSystemEvent, PatternMatchingEventHandler
+from watchdog.observers import Observer
+
+from .great_ai.constants import SERVER_NAME
+from .great_ai.context import _is_in_production_mode
+from .great_ai.deploy import GreatAI
+from .great_ai.exceptions import ArgumentValidationError, MissingArgumentError
+from .parse_arguments import parse_arguments
+from .utilities import get_logger
+
+logger = get_logger(SERVER_NAME)
+
+
+GREAT_AI_LOGGING_CONFIG = {
+    **LOGGING_CONFIG,
+    "formatters": {
+        "default": {
+            "()": "great_ai.logger.CustomFormatter",
+            "fmt": "%(asctime)s | %(levelname)8s | %(message)s",
+        },
+        "access": {
+            "()": "great_ai.logger.CustomFormatter",
+            "fmt": "%(asctime)s | %(levelname)8s | %(message)s",  # noqa: E501
+        },
+    },
+}
+
+
+def main() -> None:
+    args = parse_arguments()
+    should_auto_reload = not _is_in_production_mode(logger=None)
+
+    if args.workers > 1 and should_auto_reload:
+        raise ArgumentValidationError(
+            "Cannot use auto-reload with multiple workers: set the `--workers=1` CLI argument,"
+            + "or set the ENVIRONMENT environment variable to `production`."
+        )
+
+    common_config = dict(
+        host=args.host,
+        port=args.port,
+        timeout_keep_alive=args.timeout_keep_alive,
+        workers=args.workers,
+        server_header=False,
+        reload=False,
+        log_config=GREAT_AI_LOGGING_CONFIG,
+    )
+
+    if not should_auto_reload:
+        file_name = get_script_name(args.file_name)
+        app = find_app(file_name)
+
+        logger.info(f"Starting uvicorn server with app={app}")
+
+        uvicorn.run(app, **common_config)  # this will never return
+
+    class EventHandler(PatternMatchingEventHandler):
+        def __init__(self) -> None:
+            super().__init__(patterns=["*.py", "*.ipynb"], ignore_patterns=["__*.py"])
+            self.server: Optional[GreatAIReload] = None
+            self.restart()
+
+        def on_closed(self, event: FileSystemEvent) -> None:
+            logger.warning(f"File {event.src_path} has triggered a restart")
+            self.restart()
+
+        def restart(self) -> None:
+            file_name = get_script_name(args.file_name)
+            app = find_app(file_name)
+            if app is None:
+                logger.warning("Auto-reloading skipped")
+                return
+
+            self.stop_server()
+
+            config = Config(app, **common_config)
+            socket = config.bind_socket()
+            self.server = GreatAIReload(
+                config, target=uvicorn.Server(config=config).run, sockets=[socket]
+            )
+            self.server.startup()
+
+        def stop_server(self) -> None:
+            if self.server:
+                self.server.shutdown()
+
+    restart_handler = EventHandler()
+    observer = Observer()
+    observer.schedule(restart_handler, path=".", recursive=True)
+    observer.start()
+
+    try:
+        while True:
+            time.sleep(50)
+    finally:
+        observer.stop()
+        restart_handler.stop_server()
+        if args.file_name.endswith(".ipynb"):
+            Path(get_script_name_of_notebook(args.file_name)).unlink(missing_ok=True)
+        observer.join()
+
+
+def get_script_name(file_name_argument: str) -> str:
+    if file_name_argument.endswith(".ipynb"):
+        logger.info("Converting notebook to Python script")
+        try:
+            from nbconvert import PythonExporter
+
+            exporter = PythonExporter()
+            content, _ = exporter.from_filename(file_name_argument)
+            file_name_argument = get_script_name_of_notebook(file_name_argument)
+            with open(file_name_argument, "w", encoding="utf-8") as f:
+                f.write(content)
+        except ImportError:
+            raise ImportError(
+                "Install `nbconvert` to be able to use Jupyter notebook files or use a regular Python file instead"
+            )
+
+    return re.sub(r"\.(py|ipynb)$", "", file_name_argument)
+
+
+def get_script_name_of_notebook(notebook_name: str) -> str:
+    base_name = re.sub(r"\.ipynb$", "", notebook_name)
+    return f"__{base_name}__.py"
+
+
+module = None
+
+
+def find_app(file_name: str) -> Optional[str]:
+    global module
+
+    logging.disable(logging.CRITICAL)
+    try:
+        if module is None:
+            module = import_module(file_name)
+        else:
+            module = reload(module)
+    except Exception:
+        logging.disable(logging.NOTSET)
+        logger.exception("Could not load file because of an exception: fix your code")
+        return None
+    finally:
+        logging.disable(logging.NOTSET)
+
+    for name, value in module.__dict__.items():
+        if isinstance(value, GreatAI):
+            app_name = name
+
+    if app_name:
+        logger.info(f"Found `{app_name}` to be the GreatAI app ")
+    else:
+        raise MissingArgumentError(
+            "GreatAI app could not be found, make sure to use `@GreatAI.deploy` on your prediction function"
+        )
+
+    return f"{file_name}:{app_name}.app"
+
+
+class GreatAIReload(BaseReload):
+    def startup(self) -> None:
+        self.process = get_subprocess(
+            config=self.config, target=self.target, sockets=self.sockets
+        )
+        self.process.start()
+
+    def shutdown(self) -> None:
+        self.process.terminate()
+        self.process.join()
+
+        for sock in self.sockets:
+            sock.close()
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except KeyboardInterrupt:
+        exit()
+    except Exception as e:
+        logger.error(e)
--- a/src/great_ai/great_ai/init.py
+++ b/src/great_ai/great_ai/init.py
@ -0,0 +1,11 @@
+from .context import configure
+from .deploy import GreatAI
+from .models import save_model, use_model
+from .output_models import (
+    ClassificationOutput,
+    MultiLabelClassificationOutput,
+    RegressionOutput,
+)
+from .parameters import log_metric, parameter
+from .persistence import MongodbDriver, ParallelTinyDbDriver, TracingDatabaseDriver
+from .tracing import add_ground_truth, delete_ground_truth, query_ground_truth
--- a/src/great_ai/great_ai/constants.py
+++ b/src/great_ai/great_ai/constants.py
@ -0,0 +1,30 @@
+from ..large_file import LargeFileMongo, LargeFileS3
+from .persistence.mongodb_driver import MongodbDriver
+
+ENV_VAR_KEY = "ENVIRONMENT"
+PRODUCTION_KEY = "production"
+DASHBOARD_PATH = "/dashboard"
+
+MONGO_CONFIG_PATHS = ["mongodb.ini", "mongo.ini", "mongo_db.ini", "mongo-db.ini"]
+DEFAULT_TRACING_DATABASE_CONFIG_PATHS = {
+    MongodbDriver: MONGO_CONFIG_PATHS,
+}
+
+DEFAULT_LARGE_FILE_CONFIG_PATHS = {
+    LargeFileS3: ["s3.ini", "b2.ini"],
+    LargeFileMongo: MONGO_CONFIG_PATHS,
+}
+
+GITHUB_LINK = "https://github.com/ScoutinScience/great_ai"
+
+TRAIN_SPLIT_TAG_NAME = "train"
+TEST_SPLIT_TAG_NAME = "test"
+VALIDATION_SPLIT_TAG_NAME = "validation"
+GROUND_TRUTH_TAG_NAME = "ground_truth"
+PRODUCTION_TAG_NAME = "production"
+DEVELOPMENT_TAG_NAME = "development"
+ONLINE_TAG_NAME = "online"
+
+SERVER_NAME = "GreatAI-Server"
+
+SE4ML_WEBSITE = "https://se-ml.github.io/practices/"
--- a/src/great_ai/great_ai/context.py
+++ b/src/great_ai/great_ai/context.py
@ -0,0 +1,190 @@
+import os
+import random
+from logging import DEBUG, Logger
+from pathlib import Path
+from typing import Any, Dict, Optional, Type, cast
+
+from pydantic import BaseModel
+
+from ..large_file import LargeFile, LargeFileLocal
+from ..utilities import get_logger
+from .constants import (
+    DEFAULT_LARGE_FILE_CONFIG_PATHS,
+    DEFAULT_TRACING_DATABASE_CONFIG_PATHS,
+    ENV_VAR_KEY,
+    PRODUCTION_KEY,
+    SE4ML_WEBSITE,
+)
+from .persistence import ParallelTinyDbDriver, TracingDatabaseDriver
+
+
+class Context(BaseModel):
+    tracing_database: TracingDatabaseDriver
+    large_file_implementation: Type[LargeFile]
+    is_production: bool
+    logger: Logger
+    should_log_exception_stack: bool
+    prediction_cache_size: int
+    dashboard_table_size: int
+
+    class Config:
+        arbitrary_types_allowed = True
+
+    def to_flat_dict(self) -> Dict[str, Any]:
+        return {
+            "tracing_database": type(self.tracing_database).__name__,
+            "large_file_implementation": self.large_file_implementation.__name__,
+            "is_production": self.is_production,
+            "should_log_exception_stack": self.should_log_exception_stack,
+            "prediction_cache_size": self.prediction_cache_size,
+            "dashboard_table_size": self.dashboard_table_size,
+        }
+
+
+_context: Optional[Context] = None
+
+
+def get_context() -> Context:
+    if _context is None:
+        configure()
+
+    return cast(Context, _context)
+
+
+def configure(
+    *,
+    log_level: int = DEBUG,
+    seed: int = 42,
+    tracing_database: Optional[Type[TracingDatabaseDriver]] = None,
+    large_file_implementation: Optional[Type[LargeFile]] = None,
+    should_log_exception_stack: Optional[bool] = None,
+    prediction_cache_size: int = 512,
+    disable_se4ml_banner: bool = False,
+    dashboard_table_size: int = 50,
+) -> None:
+    global _context
+    logger = get_logger("great_ai", level=log_level)
+
+    if _context is not None:
+        logger.warn(
+            "Configuration has been already initialised, overwriting.\n"
+            + "Make sure to call `configure()` before importing your application code."
+        )
+
+    is_production = _is_in_production_mode(logger=logger)
+
+    _set_seed(seed)
+
+    tracing_database = _initialize_tracing_database(tracing_database, logger=logger)()
+
+    if not tracing_database.is_production_ready:
+        if is_production:
+            logger.error(
+                f"The selected tracing database ({type(tracing_database).__name__}) is not recommended for production"
+            )
+        else:
+            logger.warning(
+                f"The selected tracing database ({type(tracing_database).__name__}) is not recommended for production"
+            )
+
+    _context = Context(
+        tracing_database=tracing_database,
+        large_file_implementation=_initialize_large_file(
+            large_file_implementation, logger=logger
+        ),
+        is_production=is_production,
+        logger=logger,
+        should_log_exception_stack=not is_production
+        if should_log_exception_stack is None
+        else should_log_exception_stack,
+        prediction_cache_size=prediction_cache_size,
+        dashboard_table_size=dashboard_table_size,
+    )
+
+    logger.info("Settings: configured ✅")
+    for k, v in get_context().to_flat_dict().items():
+        logger.info(f"🔩 {k}: {v}")
+
+    if not is_production and not disable_se4ml_banner:
+        logger.warning(
+            "You still need to check whether you follow all best practices before trusting your deployment."
+        )
+        logger.warning(f"> Find out more at {SE4ML_WEBSITE}")
+
+
+def _is_in_production_mode(logger: Optional[Logger]) -> bool:
+    environment = os.environ.get(ENV_VAR_KEY)
+
+    if environment is None:
+        if logger:
+            logger.warning(
+                f"Environment variable {ENV_VAR_KEY} is not set, defaulting to development mode ‼️"
+            )
+        is_production = False
+    else:
+        is_production = environment.lower() == PRODUCTION_KEY
+        if logger:
+            if not is_production:
+                logger.info(
+                    f"Value of {ENV_VAR_KEY} is `{environment}` which is not equal to `{PRODUCTION_KEY}`"
+                    + "defaulting to development mode ‼️"
+                )
+            else:
+                logger.info("Running in production mode ✅")
+
+    return is_production
+
+
+def _initialize_tracing_database(
+    selected: Optional[Type[TracingDatabaseDriver]], logger: Logger
+) -> Type[TracingDatabaseDriver]:
+    for tracing_driver, paths in DEFAULT_TRACING_DATABASE_CONFIG_PATHS.items():
+        if selected is None or selected == tracing_driver:
+            if tracing_driver.initialized:
+                logger.warning(
+                    f"{tracing_driver.__name__} has been already configured: skipping initialisation"
+                )
+                return tracing_driver
+            for p in paths:
+                if Path(p).exists():
+                    logger.info(
+                        f"Found credentials file ({Path(p).absolute()}), initialising {tracing_driver.__name__}"
+                    )
+                    tracing_driver.configure_credentials_from_file(p)
+                    return tracing_driver
+    logger.warning(
+        "Cannot find credentials files, defaulting to using ParallelTinyDbDriver"
+    )
+    return ParallelTinyDbDriver
+
+
+def _initialize_large_file(
+    selected: Optional[Type[LargeFile]], logger: Logger
+) -> Type[LargeFile]:
+    for large_file, paths in DEFAULT_LARGE_FILE_CONFIG_PATHS.items():
+        if selected is None or selected == large_file:
+            if large_file.initialized:
+                logger.warning(
+                    f"{large_file.__name__} has been already configured: skipping initialisation"
+                )
+                return large_file
+            for p in paths:
+                if Path(p).exists():
+                    logger.info(
+                        f"Found credentials file ({Path(p).absolute()}), initialising {large_file.__name__}"
+                    )
+                    large_file.configure_credentials_from_file(p)
+                    return large_file
+    logger.warning("Cannot find credentials files, defaulting to using LargeFileLocal")
+    return LargeFileLocal
+
+
+def _set_seed(seed: int) -> None:
+    random.seed(seed)
+
+    try:
+        import numpy
+
+        numpy.random.seed(seed + 1)
+    except ImportError:
+        pass
--- a/src/great_ai/great_ai/deploy/init.py
+++ b/src/great_ai/great_ai/deploy/init.py
@ -0,0 +1 @@
+from .great_ai import GreatAI
--- a/src/great_ai/great_ai/deploy/great_ai.py
+++ b/src/great_ai/great_ai/deploy/great_ai.py
@ -0,0 +1,209 @@
+import inspect
+from functools import lru_cache, partial, wraps
+from typing import (
+    Any,
+    Callable,
+    Generic,
+    Iterable,
+    List,
+    Optional,
+    Type,
+    TypeVar,
+    Union,
+    cast,
+)
+
+from fastapi import APIRouter, FastAPI, status
+from pydantic import BaseModel, create_model
+
+from ...utilities import parallel_map
+from ..constants import DASHBOARD_PATH
+from ..context import get_context
+from ..helper import (
+    freeze_arguments,
+    get_function_metadata_store,
+    snake_case_to_text,
+    use_http_exceptions,
+)
+from ..parameters import automatically_decorate_parameters
+from ..tracing.tracing_context import TracingContext
+from ..views import ApiMetadata, CacheStatistics, HealthCheckResponse, Trace
+from .routes import (
+    bootstrap_docs_endpoints,
+    bootstrap_feedback_endpoints,
+    bootstrap_trace_endpoints,
+)
+from .routes.bootstrap_dashboard import bootstrap_dashboard
+
+T = TypeVar("T")
+
+
+class GreatAI(Generic[T]):
+    def __init__(self, func: Callable[..., Any], version: str):
+        func = automatically_decorate_parameters(func)
+        get_function_metadata_store(func).is_finalised = True
+
+        self._func = func
+
+        def func_in_tracing_context(*args: Any, **kwargs: Any) -> Trace[T]:
+            with TracingContext[T](func.__name__) as t:
+                result = func(*args, **kwargs)
+                output = t.finalise(output=result)
+            return output
+
+        self._cached_func = lru_cache(get_context().prediction_cache_size)(
+            func_in_tracing_context
+        )  # cannot put decorator on method, because it require the context to be setup
+
+        wraps(func)(self)
+
+        self._version = version
+
+        self.app = FastAPI(
+            title=self.name,
+            version=self.version,
+            description=self.documentation
+            + f"\n\nFind out more in the [dashboard]({DASHBOARD_PATH}).",
+            docs_url=None,
+            redoc_url=None,
+        )
+
+    @staticmethod
+    def deploy(
+        func: Optional[Callable[..., T]] = None,
+        *,
+        version: str = "0.0.1",
+        disable_rest_api: bool = False,
+        disable_docs: bool = False,
+        disable_dashboard: bool = False,
+    ) -> Union[Callable[[Callable[..., T]], "GreatAI[T]"], "GreatAI[T]"]:
+        if func is None:
+            return cast(
+                Callable[[Callable[..., T]], GreatAI[T]],
+                partial(
+                    GreatAI.deploy,
+                    disable_http=disable_rest_api,
+                    disable_docs=disable_docs,
+                    disable_dashboard=disable_dashboard,
+                ),
+            )
+
+        instance = GreatAI[T](func, version=version)
+
+        if not disable_rest_api:
+            instance._bootstrap_rest_api(
+                disable_docs=disable_docs, disable_dashboard=disable_dashboard
+            )
+
+        return instance
+
+    @freeze_arguments
+    def __call__(self, *args: Any, **kwargs: Any) -> Trace[T]:
+        return self._cached_func(*args, **kwargs)
+
+    def process_batch(
+        self,
+        batch: Iterable[Any],
+        concurrency: Optional[int] = None,
+    ) -> List[Trace[T]]:
+        return parallel_map(
+            freeze_arguments(self._cached_func), batch, concurrency=concurrency
+        )
+
+    @property
+    def name(self) -> str:
+        return snake_case_to_text(self._func.__name__)
+
+    @property
+    def version(self) -> str:
+        return (
+            f"{self._version}+{get_function_metadata_store(self._func).model_versions}"
+        )
+
+    @property
+    def documentation(self) -> str:
+        return (
+            f"GreatAI wrapper for interacting with the `{self._func.__name__}` function.\n\n"
+            + (
+                "\n".join(
+                    line.strip()
+                    for line in (self._func.__doc__ or "").split("\n")
+                    if line.strip()
+                )
+            )
+        )
+
+    def _bootstrap_rest_api(self, disable_docs: bool, disable_dashboard: bool) -> None:
+        self._bootstrap_prediction_endpoint()
+
+        if not disable_docs:
+            bootstrap_docs_endpoints(self.app)
+
+        if not disable_dashboard:
+            bootstrap_dashboard(
+                self.app,
+                function_name=self._func.__name__,
+                documentation=self.documentation,
+            )
+            bootstrap_trace_endpoints(self.app)
+
+        bootstrap_feedback_endpoints(self.app)
+        self._bootstrap_meta_endpoints()
+
+    def _bootstrap_prediction_endpoint(self) -> None:
+        router = APIRouter(
+            prefix="/predict",
+            tags=["predictions"],
+        )
+
+        schema = self._get_schema()
+
+        @router.post("/", status_code=status.HTTP_200_OK, response_model=Trace[T])
+        @use_http_exceptions
+        def predict(input_value: schema) -> Trace[T]:  # type: ignore
+            return self(**cast(BaseModel, input_value).dict())
+
+        self.app.include_router(router)
+
+    def _get_schema(self) -> Type[BaseModel]:
+        signature = inspect.signature(self._func)
+        parameters = {
+            p.name: (
+                p.annotation if p.annotation != inspect._empty else Any,
+                p.default if p.default != inspect._empty else ...,
+            )
+            for p in signature.parameters.values()
+            if p.name in get_function_metadata_store(self._func).input_parameter_names
+        }
+
+        schema: Type[BaseModel] = create_model("InputModel", **parameters)  # type: ignore
+        return schema
+
+    def _bootstrap_meta_endpoints(self) -> None:
+        router = APIRouter(
+            tags=["meta"],
+        )
+
+        @router.get("/health", status_code=status.HTTP_200_OK)
+        def check_health() -> HealthCheckResponse:
+            hits, misses, maxsize, cache_size = self._cached_func.cache_info()
+            cache_statistics = CacheStatistics(
+                hits=hits, misses=misses, size=cache_size, max_size=maxsize
+            )
+
+            return HealthCheckResponse(
+                is_healthy=True, cache_statistics=cache_statistics
+            )
+
+        @router.get(
+            "/version", response_model=ApiMetadata, status_code=status.HTTP_200_OK
+        )
+        def get_version() -> ApiMetadata:
+            return ApiMetadata(
+                name=self.name,
+                version=self.version,
+                documentation=self.documentation,
+                configuration=get_context().to_flat_dict(),
+            )
+
+        self.app.include_router(router)
--- a/src/great_ai/great_ai/deploy/routes/init.py
+++ b/src/great_ai/great_ai/deploy/routes/init.py
@ -0,0 +1,4 @@
+from .bootstrap_dashboard import bootstrap_dashboard
+from .bootstrap_docs_endpoints import bootstrap_docs_endpoints
+from .bootstrap_feedback_endpoints import bootstrap_feedback_endpoints
+from .bootstrap_trace_endpoints import bootstrap_trace_endpoints
--- a/src/great_ai/great_ai/deploy/routes/bootstrap_dashboard.py
+++ b/src/great_ai/great_ai/deploy/routes/bootstrap_dashboard.py
@ -0,0 +1,27 @@
+from pathlib import Path
+
+from fastapi import FastAPI
+from fastapi.middleware.wsgi import WSGIMiddleware
+from fastapi.responses import RedirectResponse
+from fastapi.staticfiles import StaticFiles
+
+from ...constants import DASHBOARD_PATH
+from .dashboard import create_dash_app
+
+PATH = Path(__file__).parent.resolve()
+
+
+def bootstrap_dashboard(app: FastAPI, function_name: str, documentation: str) -> None:
+    dash_app = create_dash_app(function_name, documentation)
+
+    app.mount(DASHBOARD_PATH, WSGIMiddleware(dash_app))
+
+    @app.get("/", include_in_schema=False)
+    def redirect_to_entrypoint() -> RedirectResponse:
+        return RedirectResponse(DASHBOARD_PATH)
+
+    app.mount(
+        "/assets",
+        StaticFiles(directory=PATH / "dashboard/assets"),
+        name="static",
+    )
--- a/src/great_ai/great_ai/deploy/routes/bootstrap_docs_endpoints.py
+++ b/src/great_ai/great_ai/deploy/routes/bootstrap_docs_endpoints.py
@ -0,0 +1,14 @@
+from fastapi import FastAPI
+from fastapi.openapi.docs import get_swagger_ui_html
+from fastapi.responses import RedirectResponse
+from starlette.responses import HTMLResponse
+
+
+def bootstrap_docs_endpoints(app: FastAPI) -> None:
+    @app.get("/docs", include_in_schema=False)
+    def custom_swagger_ui_html() -> HTMLResponse:
+        return get_swagger_ui_html(openapi_url="openapi.json", title=app.title)
+
+    @app.get("/docs/index.html", include_in_schema=False)
+    def redirect_to_docs() -> RedirectResponse:
+        return RedirectResponse("/docs")
--- a/src/great_ai/great_ai/deploy/routes/bootstrap_feedback_endpoints.py
+++ b/src/great_ai/great_ai/deploy/routes/bootstrap_feedback_endpoints.py
@ -0,0 +1,44 @@
+from typing import Any
+
+from fastapi import APIRouter, FastAPI, HTTPException, Response, status
+
+from ...context import get_context
+from ...views import EvaluationFeedbackRequest
+
+
+def bootstrap_feedback_endpoints(app: FastAPI) -> None:
+    router = APIRouter(
+        prefix="/traces/{trace_id}/feedback",
+        tags=["feedback"],
+    )
+
+    @router.put("/", status_code=status.HTTP_202_ACCEPTED)
+    def set_feedback(trace_id: str, input: EvaluationFeedbackRequest) -> Response:
+        trace = get_context().tracing_database.get(trace_id)
+        if trace is None:
+            raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
+
+        trace.feedback = input.feedback
+
+        get_context().tracing_database.update(trace_id, trace)
+        return Response(status_code=status.HTTP_202_ACCEPTED)
+
+    @router.get("/", status_code=status.HTTP_200_OK)
+    def get_feedback(trace_id: str) -> Any:
+        trace = get_context().tracing_database.get(trace_id)
+        if trace is None:
+            raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
+        return trace.feedback
+
+    @router.delete("/", status_code=status.HTTP_204_NO_CONTENT)
+    def delete_feedback(trace_id: str) -> Any:
+        trace = get_context().tracing_database.get(trace_id)
+        if trace is None:
+            raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
+
+        trace.feedback = None
+
+        get_context().tracing_database.update(trace_id, trace)
+        return Response(status_code=status.HTTP_204_NO_CONTENT)
+
+    app.include_router(router)
--- a/src/great_ai/great_ai/deploy/routes/bootstrap_trace_endpoints.py
+++ b/src/great_ai/great_ai/deploy/routes/bootstrap_trace_endpoints.py
@ -0,0 +1,44 @@
+from typing import List
+
+from fastapi import APIRouter, FastAPI, HTTPException, Response, status
+
+from ...context import get_context
+from ...views import Query, Trace
+
+
+def bootstrap_trace_endpoints(app: FastAPI) -> None:
+    router = APIRouter(
+        prefix="/traces",
+        tags=["traces"],
+    )
+
+    @router.post("/", status_code=status.HTTP_200_OK, response_model=List[Trace])
+    def query_traces(
+        query: Query,
+        skip: int = 0,
+        take: int = 100,
+    ) -> List[Trace]:
+        return get_context().tracing_database.query(
+            conjunctive_filters=query.filter,
+            conjunctive_tags=query.conjunctive_tags,
+            since=query.since,
+            until=query.until,
+            has_feedback=query.has_feedback,
+            sort_by=query.sort,
+            skip=skip,
+            take=take,
+        )[0]
+
+    @router.get("/{trace_id}", status_code=status.HTTP_200_OK, response_model=Trace)
+    def get_trace(trace_id: str) -> Trace:
+        result = get_context().tracing_database.get(trace_id)
+        if result is None:
+            raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
+        return result
+
+    @router.delete("/{trace_id}", status_code=status.HTTP_204_NO_CONTENT)
+    def delete_trace(trace_id: str) -> Response:
+        get_context().tracing_database.delete(trace_id)
+        return Response(status_code=status.HTTP_204_NO_CONTENT)
+
+    app.include_router(router)
--- a/src/great_ai/great_ai/deploy/routes/dashboard/init.py
+++ b/src/great_ai/great_ai/deploy/routes/dashboard/init.py
@ -0,0 +1 @@
+from .create_dash_app import create_dash_app
--- a/src/great_ai/great_ai/deploy/routes/dashboard/assets/github.png
+++ b/src/great_ai/great_ai/deploy/routes/dashboard/assets/github.png
--- a/src/great_ai/great_ai/deploy/routes/dashboard/assets/index.css
+++ b/src/great_ai/great_ai/deploy/routes/dashboard/assets/index.css
@ -0,0 +1,227 @@
+:root {
+  --important-color: #a30808;
+  --background-color: #edf5f6;
+  --small-padding: 10px;
+  --medium-padding: 20px;
+  --large-padding: 40px;
+  --border-radius: 10px;
+  --shadow: 0 4px 6px -1px rgb(0 0 0 / 10%), 0 2px 4px -1px rgb(0 0 0 / 6%);
+  --disclaimer-width: 180px;
+  --disclaimer-height: 35px;
+}
+
+@media (max-width: 900px) {
+  body {
+    zoom: 0.8;
+  }
+}
+
+@media (max-width: 550px) {
+  :root {
+    --small-padding: 5px;
+    --medium-padding: 10px;
+    --large-padding: 20px;
+    --border-radius: 8px;
+  }
+
+  .environment {
+    margin-top: calc(-1 * var(--large-padding));
+    margin-bottom: var(--large-padding);
+  }
+}
+
+@media (min-width: 551px) {
+  .environment {
+    position: absolute;
+    width: var(--disclaimer-width);
+    height: var(--disclaimer-height);
+    transform: rotate(-45deg);
+    top: calc(
+      var(--disclaimer-width) / 1.4142 - var(--disclaimer-height) / 1.4142
+    );
+    left: calc(-1 * var(--disclaimer-height) / 1.4142);
+    transform-origin: top left;
+    z-index: 100;
+  }
+}
+
+* {
+  margin: 0;
+  box-sizing: border-box;
+  word-break: break-word;
+}
+
+body {
+  background-color: var(--background-color);
+  font-family: Arial, Helvetica, sans-serif;
+}
+
+h1,
+h2,
+h3,
+h4,
+h5,
+h6 {
+  margin: var(--medium-padding) 0 var(--small-padding) 0;
+}
+
+h6 {
+  margin-top: 0;
+  font-size: 3rem;
+}
+
+html,
+body,
+#react-entry-point,
+main {
+  height: 100%;
+}
+
+main {
+  padding-top: var(--large-padding);
+  display: flex;
+  flex-direction: column;
+}
+
+.environment {
+  background-color: var(--important-color);
+  color: white;
+  text-align: center;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+}
+
+main > header,
+.configuration-container,
+.traces-table-container,
+.parallel-coordinates,
+main > footer {
+  padding: var(--large-padding);
+  flex-shrink: 0;
+  overflow: hidden;
+}
+
+main > header,
+.configuration-container,
+.traces-table-container,
+.parallel-coordinates {
+  margin: 0 var(--large-padding) var(--large-padding) var(--large-padding);
+  border-radius: var(--border-radius);
+  box-shadow: var(--shadow);
+  background-color: white;
+}
+
+main > header {
+  display: flex;
+  align-items: center;
+  flex-wrap: wrap;
+  justify-content: space-between;
+}
+
+main > header > div:nth-child(1) {
+  min-width: 350px;
+  max-width: 450px;
+  margin-bottom: var(--large-padding);
+  flex: 1;
+}
+
+main > header > div > h1 {
+  margin-top: 0;
+}
+
+main > header > *:nth-child(2) {
+  min-width: 250px;
+  max-width: 550px;
+  flex: 1;
+}
+
+main > header .placeholder {
+  opacity: 0.35;
+  font-size: 1.5rem;
+  text-align: center;
+  display: block;
+  min-width: 250px;
+  width: 60%;
+  margin: auto;
+}
+
+.configuration-container {
+  display: flex;
+  justify-content: space-between;
+  flex-wrap: wrap;
+}
+
+.configuration-item {
+  border-left: 2px solid var(--important-color);
+  padding-left: var(--small-padding);
+  margin: var(--medium-padding);
+}
+
+.configuration-item h4 {
+  font-weight: bold;
+  margin: 0 0 var(--small-padding) 0;
+}
+
+.traces-table-container {
+  padding: 0;
+}
+
+.traces-table-container header {
+  padding: var(--large-padding);
+}
+
+.traces-table-container header h2 {
+  margin-top: 0;
+}
+
+.dash-filter--case {
+  display: none;
+}
+
+.traces-table-container td > div {
+  white-space: pre !important;
+  max-height: 150px !important;
+  overflow: auto !important;
+  display: inline-block !important;
+  text-align: left !important;
+}
+
+.traces-table-container th > div {
+  text-align: left !important;
+}
+
+.space-filler {
+  flex-grow: 1;
+}
+
+main > footer {
+  opacity: 0.35;
+  margin: 0;
+}
+
+main > footer {
+  display: flex;
+  justify-content: space-between;
+  align-items: center;
+  padding: var(--large-padding);
+  background-color: #ddd;
+  position: relative;
+}
+
+.parallel-coordinates {
+  padding: 0;
+}
+
+a img {
+  display: block;
+  margin-left: var(--large-padding);
+  width: 64px;
+  height: 64px;
+  cursor: pointer;
+  transition: transform 300ms;
+}
+
+a img:hover {
+  transform: scale(1.1);
+}
--- a/src/great_ai/great_ai/deploy/routes/dashboard/create_dash_app.py
+++ b/src/great_ai/great_ai/deploy/routes/dashboard/create_dash_app.py
@ -0,0 +1,253 @@
+from math import ceil
+from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
+
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
+from dash import Dash, dcc, html
+from dash.dependencies import Input, Output
+from flask import Flask
+
+from .....utilities import unique
+from ....constants import DASHBOARD_PATH, ONLINE_TAG_NAME
+from ....context import get_context
+from ....helper import snake_case_to_text, text_to_hex_color
+from ....views import SortBy, Trace
+from .get_description import get_description
+from .get_filter_from_datatable import get_filter_from_datatable
+from .get_footer import get_footer
+from .get_traces_table import get_traces_table
+
+
+def create_dash_app(function_name: str, function_docs: str) -> Flask:
+    accent_color = text_to_hex_color(function_name)
+
+    app = Dash(
+        function_name,
+        requests_pathname_prefix=DASHBOARD_PATH + "/",
+        server=Flask(__name__),
+        title=snake_case_to_text(function_name),
+        update_title=None,
+        external_stylesheets=[
+            "/assets/index.css",
+        ],
+    )
+
+    app.layout = html.Main(
+        [
+            html.Div(
+                html.P("PRODUCTION" if get_context().is_production else "DEVELOPMENT"),
+                className="environment",
+            ),
+            html.Header(
+                [
+                    get_description(
+                        function_name=function_name,
+                        function_docs=function_docs,
+                        accent_color=accent_color,
+                    ),
+                    execution_time_histogram_container := html.Div(),
+                ],
+            ),
+            configuration_container := html.Div(
+                className="configuration-container",
+            ),
+            traces_table_container := html.Div(
+                [
+                    html.Header(
+                        [
+                            html.H2("Latest traces"),
+                            html.P(
+                                "Recent traces and aggregated metrics are presented below. Try filtering the table."
+                            ),
+                            html.A(
+                                "Filtering syntax.",
+                                href="https://dash.plotly.com/datatable/filtering",
+                                target="_blank",
+                            ),
+                        ]
+                    ),
+                    table := get_traces_table(),
+                ],
+                className="traces-table-container",
+            ),
+            parallel_coordinates := dcc.Graph(
+                className="parallel-coordinates", config={"displaylogo": False}
+            ),
+            html.Div(className="space-filler"),
+            get_footer(),
+            interval := dcc.Interval(
+                interval=4 * 1000,  # in milliseconds
+            ),
+        ]
+    )
+
+    @app.callback(
+        Output(configuration_container, "children"),
+        Input(interval, "n_intervals"),
+    )
+    def update_configuration(
+        n_intervals: int,
+    ) -> List[html.Div]:
+        config = get_context().to_flat_dict()
+        return [
+            html.Div(
+                [
+                    html.H4(snake_case_to_text(key)),
+                    html.P(str(value)),
+                ],
+                className="configuration-item",
+            )
+            for key, value in config.items()
+        ]
+
+    @app.callback(
+        Output(table, "data"),
+        Output(table, "page_count"),
+        Output(table, "columns"),
+        Output(traces_table_container, "style"),
+        Output(execution_time_histogram_container, "children"),
+        Output(parallel_coordinates, "figure"),
+        Output(parallel_coordinates, "style"),
+        Input(table, "page_current"),
+        Input(table, "page_size"),
+        Input(table, "sort_by"),
+        Input(table, "filter_query"),
+        Input(interval, "n_intervals"),
+    )
+    def update_page(
+        page_current: int,
+        page_size: int,
+        sort_by: List[Dict[str, Union[str, int]]],
+        filter_query: str,
+        n_intervals: int,
+    ) -> Tuple[
+        List[Dict[str, Any]],
+        int,
+        List[Dict[str, Sequence[str]]],
+        Dict[str, Any],
+        Any,
+        go.Figure,
+        Dict[str, Any],
+    ]:
+        conjunctive_filters = (
+            [get_filter_from_datatable(f) for f in filter_query.split(" && ")]
+            if filter_query
+            else []
+        )
+        non_null_conjunctive_filters = [f for f in conjunctive_filters if f is not None]
+
+        elements, count = get_context().tracing_database.query(
+            skip=page_current * page_size,
+            take=page_size,
+            conjunctive_filters=non_null_conjunctive_filters,
+            conjunctive_tags=[ONLINE_TAG_NAME],
+            sort_by=[SortBy.parse_obj(s) for s in sort_by],
+        )
+
+        columns, style = update_layout(elements[0] if elements else None)
+        execution_time_histogram, parallel_coords_fig, style = update_charts(
+            elements=elements, function_name=function_name, accent_color=accent_color
+        )
+
+        return (
+            [e.to_flat_dict(include_original=False) for e in elements],
+            max(1, ceil(count / page_size)),
+            columns,
+            style,
+            execution_time_histogram,
+            parallel_coords_fig,
+            style,
+        )
+
+    return app.server
+
+
+def update_layout(
+    first_element: Optional[Trace],
+) -> Tuple[List[Dict[str, Sequence[str]]], Dict[str, Any]]:
+
+    if first_element:
+        keys = list(first_element.to_flat_dict(include_original=False).keys())
+        header_height = max(len(i.split(":")) for i in keys)
+        columns = [
+            {
+                "name": [""] * (header_height - len(k.split(":")))
+                + k.replace("_flat", "").split(":"),
+                "id": k,
+            }
+            for k in keys
+        ]
+    else:
+        columns = []
+
+    return (
+        columns,
+        {"display": "none" if first_element is None else "block"},
+    )
+
+
+def update_charts(
+    elements: List[Trace], function_name: str, accent_color: str
+) -> Tuple[Any, go.Figure, Dict[str, Any]]:
+    if not elements:
+        return (
+            html.Span(
+                f"No traces yet: call your function ({function_name}) to create one.",
+                className="placeholder",
+            ),
+            go.Figure(),
+            {"display": "none"},
+        )
+
+    flat_elements = [e.to_flat_dict(include_original=False) for e in elements]
+
+    execution_time_histogram = dcc.Graph(config={"displaylogo": False})
+    df = pd.DataFrame(flat_elements)
+    fig = px.histogram(
+        df,
+        x="original_execution_time_ms",
+        labels={"original_execution_time_ms": "Execution time (ms)"},
+        nbins=20,
+        height=400,
+        log_y=True,
+        color_discrete_sequence=[accent_color],
+    )
+    fig.update_layout(
+        margin=dict(l=0, r=0, b=0, t=0, pad=0),
+    )
+    execution_time_histogram.figure = fig
+
+    parallel_coords_fig = go.Figure(
+        go.Parcoords(
+            dimensions=[
+                get_dimension_descriptor(df, c)
+                for c in df.columns
+                if c not in {"trace_id", "created", "output", "exception", "feedback"}
+                and "_flat" not in c
+            ],
+            line_color=accent_color,
+        )
+    )
+    return execution_time_histogram, parallel_coords_fig, {}
+
+
+def get_dimension_descriptor(df: pd.DataFrame, column: str) -> Dict[str, Any]:
+    dimension: Dict[str, Any] = {
+        "label": snake_case_to_text(column),
+    }
+
+    values = df[column]
+
+    try:
+        dimension["values"] = [float(v) for v in values]
+    except (TypeError, ValueError):
+        MAX_LENGTH = 40
+        unique_values = unique(values)
+        value_mapping = {str(v)[-MAX_LENGTH:]: i for i, v in enumerate(unique_values)}
+
+        dimension["values"] = [value_mapping[str(v)[-MAX_LENGTH:]] for v in values]
+        dimension["tickvals"] = list(value_mapping.values())
+        dimension["ticktext"] = [k[-MAX_LENGTH:] for k in value_mapping.keys()]
+
+    return dimension
--- a/src/great_ai/great_ai/deploy/routes/dashboard/get_description.py
+++ b/src/great_ai/great_ai/deploy/routes/dashboard/get_description.py
@ -0,0 +1,32 @@
+from dash import dcc, html
+
+from ....helper import snake_case_to_text, strip_lines
+
+
+def get_description(
+    function_name: str, function_docs: str, accent_color: str
+) -> html.Div:
+    return html.Div(
+        [
+            html.H1(
+                f"{snake_case_to_text(function_name)} - dashboard",
+                style={"color": accent_color},
+            ),
+            dcc.Markdown(
+                strip_lines(
+                    f"""
+                > View the live data of your deployment here.
+
+                ## Using the API
+
+                You can find the available endpoints at [/docs](/docs).
+
+                ## Details
+
+                {function_docs}
+                """
+                ),
+                className="description",
+            ),
+        ]
+    )
--- a/src/great_ai/great_ai/deploy/routes/dashboard/get_filter_from_datatable.py
+++ b/src/great_ai/great_ai/deploy/routes/dashboard/get_filter_from_datatable.py
@ -0,0 +1,23 @@
+from typing import Optional, Union
+
+from ....views import Filter, operators
+
+
+def get_filter_from_datatable(description: str) -> Optional[Filter]:
+    for operator in operators:
+        if operator in description:
+            name_part, value_part = description.split(operator, 1)
+            value_part = value_part.strip()
+            name_part = name_part[name_part.find("{") + 1 : name_part.rfind("}")]
+
+            v0 = value_part[0]
+            if v0 == value_part[-1] and v0 in ("'", '"', "`"):
+                value: Union[str, float] = value_part[1:-1].replace("\\" + v0, v0)
+            else:
+                try:
+                    value = float(value_part)
+                except ValueError:
+                    value = value_part
+            return Filter(property=name_part, operator=operator, value=value)
+
+    return None
--- a/src/great_ai/great_ai/deploy/routes/dashboard/get_footer.py
+++ b/src/great_ai/great_ai/deploy/routes/dashboard/get_footer.py
@ -0,0 +1,23 @@
+from dash import html
+
+from ....constants import GITHUB_LINK
+
+
+def get_footer() -> html.Footer:
+    return html.Footer(
+        [
+            html.Div(
+                [
+                    html.H6("GreatAI"),
+                    html.P(
+                        "A human-friendly framework for robust end-to-end AI deployments."
+                    ),
+                ]
+            ),
+            html.A(
+                html.Img(src="/assets/github.png"),
+                href=GITHUB_LINK,
+                target="_blank",
+            ),
+        ],
+    )
--- a/src/great_ai/great_ai/deploy/routes/dashboard/get_traces_table.py
+++ b/src/great_ai/great_ai/deploy/routes/dashboard/get_traces_table.py
@ -0,0 +1,33 @@
+from dash import dash_table
+
+from ....context import get_context
+
+
+def get_traces_table() -> dash_table.DataTable:
+    return dash_table.DataTable(
+        page_current=0,
+        page_size=get_context().dashboard_table_size,
+        page_action="custom",
+        filter_action="custom",
+        sort_action="custom",
+        sort_mode="multi",
+        sort_by=[
+            {"column_id": "created", "direction": "desc"},
+        ],
+        style_data={
+            "white-space": "normal",
+            "height": "auto",
+            "max-height": "300px",
+            "overflow": "hidden",
+            "text-overflow": "ellipsis",
+        },
+        style_cell={"padding": "5px"},
+        style_header={
+            "background-color": "white",
+            "font-weight": "bold",
+        },
+        merge_duplicate_headers=True,
+        style_cell_conditional=[
+            {"if": {"column_id": "output"}, "width": 1500},
+        ],
+    )
--- a/src/great_ai/great_ai/exceptions/init.py
+++ b/src/great_ai/great_ai/exceptions/init.py
@ -0,0 +1,2 @@
+from .argument_validation_error import ArgumentValidationError
+from .missing_argument_error import MissingArgumentError
--- a/src/great_ai/great_ai/exceptions/argument_validation_error.py
+++ b/src/great_ai/great_ai/exceptions/argument_validation_error.py
@ -0,0 +1,2 @@
+class ArgumentValidationError(Exception):
+    pass
--- a/src/great_ai/great_ai/exceptions/missing_argument_error.py
+++ b/src/great_ai/great_ai/exceptions/missing_argument_error.py
@ -0,0 +1,2 @@
+class MissingArgumentError(Exception):
+    pass
--- a/src/great_ai/great_ai/helper/init.py
+++ b/src/great_ai/great_ai/helper/init.py
@ -0,0 +1,8 @@
+from .freeze_arguments import freeze_arguments
+from .get_arguments import get_arguments
+from .get_function_metadata_store import get_function_metadata_store
+from .hashable_base_model import HashableBaseModel
+from .snake_case_to_text import snake_case_to_text
+from .strip_lines import strip_lines
+from .text_to_hex_color import text_to_hex_color
+from .use_http_exceptions import use_http_exceptions
--- a/src/great_ai/great_ai/helper/assert_function_is_not_finalised.py
+++ b/src/great_ai/great_ai/helper/assert_function_is_not_finalised.py
@ -0,0 +1,15 @@
+from typing import Any, Callable
+
+from ..context import get_context
+from .get_function_metadata_store import get_function_metadata_store
+
+
+def assert_function_is_not_finalised(func: Callable[..., Any]) -> None:
+    error_message = (
+        "The outer-most (first) decorator has to be `@GreatAI.deploy`. "
+        + f"In the case of `{func.__name__}`, it is not: fix this by moving `@GreatAI.deploy` to the top."
+    )
+
+    if get_function_metadata_store(func).is_finalised:
+        get_context().logger.error(error_message)
+        exit(-1)
--- a/src/great_ai/great_ai/helper/freeze_arguments.py
+++ b/src/great_ai/great_ai/helper/freeze_arguments.py
@ -0,0 +1,25 @@
+from functools import wraps
+from typing import Any, Callable, Dict, List
+
+
+class FrozenDict(dict):
+    def __hash__(self) -> int:  # type: ignore
+        return hash(frozenset(self.items()))
+
+
+def freeze_arguments(func: Callable[..., Any]) -> Callable[..., Any]:
+    """Transform mutable dictionary
+    Into immutable
+    Useful to be compatible with cache
+    source: https://stackoverflow.com/questions/6358481/using-functools-lru-cache-with-dictionary-arguments
+    """
+
+    @wraps(func)
+    def wrapper(*args: List[Any], **kwargs: Dict[str, Any]) -> Any:
+        args = tuple(FrozenDict(arg) if isinstance(arg, dict) else arg for arg in args)
+        kwargs = {
+            k: FrozenDict(v) if isinstance(v, dict) else v for k, v in kwargs.items()
+        }
+        return func(*args, **kwargs)
+
+    return wrapper
--- a/src/great_ai/great_ai/helper/get_arguments.py
+++ b/src/great_ai/great_ai/helper/get_arguments.py
@ -0,0 +1,24 @@
+import inspect
+from typing import Any, Callable, Dict, Mapping, Sequence
+
+
+def get_arguments(
+    func: Callable[..., Any], args: Sequence[Any], kwargs: Mapping[str, Any]
+) -> Dict[str, Any]:
+    """Return mapping from parameter names to actual argument values"""
+
+    signature = inspect.signature(func)
+
+    defaults = {
+        p.name: p.default
+        for p in signature.parameters.values()
+        if p.default != inspect._empty
+    }
+
+    filter_keys = [
+        param.name
+        for param in signature.parameters.values()
+        if param.kind == param.POSITIONAL_OR_KEYWORD
+    ]
+
+    return {**defaults, **dict(zip(filter_keys, args)), **kwargs}
--- a/src/great_ai/great_ai/helper/get_function_metadata_store.py
+++ b/src/great_ai/great_ai/helper/get_function_metadata_store.py
@ -0,0 +1,12 @@
+from typing import Any, Callable, cast
+
+from ..views.function_metadata import FunctionMetadata
+
+
+def get_function_metadata_store(func: Callable[..., Any]) -> FunctionMetadata:
+    any_func = cast(Any, func)
+
+    if not hasattr(any_func, "_great_ai_metadata"):
+        any_func._great_ai_metadata = FunctionMetadata()
+
+    return any_func._great_ai_metadata
--- a/src/great_ai/great_ai/helper/hashable_base_model.py
+++ b/src/great_ai/great_ai/helper/hashable_base_model.py
@ -0,0 +1,6 @@
+from pydantic import BaseModel
+
+
+class HashableBaseModel(BaseModel):
+    def __hash__(self) -> int:
+        return hash((type(self),) + tuple(self.__dict__.values()))
--- a/src/great_ai/great_ai/helper/snake_case_to_text.py
+++ b/src/great_ai/great_ai/helper/snake_case_to_text.py
@ -0,0 +1,2 @@
+def snake_case_to_text(snake_case: str) -> str:
+    return snake_case.capitalize().replace("_", " ")
--- a/src/great_ai/great_ai/helper/strip_lines.py
+++ b/src/great_ai/great_ai/helper/strip_lines.py
@ -0,0 +1,2 @@
+def strip_lines(text: str) -> str:
+    return "\n".join(line.strip() for line in text.split("\n"))
--- a/src/great_ai/great_ai/helper/text_to_hex_color.py
+++ b/src/great_ai/great_ai/helper/text_to_hex_color.py
@ -0,0 +1,13 @@
+import colorsys
+from hashlib import md5
+
+
+def text_to_hex_color(text: str) -> str:
+    ascii_bytes = text.encode("ascii")
+    digest = md5(
+        ascii_bytes
+    ).hexdigest()  # the built-in hash function is salted differently in each process
+    integer = int(digest, 16)
+    hue = integer % 6311 / 6311.0
+    rgb = colorsys.hsv_to_rgb(hue, 0.75, 0.6)
+    return "#" + "".join("%02X" % round(i * 255) for i in rgb)
--- a/src/great_ai/great_ai/helper/use_http_exceptions.py
+++ b/src/great_ai/great_ai/helper/use_http_exceptions.py
@ -0,0 +1,20 @@
+from functools import wraps
+from typing import Any, Callable, Dict, List, TypeVar, cast
+
+from fastapi import HTTPException, status
+
+F = TypeVar("F", bound=Callable[..., Any])
+
+
+def use_http_exceptions(func: F) -> F:
+    @wraps(func)
+    def wrapper(*args: List[Any], **kwargs: Dict[str, Any]) -> Any:
+        try:
+            return func(*args, **kwargs)
+        except Exception as e:
+            raise HTTPException(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail=f"The following exception has occurred: {type(e).__name__}: {e}",
+            )
+
+    return cast(F, wrapper)
--- a/src/great_ai/great_ai/models/init.py
+++ b/src/great_ai/great_ai/models/init.py
@ -0,0 +1,2 @@
+from .save_model import save_model
+from .use_model import use_model
--- a/src/great_ai/great_ai/models/load_model.py
+++ b/src/great_ai/great_ai/models/load_model.py
@ -0,0 +1,17 @@
+from typing import Any, Optional, Tuple
+
+from joblib import load
+
+from ..context import get_context
+
+
+def load_model(
+    key: str, version: Optional[int] = None, return_path: bool = False
+) -> Tuple[Any, int]:
+    file = get_context().large_file_implementation(name=key, mode="rb", version=version)
+
+    if return_path:
+        return file.get(), file.version
+
+    with file as f:
+        return load(f), file.version
--- a/src/great_ai/great_ai/models/save_model.py
+++ b/src/great_ai/great_ai/models/save_model.py
@ -0,0 +1,24 @@
+from pathlib import Path
+from typing import Optional, Union
+
+from joblib import dump
+
+from ..context import get_context
+
+
+def save_model(
+    model: Union[Path, str, object], key: str, *, keep_last_n: Optional[int] = None
+) -> str:
+    file = get_context().large_file_implementation(
+        name=key, mode="wb", keep_last_n=keep_last_n
+    )
+
+    if isinstance(model, Path) or isinstance(model, str):
+        file.push(model)
+    else:
+        with file as f:
+            dump(model, f)
+
+    get_context().logger.info(f"Model {key} uploaded with version {file.version}")
+
+    return f"{key}:{file.version}"
--- a/src/great_ai/great_ai/models/use_model.py
+++ b/src/great_ai/great_ai/models/use_model.py
@ -0,0 +1,48 @@
+from functools import wraps
+from typing import Any, Callable, Dict, List, Literal, TypeVar, Union, cast
+
+from ..helper import get_function_metadata_store
+from ..helper.assert_function_is_not_finalised import assert_function_is_not_finalised
+from ..tracing.tracing_context import TracingContext
+from ..views import Model
+from .load_model import load_model
+
+F = TypeVar("F", bound=Callable[..., Any])
+
+
+def use_model(
+    key: str,
+    *,
+    version: Union[int, Literal["latest"]],
+    return_path: bool = False,
+    model_kwarg_name: str = "model",
+) -> Callable[[F], F]:
+    assert (
+        isinstance(version, int) or version == "latest"
+    ), "Only integers or the string literal `latest` is allowed as version"
+
+    model, actual_version = load_model(
+        key=key,
+        version=None if version == "latest" else version,
+        return_path=return_path,
+    )
+
+    def decorator(func: F) -> F:
+        assert_function_is_not_finalised(func)
+
+        store = get_function_metadata_store(func)
+        store.model_parameter_names.append(model_kwarg_name)
+        if store.model_versions:
+            store.model_versions += "."
+        store.model_versions += f"{key}-v{actual_version}"
+
+        @wraps(func)
+        def wrapper(*args: List[Any], **kwargs: Dict[str, Any]) -> Any:
+            tracing_context = TracingContext.get_current_tracing_context()
+            if tracing_context:
+                tracing_context.log_model(Model(key=key, version=actual_version))
+            return func(*args, **kwargs, **{model_kwarg_name: model})
+
+        return cast(F, wrapper)
+
+    return decorator
--- a/src/great_ai/great_ai/output_models/init.py
+++ b/src/great_ai/great_ai/output_models/init.py
@ -0,0 +1,3 @@
+from .classification_output import ClassificationOutput
+from .multi_label_classification_output import MultiLabelClassificationOutput
+from .regression_output import RegressionOutput
--- a/src/great_ai/great_ai/output_models/classification_output.py
+++ b/src/great_ai/great_ai/output_models/classification_output.py
@ -0,0 +1,9 @@
+from typing import Any, Optional, Union
+
+from ..helper import HashableBaseModel
+
+
+class ClassificationOutput(HashableBaseModel):
+    label: Union[str, int]
+    confidence: float
+    explanation: Optional[Any]
--- a/src/great_ai/great_ai/output_models/multi_label_classification_output.py
+++ b/src/great_ai/great_ai/output_models/multi_label_classification_output.py
@ -0,0 +1,8 @@
+from typing import List
+
+from ..helper import HashableBaseModel
+from .classification_output import ClassificationOutput
+
+
+class MultiLabelClassificationOutput(HashableBaseModel):
+    labels: List[ClassificationOutput] = []
--- a/src/great_ai/great_ai/output_models/regression_output.py
+++ b/src/great_ai/great_ai/output_models/regression_output.py
@ -0,0 +1,8 @@
+from typing import Any, Optional, Union
+
+from ..helper import HashableBaseModel
+
+
+class RegressionOutput(HashableBaseModel):
+    value: Union[int, float]
+    explanation: Optional[Any]
--- a/src/great_ai/great_ai/output_models/sequence_labeling_output.py
+++ b/src/great_ai/great_ai/output_models/sequence_labeling_output.py
@ -0,0 +1 @@
+# todo
--- a/src/great_ai/great_ai/parameters/init.py
+++ b/src/great_ai/great_ai/parameters/init.py
@ -0,0 +1,3 @@
+from .automatically_decorate_parameters import automatically_decorate_parameters
+from .log_metric import log_metric
+from .parameter import parameter
--- a/src/great_ai/great_ai/parameters/automatically_decorate_parameters.py
+++ b/src/great_ai/great_ai/parameters/automatically_decorate_parameters.py
@ -0,0 +1,27 @@
+import inspect
+from typing import Any, Callable, TypeVar
+
+from ..helper.get_function_metadata_store import get_function_metadata_store
+from .parameter import parameter
+
+F = TypeVar("F", bound=Callable[..., Any])
+
+
+def automatically_decorate_parameters(func: F) -> F:
+    signature = inspect.signature(func)
+    parameter_names = [
+        param.name
+        for param in signature.parameters.values()
+        if param.kind == param.POSITIONAL_OR_KEYWORD
+    ]
+
+    metadata = get_function_metadata_store(func)
+
+    for name in parameter_names:
+        if (
+            name not in metadata.model_parameter_names
+            and name not in metadata.input_parameter_names
+        ):
+            func = parameter(name)(func)
+
+    return func
--- a/src/great_ai/great_ai/parameters/log_metric.py
+++ b/src/great_ai/great_ai/parameters/log_metric.py
@ -0,0 +1,15 @@
+import inspect
+from typing import Any
+
+from ..context import get_context
+from ..tracing import TracingContext
+
+
+def log_metric(argument_name: str, value: Any) -> None:
+    tracing_context = TracingContext.get_current_tracing_context()
+    caller = inspect.stack()[1].function
+    actual_name = f"metric:{caller}:{argument_name}"
+    if tracing_context:
+        tracing_context.log_value(name=actual_name, value=value)
+
+    get_context().logger.info(f"{actual_name}={value}")
--- a/src/great_ai/great_ai/parameters/parameter.py
+++ b/src/great_ai/great_ai/parameters/parameter.py
@ -0,0 +1,51 @@
+from functools import wraps
+from typing import Any, Callable, Dict, TypeVar, cast
+
+from ..exceptions import ArgumentValidationError
+from ..helper import get_arguments, get_function_metadata_store
+from ..helper.assert_function_is_not_finalised import assert_function_is_not_finalised
+from ..tracing.tracing_context import TracingContext
+
+F = TypeVar("F", bound=Callable[..., Any])
+
+
+def parameter(
+    parameter_name: str,
+    *,
+    validator: Callable[[Any], bool] = lambda _: True,
+    disable_logging: bool = False,
+) -> Callable[[F], F]:
+    def decorator(func: F) -> F:
+        get_function_metadata_store(func).input_parameter_names.append(parameter_name)
+        assert_function_is_not_finalised(func)
+
+        actual_name = f"arg:{parameter_name}"
+
+        @wraps(func)
+        def wrapper(*args: Any, **kwargs: Dict[str, Any]) -> Any:
+            arguments = get_arguments(func, args, kwargs)
+            argument = arguments[parameter_name]
+
+            expected_type = func.__annotations__.get(parameter_name)
+
+            if expected_type is not None and not isinstance(argument, expected_type):
+                raise ArgumentValidationError(
+                    f"Argument {parameter_name} in {func.__name__} has the wrong type, expected: {expected_type.__name__}, got: {type(argument).__name__}"
+                )
+
+            if not validator(argument):
+                raise ArgumentValidationError(
+                    f"Argument {parameter_name} in {func.__name__} did not pass validation"
+                )
+
+            context = TracingContext.get_current_tracing_context()
+            if context and not disable_logging:
+                context.log_value(name=f"{actual_name}:value", value=argument)
+                if isinstance(argument, str):
+                    context.log_value(name=f"{actual_name}:length", value=len(argument))
+
+            return func(*args, **kwargs)
+
+        return cast(F, wrapper)
+
+    return decorator
--- a/src/great_ai/great_ai/persistence/init.py
+++ b/src/great_ai/great_ai/persistence/init.py
@ -0,0 +1,3 @@
+from .mongodb_driver import MongodbDriver
+from .parallel_tinydb_driver import ParallelTinyDbDriver
+from .tracing_database_driver import TracingDatabaseDriver
--- a/src/great_ai/great_ai/persistence/mongodb_driver.py
+++ b/src/great_ai/great_ai/persistence/mongodb_driver.py
@ -0,0 +1,137 @@
+from datetime import datetime
+from typing import Any, List, Mapping, Optional, Sequence, Tuple
+
+from pymongo import MongoClient
+
+from ..views import Filter, SortBy, Trace
+from .tracing_database_driver import TracingDatabaseDriver
+
+operator_mapping = {
+    "=": "$eq",
+    "!=": "$ne",
+    "<": "$lt",
+    "<=": "$lte",
+    ">": "$gt",
+    ">=": "$gte",
+    "contains": "$regex",
+}
+
+
+class MongodbDriver(TracingDatabaseDriver):
+    is_production_ready = True
+
+    def __init__(self) -> None:
+        super().__init__()
+        if self.mongo_connection_string is None or self.mongo_database is None:
+            raise ValueError(
+                "Please configure the MongoDB access options by calling MongodbDriver.configure_credentials"
+            )
+
+    @classmethod
+    def configure_credentials(  # type: ignore
+        cls,
+        *,
+        mongo_connection_string: str,
+        mongo_database: str,
+        **_: Mapping[str, Any],
+    ) -> None:
+        cls.mongo_connection_string = mongo_connection_string
+        cls.mongo_database = mongo_database
+        super().configure_credentials()
+
+    def save(self, trace: Trace) -> str:
+        serialized = trace.to_flat_dict()
+        serialized["_id"] = trace.trace_id
+
+        with MongoClient(self.mongo_connection_string) as client:
+            return client[self.mongo_database].traces.insert_one(serialized)
+
+    def save_batch(self, documents: List[Trace]) -> List[str]:
+        serialized = [d.to_flat_dict() for d in documents]
+        for s in serialized:
+            s["_id"] = s["trace_id"]
+
+        with MongoClient(self.mongo_connection_string) as client:
+            return client[self.mongo_database].traces.insert_many(
+                serialized, ordered=False
+            )
+
+    def get(self, id: str) -> Optional[Trace]:
+        with MongoClient(self.mongo_connection_string) as client:
+            value = client[self.mongo_database].traces.find_one(id)
+
+        if value:
+            value = Trace.parse_obj(value)
+
+        return value
+
+    def _get_operator(self, filter: Filter) -> str:
+        if filter.operator == "contains" and not isinstance(filter.value, str):
+            return operator_mapping["="]
+        return operator_mapping[filter.operator]
+
+    def query(
+        self,
+        *,
+        skip: int = 0,
+        take: Optional[int] = None,
+        conjunctive_filters: Sequence[Filter] = [],
+        conjunctive_tags: Sequence[str] = [],
+        since: Optional[datetime] = None,
+        until: Optional[datetime] = None,
+        has_feedback: Optional[bool] = None,
+        sort_by: Sequence[SortBy] = [],
+    ) -> Tuple[List[Trace], int]:
+
+        query = {
+            "filter": {
+                "$and": [{"tags": tag} for tag in conjunctive_tags]
+                + [
+                    {f.property: {self._get_operator(f): f.value}}
+                    for f in conjunctive_filters
+                ]
+                + [{}]
+            },
+            "sort": [
+                (col.column_id, 1 if col.direction == "asc" else -1) for col in sort_by
+            ],
+        }
+
+        if skip:
+            query["skip"] = skip
+
+        if take:
+            query["limit"] = take
+
+        if since:
+            query["filter"]["$and"].append({"created": {"$gte": since}})
+
+        if until:
+            query["filter"]["$and"].append({"created": {"$lte": until}})
+
+        if has_feedback is not None:
+            query["filter"]["$and"].append(
+                {"feedback": {"$ne": None}} if has_feedback else {"feedback": None}
+            )
+
+        with MongoClient(self.mongo_connection_string) as client:
+            values = client[self.mongo_database].traces.find(**query)
+            documents = [Trace.parse_obj(t) for t in values]
+
+        return documents, len(documents)
+
+    def update(self, id: str, new_version: Trace) -> None:
+        serialized = new_version.dict()
+        serialized["_id"] = new_version.trace_id
+        with MongoClient(self.mongo_connection_string) as client:
+            client[self.mongo_database].traces.update_one(id, new_version)
+
+    def delete(self, id: str) -> None:
+        with MongoClient(self.mongo_connection_string) as client:
+            client[self.mongo_database].traces.delete_one(id)
+
+    def delete_batch(self, ids: List[str]) -> List[str]:
+        delete_filter = {"_id": {"$in": ids}}
+
+        with MongoClient(self.mongo_connection_string) as client:
+            return client[self.mongo_database].traces.delete_many(delete_filter)
--- a/src/great_ai/great_ai/persistence/parallel_tinydb_driver.py
+++ b/src/great_ai/great_ai/persistence/parallel_tinydb_driver.py
@ -0,0 +1,112 @@
+from datetime import datetime
+from multiprocessing import Lock
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, cast
+
+import pandas as pd
+from tinydb import TinyDB
+
+from ..views import Filter, SortBy, Trace
+from .tracing_database_driver import TracingDatabaseDriver
+
+DEFAULT_TRACING_DB_FILENAME = "tracing_database.json"
+lock = Lock()
+
+
+operator_mapping = {"=": "eq", "!=": "ne", "<": "lt", "<=": "le", ">": "gt", ">=": "ge"}
+
+
+class ParallelTinyDbDriver(TracingDatabaseDriver):
+    is_production_ready = False
+    path_to_db = Path(DEFAULT_TRACING_DB_FILENAME)
+
+    def save(self, trace: Trace) -> str:
+        return self._safe_execute(lambda db: db.insert(trace.dict()))
+
+    def save_batch(self, documents: List[Trace]) -> List[str]:
+        traces = [d.dict() for d in documents]
+        return self._safe_execute(lambda db: db.insert_multiple(traces))
+
+    def get(self, id: str) -> Optional[Trace]:
+        value = self._safe_execute(lambda db: db.get(lambda d: d["trace_id"] == id))
+        if value:
+            value = Trace.parse_obj(value)
+        return value
+
+    def query(
+        self,
+        *,
+        skip: int = 0,
+        take: Optional[int] = None,
+        conjunctive_filters: Sequence[Filter] = [],
+        conjunctive_tags: Sequence[str] = [],
+        since: Optional[datetime] = None,
+        until: Optional[datetime] = None,
+        has_feedback: Optional[bool] = None,
+        sort_by: Sequence[SortBy] = []
+    ) -> Tuple[List[Trace], int]:
+        def does_match(d: Dict[str, Any]) -> bool:
+            return (
+                not set(conjunctive_tags) - set(d["tags"])
+                and (
+                    since is None
+                    or cast(datetime, datetime.fromisoformat(d["created"])) >= since
+                )
+                and (
+                    until is None
+                    or cast(datetime, datetime.fromisoformat(d["created"])) <= until
+                )
+                and (
+                    has_feedback is None or has_feedback == (d["feedback"] is not None)
+                )
+            )
+
+        documents: List[Trace] = [
+            Trace.parse_obj(t)
+            for t in self._safe_execute(lambda db: db.search(does_match))
+        ]
+
+        if not documents:
+            return [], 0
+
+        df = pd.DataFrame([d.to_flat_dict() for d in documents])
+
+        for f in conjunctive_filters:
+            operator = f.operator.lower()
+            if operator in operator_mapping:
+                df = df.loc[
+                    getattr(df[f.property], operator_mapping[f.operator])(f.value)
+                ]
+            elif operator == "contains":
+                df = df.loc[df[f.property].str.contains(f.value, case=False)]
+
+        if sort_by:
+            df.sort_values(
+                [col["column_id"] for col in sort_by],
+                ascending=[col["direction"] == "asc" for col in sort_by],
+                inplace=True,
+            )
+
+        count = len(df)
+        result = df.iloc[skip:] if take is None else df.iloc[skip : skip + take]
+        return [
+            next(d for d in documents if d.trace_id == trace_id)
+            for trace_id in result["trace_id"]
+        ], count
+
+    def update(self, id: str, new_version: Trace) -> None:
+        self._safe_execute(
+            lambda db: db.update(new_version.dict(), lambda d: d["trace_id"] == id)
+        )
+
+    def delete(self, id: str) -> None:
+        self._safe_execute(lambda db: db.remove(lambda d: d["trace_id"] == id))
+
+    def delete_batch(self, ids: List[str]) -> List[str]:
+        for i in ids:
+            self.delete(i)
+
+    def _safe_execute(self, func: Callable[[TinyDB], Any]) -> Any:
+        with lock:
+            with TinyDB(self.path_to_db) as db:
+                return func(db)
--- a/src/great_ai/great_ai/persistence/tracing_database_driver.py
+++ b/src/great_ai/great_ai/persistence/tracing_database_driver.py
@ -0,0 +1,70 @@
+from abc import ABC, abstractclassmethod, abstractmethod
+from datetime import datetime
+from pathlib import Path
+from typing import List, Optional, Sequence, Tuple, Union
+
+from ...utilities import ConfigFile
+from ..views import Filter, SortBy, Trace
+
+
+class TracingDatabaseDriver(ABC):
+    is_production_ready: bool
+    initialized: bool = False
+
+    @classmethod
+    def configure_credentials_from_file(
+        cls,
+        secrets_path: Union[Path, str],
+    ) -> None:
+        cls.configure_credentials(**ConfigFile(secrets_path))
+
+    @abstractclassmethod
+    def configure_credentials(
+        cls,
+    ) -> None:
+        cls.initialized = True
+
+    @abstractmethod
+    def save(self, document: Trace) -> str:
+        pass
+
+    @abstractmethod
+    def save_batch(
+        self,
+        documents: List[Trace],
+    ) -> List[str]:
+        pass
+
+    @abstractmethod
+    def get(self, id: str) -> Optional[Trace]:
+        pass
+
+    @abstractmethod
+    def query(
+        self,
+        *,
+        skip: int = 0,
+        take: Optional[int] = None,
+        conjunctive_filters: Sequence[Filter] = [],
+        conjunctive_tags: Sequence[str] = [],
+        until: Optional[datetime] = None,
+        since: Optional[datetime] = None,
+        has_feedback: Optional[bool] = None,
+        sort_by: Sequence[SortBy] = []
+    ) -> Tuple[List[Trace], int]:
+        pass
+
+    @abstractmethod
+    def update(self, id: str, new_version: Trace) -> None:
+        pass
+
+    @abstractmethod
+    def delete(self, id: str) -> None:
+        pass
+
+    @abstractmethod
+    def delete_batch(
+        self,
+        ids: List[str],
+    ) -> None:
+        pass
--- a/src/great_ai/great_ai/tracing/init.py
+++ b/src/great_ai/great_ai/tracing/init.py
@ -0,0 +1,4 @@
+from .add_ground_truth import add_ground_truth
+from .delete_ground_truth import delete_ground_truth
+from .query_ground_truth import query_ground_truth
+from .tracing_context import TracingContext
--- a/src/great_ai/great_ai/tracing/add_ground_truth.py
+++ b/src/great_ai/great_ai/tracing/add_ground_truth.py
@ -0,0 +1,67 @@
+from datetime import datetime
+from math import ceil
+from random import shuffle
+from typing import Any, Iterable, List, TypeVar
+
+from ..constants import (
+    GROUND_TRUTH_TAG_NAME,
+    TEST_SPLIT_TAG_NAME,
+    TRAIN_SPLIT_TAG_NAME,
+    VALIDATION_SPLIT_TAG_NAME,
+)
+from ..context import get_context
+from ..views import Trace
+
+T = TypeVar("T")
+
+
+def add_ground_truth(
+    inputs: Iterable[Any],
+    expected_outputs: Iterable[T],
+    *,
+    tags: List[str] = [],
+    train_split_ratio: float,
+    test_split_ratio: float,
+    validation_split_ratio: float = 0
+) -> None:
+    get_context()  # this resets the seed
+
+    inputs = list(inputs)
+    expected_outputs = list(expected_outputs)
+    assert len(inputs) == len(
+        expected_outputs
+    ), "The length of the inputs and expected_outputs must be equal"
+
+    sum_ratio = train_split_ratio + test_split_ratio + validation_split_ratio
+    assert sum_ratio > 0, "The sum of the split ratios must be a positive number"
+
+    train_split_ratio /= sum_ratio
+    test_split_ratio /= sum_ratio
+    validation_split_ratio /= sum_ratio
+
+    values = list(zip(inputs, expected_outputs))
+    shuffle(values)
+
+    split_tags = (
+        [TRAIN_SPLIT_TAG_NAME] * ceil(train_split_ratio * len(inputs))
+        + [TEST_SPLIT_TAG_NAME] * ceil(test_split_ratio * len(inputs))
+        + [VALIDATION_SPLIT_TAG_NAME] * ceil(validation_split_ratio * len(inputs))
+    )
+    shuffle(split_tags)
+
+    created = datetime.utcnow().isoformat()
+    traces = [
+        Trace(
+            created=created,
+            original_execution_time_ms=0,
+            logged_values=X if isinstance(X, dict) else {"input": X},
+            models=[],
+            output=y,
+            feedback=y,
+            exception=None,
+            tags=[GROUND_TRUTH_TAG_NAME, split_tag, *tags],
+        )
+        for ((X, y), split_tag) in zip(values, split_tags)
+    ]
+
+    get_context().tracing_database.save_batch(traces)
--- a/src/great_ai/great_ai/tracing/delete_ground_truth.py
+++ b/src/great_ai/great_ai/tracing/delete_ground_truth.py
@ -0,0 +1,22 @@
+from datetime import datetime
+from typing import List, Optional, Union
+
+from ..context import get_context
+
+
+def delete_ground_truth(
+    conjunctive_tags: Union[List[str], str] = [],
+    *,
+    until: Optional[datetime] = None,
+    since: Optional[datetime] = None,
+) -> None:
+    tags = (
+        conjunctive_tags if isinstance(conjunctive_tags, list) else [conjunctive_tags]
+    )
+    db = get_context().tracing_database
+
+    items, length = db.query(
+        conjunctive_tags=tags, until=until, since=since, has_feedback=True
+    )
+
+    db.delete_batch([i.trace_id for i in items])
--- a/src/great_ai/great_ai/tracing/query_ground_truth.py
+++ b/src/great_ai/great_ai/tracing/query_ground_truth.py
@ -0,0 +1,22 @@
+from datetime import datetime
+from typing import List, Optional, Union
+
+from ..context import get_context
+from ..views import Trace
+
+
+def query_ground_truth(
+    conjunctive_tags: Union[List[str], str] = [],
+    *,
+    since: Optional[datetime] = None,
+    return_max_count: Optional[int] = None
+) -> List[Trace]:
+    tags = (
+        conjunctive_tags if isinstance(conjunctive_tags, list) else [conjunctive_tags]
+    )
+    db = get_context().tracing_database
+
+    items, length = db.query(
+        conjunctive_tags=tags, since=since, take=return_max_count, has_feedback=True
+    )
+    return items
--- a/src/great_ai/great_ai/tracing/tracing_context.py
+++ b/src/great_ai/great_ai/tracing/tracing_context.py
@ -0,0 +1,95 @@
+import threading
+from collections import defaultdict
+from datetime import datetime
+from types import TracebackType
+from typing import (
+    Any,
+    DefaultDict,
+    Dict,
+    Generic,
+    List,
+    Literal,
+    Optional,
+    Type,
+    TypeVar,
+)
+
+from ..constants import DEVELOPMENT_TAG_NAME, ONLINE_TAG_NAME, PRODUCTION_TAG_NAME
+from ..context import get_context
+from ..views import Model, Trace
+
+T = TypeVar("T")
+
+
+class TracingContext(Generic[T]):
+    _contexts: DefaultDict[int, List["TracingContext"]] = defaultdict(lambda: [])
+
+    def __init__(self, function_name: str) -> None:
+        self._models: List[Model] = []
+        self._values: Dict[str, Any] = {}
+        self._trace: Optional[Trace[T]] = None
+        self._start_time = datetime.utcnow()
+        self._name = function_name
+
+    def log_value(self, name: str, value: Any) -> None:
+        self._values[name] = value
+
+    def log_model(self, model: Model) -> None:
+        self._models.append(model)
+
+    def finalise(self, output: T = None, exception: BaseException = None) -> Trace[T]:
+        assert self._trace is None, "has been already finalised"
+
+        delta_time = (datetime.utcnow() - self._start_time).microseconds / 1000
+        self._trace = Trace(
+            created=self._start_time.isoformat(),
+            original_execution_time_ms=delta_time,
+            logged_values=self._values,
+            models=self._models,
+            output=output,
+            exception=None
+            if exception is None
+            else f"{type(exception).__name__}: {exception}",
+            tags=[
+                self._name,
+                ONLINE_TAG_NAME,
+                PRODUCTION_TAG_NAME
+                if get_context().is_production
+                else DEVELOPMENT_TAG_NAME,
+            ],
+        )
+
+        return self._trace
+
+    @classmethod
+    def get_current_tracing_context(cls) -> Optional["TracingContext"]:
+        if cls._contexts[threading.get_ident()]:
+            return cls._contexts[threading.get_ident()][-1]
+        return None
+
+    def __enter__(self) -> "TracingContext":
+        self._contexts[threading.get_ident()].append(self)
+        return self
+
+    def __exit__(
+        self,
+        type: Optional[Type[BaseException]],
+        exception: Optional[BaseException],
+        traceback: Optional[TracebackType],
+    ) -> Literal[False]:
+        assert self._contexts[threading.get_ident()][-1] == self
+        self._contexts[threading.get_ident()].remove(self)
+
+        if exception is not None and type is not None:
+            self.finalise(exception=exception)
+            if get_context().should_log_exception_stack:
+                get_context().logger.exception("Could not finish operation")
+            else:
+                get_context().logger.error(
+                    f"Could not finish operation because of {type.__name__}: {exception}"
+                )
+
+        assert self._trace is not None
+        get_context().tracing_database.save(self._trace)
+
+        return False
--- a/src/great_ai/great_ai/views/init.py
+++ b/src/great_ai/great_ai/views/init.py
@ -0,0 +1,11 @@
+from .api_metadata import ApiMetadata
+from .cache_statistics import CacheStatistics
+from .evaluation_feedback_request import EvaluationFeedbackRequest
+from .filter import Filter
+from .function_metadata import FunctionMetadata
+from .health_check_response import HealthCheckResponse
+from .model import Model
+from .operators import operators
+from .query import Query
+from .sort_by import SortBy
+from .trace import Trace
--- a/src/great_ai/great_ai/views/api_metadata.py
+++ b/src/great_ai/great_ai/views/api_metadata.py
@ -0,0 +1,10 @@
+from typing import Any
+
+from pydantic import BaseModel
+
+
+class ApiMetadata(BaseModel):
+    name: str
+    version: str
+    documentation: str
+    configuration: Any
--- a/src/great_ai/great_ai/views/cache_statistics.py
+++ b/src/great_ai/great_ai/views/cache_statistics.py
@ -0,0 +1,8 @@
+from pydantic import BaseModel
+
+
+class CacheStatistics(BaseModel):
+    hits: int
+    misses: int
+    size: int
+    max_size: int
--- a/src/great_ai/great_ai/views/evaluation_feedback_request.py
+++ b/src/great_ai/great_ai/views/evaluation_feedback_request.py
@ -0,0 +1,7 @@
+from typing import Any
+
+from pydantic import BaseModel
+
+
+class EvaluationFeedbackRequest(BaseModel):
+    feedback: Any
--- a/src/great_ai/great_ai/views/filter.py
+++ b/src/great_ai/great_ai/views/filter.py
@ -0,0 +1,11 @@
+from typing import Union
+
+from pydantic import BaseModel
+
+from .operators import Operator
+
+
+class Filter(BaseModel):
+    property: str
+    operator: Operator
+    value: Union[float, str]
--- a/src/great_ai/great_ai/views/function_metadata.py
+++ b/src/great_ai/great_ai/views/function_metadata.py
@ -0,0 +1,10 @@
+from typing import List
+
+from pydantic import BaseModel
+
+
+class FunctionMetadata(BaseModel):
+    input_parameter_names: List[str] = []
+    model_parameter_names: List[str] = []
+    model_versions: str = ""
+    is_finalised: bool = False
--- a/src/great_ai/great_ai/views/health_check_response.py
+++ b/src/great_ai/great_ai/views/health_check_response.py
@ -0,0 +1,8 @@
+from pydantic import BaseModel
+
+from .cache_statistics import CacheStatistics
+
+
+class HealthCheckResponse(BaseModel):
+    is_healthy: bool
+    cache_statistics: CacheStatistics
--- a/src/great_ai/great_ai/views/model.py
+++ b/src/great_ai/great_ai/views/model.py
@ -0,0 +1,6 @@
+from pydantic import BaseModel
+
+
+class Model(BaseModel):
+    key: str
+    version: int
--- a/src/great_ai/great_ai/views/operators.py
+++ b/src/great_ai/great_ai/views/operators.py
@ -0,0 +1,5 @@
+from typing import List, Literal
+
+Operator = Literal[">=", "<=", "<", ">", "!=", "=", "contains"]
+
+operators: List[Operator] = [">=", "<=", "<", ">", "!=", "=", "contains"]
--- a/src/great_ai/great_ai/views/query.py
+++ b/src/great_ai/great_ai/views/query.py
@ -0,0 +1,35 @@
+from datetime import datetime
+from typing import List, Optional, Sequence
+
+from pydantic import BaseModel
+
+from .filter import Filter
+from .sort_by import SortBy
+
+
+class Query(BaseModel):
+    filter: List[Filter] = []
+    sort: List[SortBy] = []
+    conjunctive_tags: Sequence[str] = []
+    since: Optional[datetime] = None
+    until: Optional[datetime] = None
+    has_feedback: Optional[bool] = None
+
+    class Config:
+        schema_extra = {
+            "example": {
+                "filter": [
+                    {
+                        "property": "original_execution_time_ms",
+                        "operator": ">",
+                        "value": 100,
+                    }
+                ],
+                "sort": [
+                    {"column_id": "original_execution_time_ms", "direction": "asc"},
+                    {"column_id": "id", "direction": "desc"},
+                ],
+                "conjunctive_tags": ["online"],
+                "has_feedback": False,
+            }
+        }
--- a/src/great_ai/great_ai/views/sort_by.py
+++ b/src/great_ai/great_ai/views/sort_by.py
@ -0,0 +1,8 @@
+from typing import Literal
+
+from pydantic import BaseModel
+
+
+class SortBy(BaseModel):
+    column_id: str
+    direction: Literal["asc", "desc"]
--- a/src/great_ai/great_ai/views/trace.py
+++ b/src/great_ai/great_ai/views/trace.py
@ -0,0 +1,86 @@
+from pprint import pformat
+from typing import Any, Dict, Generic, List, Optional, TypeVar
+from uuid import uuid4
+
+from pydantic import Extra, validator
+
+from ..helper import HashableBaseModel
+from .model import Model
+
+T = TypeVar("T")
+
+
+class Trace(Generic[T], HashableBaseModel):
+    trace_id: Optional[str]
+    created: str
+    original_execution_time_ms: float
+    logged_values: Dict[str, Any]
+    models: List[Model]
+    exception: Optional[str]
+    output: T
+    feedback: Any = None
+    tags: List[str]
+
+    class Config:
+        extra = Extra.ignore
+
+    @validator("trace_id", always=True)
+    def generate_id(cls, v: Optional[str], values: Dict[str, Any]) -> Optional[str]:
+        if not v:
+            return str(uuid4())
+        return v
+
+    @property
+    def input(self) -> Any:
+        return (
+            self.logged_values["input"]
+            if list(self.logged_values.keys()) == ["input"]
+            else self.logged_values
+        )
+
+    @property
+    def models_flat(self) -> str:
+        return ", ".join(f"{m.key}:{m.version}" for m in self.models)
+
+    @property
+    def output_flat(self) -> str:
+        return pformat(self.output, indent=2, compact=True)
+
+    @property
+    def exception_flat(self) -> str:
+        return (
+            "null"
+            if self.exception is None
+            else pformat(self.exception, indent=2, compact=True)
+        )
+
+    @property
+    def feedback_flat(self) -> str:
+        return (
+            "null"
+            if self.feedback is None
+            else pformat(self.feedback, indent=2, compact=True)
+        )
+
+    @property
+    def tags_flat(self) -> str:
+        return ",\n".join(self.tags)
+
+    def to_flat_dict(self, include_original: bool = True) -> Dict[str, Any]:
+        return {
+            **(
+                self.dict()
+                if include_original
+                else {
+                    "trace_id": self.trace_id,
+                    "created": self.created,
+                    "original_execution_time_ms": self.original_execution_time_ms,
+                }
+            ),
+            **self.logged_values,
+            "models_flat": self.models_flat,
+            "exception_flat": self.exception_flat,
+            "output_flat": self.output_flat,
+            "feedback_flat": self.feedback_flat,
+            "tags_flat": self.tags_flat,
+        }
--- a/src/great_ai/large_file/init.py
+++ b/src/great_ai/large_file/init.py
@ -0,0 +1 @@
+from .large_file import LargeFile, LargeFileLocal, LargeFileMongo, LargeFileS3
--- a/src/great_ai/large_file/main.py
+++ b/src/great_ai/large_file/main.py
@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+
+from argparse import Namespace
+from pathlib import Path
+from typing import Mapping, Type
+
+from ..utilities import get_logger
+from .large_file import LargeFile, LargeFileLocal, LargeFileMongo, LargeFileS3
+from .parse_arguments import parse_arguments
+
+logger = get_logger("large_file")
+
+
+def main() -> None:
+    parser, args = parse_arguments()
+
+    large_file = get_class(args)
+
+    if not args.cache and not args.push and not args.delete:
+        logger.warning("No action required.")
+        parser.print_help()
+
+    if args.cache:
+        for c in args.cache:
+            split = c.split(":")
+            file_name = split[0]
+            version = None if len(split) == 1 else int(split[1])
+            large_file(file_name, "r", version=version).get()
+
+    if args.push:
+        for p in args.push:
+            path = Path(p)
+            large_file(path.name, "w").push(path)
+
+    if args.delete:
+        for f in args.delete:
+            large_file(f).delete()
+
+
+def get_class(args: Namespace) -> Type[LargeFile]:
+    factory: Mapping[str, Type[LargeFile]] = {
+        "s3": LargeFileS3,
+        "local": LargeFileLocal,
+        "mongodb": LargeFileMongo,
+    }
+
+    if args.backend not in factory:
+        raise ValueError(
+            f"Backend {args.backend} does not exits, available options: {' ,'.join(factory.keys())}"
+        )
+
+    large_file = factory[args.backend]
+
+    if args.backend != "local":
+        if args.secrets is None:
+            raise ValueError(
+                "Providing a credentials file is required when the backend mode is not `local`."
+            )
+        large_file.configure_credentials_from_file(args.secrets)
+
+    return large_file
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except KeyboardInterrupt:
+        logger.warning("Exiting")
+        exit()
+    except Exception as e:
+        logger.exception(e)
--- a/src/great_ai/large_file/helper/init.py
+++ b/src/great_ai/large_file/helper/init.py
@ -0,0 +1,2 @@
+from .human_readable_to_byte import human_readable_to_byte
+from .progress_bar import DownloadProgressBar, UploadProgressBar
--- a/src/great_ai/large_file/helper/bytes_to_megabytes.py
+++ b/src/great_ai/large_file/helper/bytes_to_megabytes.py
@ -0,0 +1,2 @@
+def bytes_to_megabytes(bytes: int) -> str:
+    return f"{round(bytes / 1000 / 1000, 2):.2f}"
--- a/src/great_ai/large_file/helper/human_readable_to_byte.py
+++ b/src/great_ai/large_file/helper/human_readable_to_byte.py
@ -0,0 +1,31 @@
+import re
+
+
+def human_readable_to_byte(size: str) -> int:
+    """Case is ignored, kb, kB, Kb, and KB are all treated as kilobyte."""
+
+    if size.strip() == "0":
+        return 0
+
+    possible_units = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
+    units_re = "|".join(possible_units)
+    regex = re.compile(
+        rf"""
+        \s*                         # trim
+        (?P<scalar>\d+(.\d+)?)      # get scalar, it might be a float
+        \s*                         # ignore optional whitespace
+        (?P<unit>{units_re})        # capture the unit
+    """,
+        flags=re.VERBOSE | re.IGNORECASE,
+    )
+
+    match = regex.match(size)
+    if not match:
+        raise ValueError(f'Could not find values in "{size}"')
+
+    results = match.groupdict()
+
+    scalar = float(results["scalar"])
+    idx = possible_units.index(results["unit"].upper())
+    factor = 1024**idx
+    return round(scalar * factor)
--- a/src/great_ai/large_file/helper/progress_bar.py
+++ b/src/great_ai/large_file/helper/progress_bar.py
@ -0,0 +1,51 @@
+import os
+import threading
+from logging import Logger
+from pathlib import Path
+
+from .bytes_to_megabytes import bytes_to_megabytes
+
+
+class ProgressBar:
+    min_progress_percentage_change = 10
+
+    def __init__(self, file_size: int, logger: Logger, prefix: str):
+        self._file_size = file_size
+        self._logger = logger
+        self._prefix = prefix
+
+        self._last_percentage: float = 0
+        self._seen_so_far = 0
+        self._lock = threading.Lock()
+
+    def __call__(self, bytes_amount: int) -> None:
+        with self._lock:
+            self._seen_so_far += bytes_amount
+            percentage = (self._seen_so_far / float(self._file_size)) * 100
+
+            if (
+                percentage != 100
+                and percentage - self._last_percentage
+                < self.min_progress_percentage_change
+            ):
+                return
+
+            self._last_percentage += self.min_progress_percentage_change
+
+            file_size_mb = bytes_to_megabytes(self._file_size)
+            seen_so_far_mb = bytes_to_megabytes(self._seen_so_far)
+            progress = seen_so_far_mb.rjust(len(file_size_mb))
+            self._logger.info(
+                f"{self._prefix} {progress}/{file_size_mb} MB ({percentage:.1f}%)"
+            )
+
+
+class DownloadProgressBar(ProgressBar):
+    def __init__(self, name: str, size: int, logger: Logger):
+        super().__init__(file_size=size, logger=logger, prefix=f"Downloading {name}")
+
+
+class UploadProgressBar(ProgressBar):
+    def __init__(self, path: Path, logger: Logger):
+        size = os.path.getsize(path)
+        super().__init__(file_size=size, logger=logger, prefix=f"Uploading {path.name}")
--- a/src/great_ai/large_file/large_file/init.py
+++ b/src/great_ai/large_file/large_file/init.py
@ -0,0 +1,4 @@
+from .large_file import LargeFile
+from .large_file_local import LargeFileLocal
+from .large_file_mongo import LargeFileMongo
+from .large_file_s3 import LargeFileS3
--- a/src/great_ai/large_file/large_file/large_file.py
+++ b/src/great_ai/large_file/large_file/large_file.py
@ -0,0 +1,313 @@
+import os
+import shutil
+import tempfile
+from abc import ABC, abstractmethod
+from pathlib import Path
+from types import TracebackType
+from typing import IO, Any, List, Optional, Type, Union, cast
+
+from ...utilities import ConfigFile, get_logger
+from ..helper import human_readable_to_byte
+from ..models import DataInstance
+
+logger = get_logger("large_file")
+
+
+CACHE_NAME_VERSION_SEPARATOR = "-"
+COMPRESSION_ALGORITHM = "gztar"
+ARCHIVE_EXTENSION = ".tar.gz"
+
+
+class LargeFile(ABC):
+    """
+    Store large files remotely. Use local cache for speed up.
+
+    Examples:
+
+    ```
+    with LargeFile("test.txt", "w", keep_last_n=3) as f:
+        for i in range(1000000):
+            f.write('test\n')
+
+    with LargeFile("test.txt", "r") as f:
+        print(f.readlines()[0])
+
+    path_to_cached_text_file = LargeFile("test.txt", version=0).get()
+    ```
+
+    By default, files are stored in the ".cache" folder and the
+    least recently use is deleted after the overall size reaches 30 GBs.
+
+    Change it with the following properties.
+
+    ```
+    LargeFile.cache_path = Path(".cache")
+    LargeFile.max_cache_size = "30GB"
+    ```
+    """
+
+    initialized = False
+    cache_path = Path(".cache")
+    max_cache_size: Optional[str] = "30GB"
+
+    def __init__(
+        self,
+        name: str,
+        mode: str = "r",
+        *,
+        buffering: int = -1,
+        encoding: Optional[str] = None,
+        errors: Optional[str] = None,
+        newline: Optional[str] = None,
+        version: Optional[int] = None,
+        keep_last_n: Optional[int] = None,
+        cache_only_mode: bool = False,
+    ):
+        self._name = name
+        self._version = version
+        self._mode = mode
+        self._keep_last_n = keep_last_n
+        self._cache_only_mode = cache_only_mode
+
+        self._buffering = buffering
+        self._encoding = encoding
+        self._errors = errors
+        self._newline = newline
+
+        LargeFile.cache_path.mkdir(parents=True, exist_ok=True)
+
+        self._find_instances()
+        self._check_mode_and_set_version()
+
+    @classmethod
+    def configure_credentials_from_file(
+        cls,
+        secrets_path: Union[Path, str],
+    ) -> None:
+        cls.configure_credentials(**ConfigFile(secrets_path))
+
+    @classmethod
+    def configure_credentials(
+        cls,
+    ) -> None:
+        cls.initialized = True
+
+    def __enter__(self) -> IO:
+        self._file: IO[Any] = (
+            tempfile.NamedTemporaryFile(
+                mode=self._mode,
+                buffering=self._buffering,
+                encoding=self._encoding,
+                newline=self._newline,
+                errors=self._errors,
+                delete=False,
+                prefix="large_file-",
+            )
+            if "w" in self._mode
+            else open(
+                self.get(),
+                mode=self._mode,
+                buffering=self._buffering,
+                encoding=self._encoding,
+                newline=self._newline,
+                errors=self._errors,
+            )
+        )
+
+        return self._file
+
+    def __exit__(
+        self,
+        type: Optional[Type[BaseException]],
+        exc: Optional[BaseException],
+        traceback: Optional[TracebackType],
+    ) -> bool:
+        self._file.close()
+
+        if type is None:
+            if "w" in self._mode:
+                self.push(Path(self._file.name))
+                os.unlink(self._file.name)
+        else:
+            logger.exception("Could not finish operation.")
+
+        return True
+
+    @property
+    def _local_name(self) -> str:
+        return f"{self._name}{CACHE_NAME_VERSION_SEPARATOR}{self.version}"
+
+    @property
+    def version(self) -> int:
+        return cast(int, self._version)
+
+    def get(self, hide_progress: bool = False) -> Path:
+        remote_path = next(
+            i.remote_path for i in self._instances if i.version == self._version
+        )
+
+        destination = self.cache_path / self._local_name
+        if not destination.exists():
+            logger.info(f"File {self._local_name} does not exist locally")
+
+            with tempfile.TemporaryDirectory() as tmp:
+                local_root_path = Path(tmp)
+                tmp_file_archive = (
+                    local_root_path / f"{self._local_name}{ARCHIVE_EXTENSION}"
+                )
+                self._download(
+                    remote_path, tmp_file_archive, hide_progress=hide_progress
+                )
+
+                logger.info(f"Decompressing {self._local_name}")
+                shutil.unpack_archive(str(tmp_file_archive), tmp, COMPRESSION_ALGORITHM)
+                shutil.move(str(local_root_path / self._local_name), str(destination))
+        else:
+            logger.info(f"File {self._local_name} found in cache")
+
+        return destination
+
+    def push(self, path: Union[Path, str], hide_progress: bool = False) -> None:
+        if isinstance(path, str):
+            path = Path(path)
+
+        with tempfile.TemporaryDirectory() as tmp:
+            if path.is_file():
+                logger.info(f"Copying file for {self._local_name}")
+                copy: Any = shutil.copy
+            else:
+                logger.info(f"Copying directory for {self._local_name}")
+                copy = shutil.copytree
+
+            try:
+                # Make local copy in the cache
+                shutil.rmtree(self.cache_path / self._local_name, ignore_errors=True)
+                copy(str(path), str(self.cache_path / self._local_name))
+            except shutil.SameFileError:
+                pass  # No worries
+
+            copy(str(path), str(Path(tmp) / self._local_name))
+
+            with tempfile.TemporaryDirectory() as tmp2:
+                # A directory has to be zipped and it cannot contain the output of the zipping
+                logger.info(f"Compressing {self._local_name}")
+                shutil.make_archive(
+                    str(Path(tmp2) / self._local_name),
+                    COMPRESSION_ALGORITHM,
+                    tmp,
+                )
+
+                file_to_be_uploaded = (
+                    Path(tmp2) / f"{self._local_name}{ARCHIVE_EXTENSION}"
+                )
+                self._upload(file_to_be_uploaded, hide_progress=hide_progress)
+
+        self.clean_up()
+
+    def delete(self) -> None:
+        self._keep_last_n = 0
+        self._delete_old_remote_versions()
+
+    def _find_instances(self) -> None:
+        if self._cache_only_mode:
+            self._instances = self._find_instances_from_cache()
+        else:
+            self._instances = self._find_remote_instances()
+
+        self._instances = sorted(self._instances, key=lambda i: i.version)
+
+    def _find_instances_from_cache(self) -> List[DataInstance]:
+        logger.info(f"Fetching cached versions of {self._name}")
+
+        candidates = [
+            DataInstance(
+                name=CACHE_NAME_VERSION_SEPARATOR.join(
+                    f.name.split(CACHE_NAME_VERSION_SEPARATOR)[:-1]
+                ),
+                version=int(f.name.split(CACHE_NAME_VERSION_SEPARATOR)[-1]),
+                remote_path=f,
+            )
+            for f in self.cache_path.glob(
+                f"{self._name}{CACHE_NAME_VERSION_SEPARATOR}*"
+            )
+        ]
+
+        return [c for c in candidates if c.name == self._name]
+
+    def _check_mode_and_set_version(self) -> None:
+        if "+" in self._mode:
+            raise ValueError(
+                f"File mode `{self._mode}` is not allowed3, remove the `+`."
+            )
+
+        if "w" in self._mode:
+            if self._version is not None:
+                raise ValueError("Providing a version is not allowed in write mode.")
+
+            self._version = self._instances[-1].version + 1 if self._instances else 0
+
+        elif "r" in self._mode:
+            if not self._instances:
+                raise FileNotFoundError(
+                    f"File {self._name} not found. No versions are available."
+                )
+
+            if self._version is None:
+                self._version = self._instances[-1].version
+                logger.info(
+                    f"Latest version of {self._name} is {self._version} "
+                    + f"(from versions: {self.versions_pretty})"
+                )
+            elif self._version not in [i.version for i in self._instances]:
+                raise FileNotFoundError(
+                    f"File {self._name} not found with version {self._version}. "
+                    + f"(from versions: {self.versions_pretty})"
+                )
+        else:
+            raise ValueError("Unsupported file mode.")
+
+    @property
+    def versions_pretty(self) -> str:
+        return ", ".join((str(i.version) for i in self._instances))
+
+    def clean_up(self) -> None:
+        self._delete_old_remote_versions()
+        self._prune_cache()
+
+    def _prune_cache(self) -> None:
+        self.cache_path.mkdir(parents=True, exist_ok=True)
+
+        if self.max_cache_size is None:
+            return
+
+        allowed_size = human_readable_to_byte(self.max_cache_size)
+        assert allowed_size >= 0
+
+        least_recently_read = sorted(
+            [f for f in self.cache_path.glob("*")], key=lambda f: f.stat().st_atime
+        )
+
+        while sum(os.path.getsize(f) for f in least_recently_read) > allowed_size:
+            file = least_recently_read.pop(0)
+            logger.info(
+                f"Deleting file from cache to meet quota (max_cache_size={self.max_cache_size}): {file}"
+            )
+            os.unlink(file)
+
+    @abstractmethod
+    def _find_remote_instances(self) -> List[DataInstance]:
+        pass
+
+    @abstractmethod
+    def _download(
+        self, remote_path: Any, local_path: Path, hide_progress: bool
+    ) -> None:
+        pass
+
+    @abstractmethod
+    def _upload(self, local_path: Path, hide_progress: bool) -> None:
+        pass
+
+    @abstractmethod
+    def _delete_old_remote_versions(self) -> None:
+        pass
--- a/src/great_ai/large_file/large_file/large_file_local.py
+++ b/src/great_ai/large_file/large_file/large_file_local.py
@ -0,0 +1,58 @@
+from pathlib import Path
+from typing import Any, List, Optional
+
+from ...utilities import get_logger
+from ..models import DataInstance
+from .large_file import LargeFile
+
+logger = get_logger("large_file")
+
+
+class LargeFileLocal(LargeFile):
+    def __init__(
+        self,
+        name: str,
+        mode: str = "r",
+        *,
+        buffering: int = -1,
+        encoding: Optional[str] = None,
+        errors: Optional[str] = None,
+        newline: Optional[str] = None,
+        version: Optional[int] = None,
+        keep_last_n: Optional[int] = None,
+    ):
+        super().__init__(
+            name,
+            mode,
+            buffering=buffering,
+            encoding=encoding,
+            errors=errors,
+            newline=newline,
+            version=version,
+            keep_last_n=keep_last_n,
+            cache_only_mode=True,
+        )
+        super().configure_credentials()
+
+    def _find_remote_instances(self) -> List[DataInstance]:
+        return []
+
+    def _download(
+        self, remote_path: Any, local_path: Path, hide_progress: bool
+    ) -> None:
+        raise NotImplementedError()
+
+    def _upload(self, local_path: Path, hide_progress: bool) -> None:
+        pass  # the "upload" is already done py the parent's caching mechanism
+
+    def _delete_old_remote_versions(self) -> None:
+        if self._keep_last_n is not None:
+            for i in (
+                self._instances[: -self._keep_last_n]
+                if self._keep_last_n > 0
+                else self._instances
+            ):
+                logger.info(
+                    f"Removing old version (keep_last_n={self._keep_last_n}): {i.remote_path}"
+                )
+                i.remote_path.unlink()
--- a/src/great_ai/large_file/large_file/large_file_mongo.py
+++ b/src/great_ai/large_file/large_file/large_file_mongo.py
@ -0,0 +1,121 @@
+import re
+from functools import cached_property
+from pathlib import Path
+from typing import Any, List, Mapping
+
+from gridfs import DEFAULT_CHUNK_SIZE, Database, GridFSBucket
+from pymongo import MongoClient
+
+from ...utilities import get_logger
+from ..helper import DownloadProgressBar, UploadProgressBar
+from ..models import DataInstance
+from .large_file import LargeFile
+
+logger = get_logger("large_file")
+
+
+MONGO_NAME_VERSION_SEPARATOR = "_"
+
+
+class LargeFileMongo(LargeFile):
+    mongo_connection_string = None
+    mongo_database = None
+
+    @classmethod
+    def configure_credentials(  # type: ignore
+        cls,
+        *,
+        mongo_connection_string: str,
+        mongo_database: str,
+        **_: Mapping[str, Any],
+    ) -> None:
+        cls.mongo_connection_string = mongo_connection_string
+        cls.mongo_database = mongo_database
+        super().configure_credentials()
+
+    @cached_property
+    def _client(self) -> GridFSBucket:
+        if self.mongo_connection_string is None or self.mongo_database is None:
+            raise ValueError(
+                "Please configure the MongoDB access options by calling LargeFileMongo.configure_credentials or set offline_mode=True in the constructor."
+            )
+
+        db: Database = MongoClient(self.mongo_connection_string)[self.mongo_database]
+        return GridFSBucket(db)
+
+    def _find_remote_instances(self) -> List[DataInstance]:
+        logger.debug(f"Fetching Mongo (GridFS) versions of {self._name}")
+
+        return [
+            DataInstance(
+                name=MONGO_NAME_VERSION_SEPARATOR.join(
+                    f.name.split(MONGO_NAME_VERSION_SEPARATOR)[:-1]
+                ),
+                version=int(f.name.split(MONGO_NAME_VERSION_SEPARATOR)[-1]),
+                remote_path=(f._id, f.length),
+                origin="mongodb",
+            )
+            for f in self._client.find(
+                {
+                    "filename": re.compile(
+                        re.escape(self._name + MONGO_NAME_VERSION_SEPARATOR) + ".*"
+                    )
+                }
+            )
+        ]
+
+    def _download(
+        self, remote_path: Any, local_path: Path, hide_progress: bool
+    ) -> None:
+        logger.info(f"Downloading {remote_path[0]} from Mongo (GridFS)")
+
+        progress = (
+            DownloadProgressBar(
+                name=str(remote_path[0]), size=remote_path[1], logger=logger
+            )
+            if not hide_progress
+            else None
+        )
+        with self._client.open_download_stream(remote_path[0]) as stream:
+            with open(local_path, "wb") as f:
+                while True:
+                    content = stream.read(DEFAULT_CHUNK_SIZE)
+                    f.write(content)
+
+                    if progress:
+                        progress(len(content))
+                    if len(content) < DEFAULT_CHUNK_SIZE:
+                        break
+
+    def _upload(self, local_path: Path, hide_progress: bool) -> None:
+        logger.info(f"Uploading {local_path} to Mongo (GridFS)")
+
+        progress = (
+            UploadProgressBar(path=local_path, logger=logger)
+            if not hide_progress
+            else None
+        )
+        with self._client.open_upload_stream(
+            f"{self._name}{MONGO_NAME_VERSION_SEPARATOR}{self.version}"
+        ) as stream:
+            with open(local_path, "rb") as f:
+                while True:
+                    content = f.read(DEFAULT_CHUNK_SIZE)
+                    stream.write(content)
+
+                    if progress:
+                        progress(len(content))
+                    if len(content) < DEFAULT_CHUNK_SIZE:
+                        break
+
+    def _delete_old_remote_versions(self) -> None:
+        if self._keep_last_n is not None:
+            for i in (
+                self._instances[: -self._keep_last_n]
+                if self._keep_last_n > 0
+                else self._instances
+            ):
+                logger.info(
+                    f"Removing old version from MongoDB (GridFS) (keep_last_n={self._keep_last_n}): {i.name}{MONGO_NAME_VERSION_SEPARATOR}{i.version}"
+                )
+                self._client.delete(i.remote_path[0])
--- a/src/great_ai/large_file/large_file/large_file_s3.py
+++ b/src/great_ai/large_file/large_file/large_file_s3.py
@ -0,0 +1,151 @@
+from functools import cached_property
+from pathlib import Path
+from typing import Any, List, Mapping, Optional
+
+import boto3
+
+from ...utilities import get_logger
+from ..helper import DownloadProgressBar, UploadProgressBar
+from ..models import DataInstance
+from .large_file import LargeFile
+
+logger = get_logger("large_file")
+
+
+S3_NAME_VERSION_SEPARATOR = "/"
+
+
+class LargeFileS3(LargeFile):
+    """
+    Store large files in S3. Use local cache for speed up.
+
+    Examples:
+
+    ```
+    with LargeFile("test.txt", "w", keep_last_n=3) as f:
+        for i in range(1000000):
+            f.write('test\n')
+
+    with LargeFile("test.txt", "r") as f:
+        print(f.readlines()[0])
+
+    path_to_cached_text_file = LargeFile("test.txt", version=0).get()
+    ```
+
+    By default, files are stored in the ".cache" folder and the
+    least recently use is deleted after the overall size reaches 30 GBs.
+
+    Change it with the following properties.
+
+    ```
+    LargeFile.cache_path = Path(".cache")
+    LargeFile.max_cache_size = "30GB"
+    ```
+    """
+
+    region_name = None
+    access_key_id = None
+    secret_access_key = None
+    bucket_name = None
+    endpoint_url = None
+
+    @classmethod
+    def configure_credentials(  # type: ignore
+        cls,
+        *,
+        aws_region_name: str,
+        aws_access_key_id: str,
+        aws_secret_access_key: str,
+        large_files_bucket_name: str,
+        aws_endpoint_url: Optional[str] = None,
+        **_: Mapping[str, Any],
+    ) -> None:
+        cls.region_name = aws_region_name
+        cls.access_key_id = aws_access_key_id
+        cls.secret_access_key = aws_secret_access_key
+        cls.bucket_name = large_files_bucket_name
+        cls.endpoint_url = aws_endpoint_url
+        super().configure_credentials()
+
+    @cached_property
+    def _client(self) -> boto3.client:
+        if (
+            self.region_name is None
+            or self.access_key_id is None
+            or self.secret_access_key is None
+            or self.bucket_name is None
+        ):
+            raise ValueError(
+                "Please configure the S3 access options by calling LargeFileS3.configure_credentials or set offline_mode=True in the constructor."
+            )
+
+        return boto3.client(
+            "s3",
+            aws_access_key_id=self.access_key_id,
+            aws_secret_access_key=self.secret_access_key,
+            region_name=self.region_name,
+            endpoint_url=self.endpoint_url,
+        )
+
+    def _find_remote_instances(self) -> List[DataInstance]:
+        logger.debug(f"Fetching S3 versions of {self._name}")
+
+        found_objects = self._client.list_objects_v2(
+            Bucket=self.bucket_name, Prefix=self._name
+        )
+        return (
+            [
+                DataInstance(
+                    name=o["Key"].split(S3_NAME_VERSION_SEPARATOR)[0],
+                    version=int(o["Key"].split(S3_NAME_VERSION_SEPARATOR)[-1]),
+                    remote_path=o["Key"],
+                )
+                for o in found_objects["Contents"]
+                if o["Key"].split(S3_NAME_VERSION_SEPARATOR)[0] == self._name
+            ]
+            if "Contents" in found_objects
+            else []
+        )
+
+    def _download(
+        self, remote_path: Any, local_path: Path, hide_progress: bool
+    ) -> None:
+        logger.info(f"Downloading {remote_path} from S3")
+
+        size = self._client.head_object(Bucket=self.bucket_name, Key=remote_path)[
+            "ContentLength"
+        ]
+
+        self._client.download_file(
+            Bucket=self.bucket_name,
+            Key=remote_path,
+            Filename=str(local_path),
+            Callback=None
+            if hide_progress
+            else DownloadProgressBar(name=str(remote_path), size=size, logger=logger),
+        )
+
+    def _upload(self, local_path: Path, hide_progress: bool) -> None:
+        key = f"{self._name}/{self.version}"
+        logger.info(f"Uploading {self._local_name} to S3 as {key}")
+
+        self._client.upload_file(
+            Filename=str(local_path),
+            Bucket=self.bucket_name,
+            Key=key,
+            Callback=None
+            if hide_progress
+            else UploadProgressBar(path=local_path, logger=logger),
+        )
+
+    def _delete_old_remote_versions(self) -> None:
+        if self._keep_last_n is not None:
+            for i in (
+                self._instances[: -self._keep_last_n]
+                if self._keep_last_n > 0
+                else self._instances
+            ):
+                logger.info(
+                    f"Removing old version from S3 (keep_last_n={self._keep_last_n}): {i.remote_path}"
+                )
+                self._client.delete_object(Bucket=self.bucket_name, Key=i.remote_path)
--- a/src/great_ai/large_file/models/init.py
+++ b/src/great_ai/large_file/models/init.py
@ -0,0 +1 @@
+from .data_instance import DataInstance
--- a/src/great_ai/large_file/models/data_instance.py
+++ b/src/great_ai/large_file/models/data_instance.py
@ -0,0 +1,9 @@
+from typing import Any
+
+from pydantic import BaseModel
+
+
+class DataInstance(BaseModel):
+    name: str
+    version: int
+    remote_path: Any
--- a/src/great_ai/large_file/parse_arguments.py
+++ b/src/great_ai/large_file/parse_arguments.py
@ -0,0 +1,56 @@
+from argparse import ArgumentParser, Namespace
+from typing import Tuple
+
+
+def parse_arguments() -> Tuple[ArgumentParser, Namespace]:
+    parser = ArgumentParser(
+        description="Store and version large files in S3; open them like regular files. Caching included.",
+    )
+
+    parser.add_argument(
+        "-b",
+        "--backend",
+        type=str,
+        help="choose which backend to use, available options: `local`, `s3`, `mongodb`",
+        required=True,
+    )
+
+    parser.add_argument(
+        "-s",
+        "--secrets",
+        type=str,
+        help="path to an .ini configuration file with your S3 credentials",
+        required=False,
+    )
+
+    parser.add_argument(
+        "-c",
+        "--cache",
+        nargs="+",
+        type=str,
+        help="download file into local cache, example: file_name.txt:version",
+        required=False,
+    )
+
+    parser.add_argument(
+        "-p",
+        "--push",
+        nargs="+",
+        type=str,
+        help="push a local file into S3 and set it as the most recent version",
+        required=False,
+    )
+
+    parser.add_argument(
+        "-d",
+        "--delete",
+        nargs="+",
+        type=str,
+        help="delete every version of file from S3",
+        required=False,
+    )
+
+    parser.print_usage = parser.print_help  # type: ignore
+    args = parser.parse_args()
+
+    return parser, args
--- a/src/great_ai/parse_arguments.py
+++ b/src/great_ai/parse_arguments.py
@ -0,0 +1,47 @@
+from argparse import ArgumentParser, Namespace
+
+
+def parse_arguments() -> Namespace:
+    parser = ArgumentParser(
+        description="GreatAI-Server for deploying you AI applications with ease.",
+    )
+
+    parser.add_argument(
+        "file_name",
+        type=str,
+        help="the name of the file containing your to-be-served function such as `main.py`\n",
+    )
+
+    parser.add_argument(
+        "--host",
+        type=str,
+        help="it is passed to uvicorn which starts a server listening on this address",
+        default="0.0.0.0",
+        required=False,
+    )
+
+    parser.add_argument(
+        "--port",
+        type=int,
+        help="it is passed to uvicorn which starts a server listening on this port",
+        default=6060,
+        required=False,
+    )
+
+    parser.add_argument(
+        "--timeout_keep_alive",
+        type=int,
+        help="it is passed to uvicorn which uses it for timing out requests taking longer than this many seconds",
+        default=600,
+        required=False,
+    )
+
+    parser.add_argument(
+        "--workers",
+        type=int,
+        help="it is passed to uvicorn which starts this many server processes",
+        default=1,
+        required=False,
+    )
+
+    return parser.parse_args()
--- a/src/great_ai/utilities/init.py
+++ b/src/great_ai/utilities/init.py
@ -0,0 +1,14 @@
+from .clean import clean
+from .config_file import ConfigFile
+from .evaluate_ranking import evaluate_ranking
+from .get_sentences import get_sentences
+from .language import english_name_of_language, is_english, predict_language
+from .lemmatize_text import lemmatize_text
+from .lemmatize_token import lemmatize_token
+from .logger import get_logger
+from .match_names import match_names
+from .matcher import fast_tokenize, filter_sentences, normalize
+from .nlp import nlp
+from .parallel_map import parallel_map
+from .publication_tei import PublicationTEI
+from .unique import unique
--- a/src/great_ai/utilities/clean.py
+++ b/src/great_ai/utilities/clean.py
@ -0,0 +1,68 @@
+import html
+import re
+import unicodedata
+
+import unidecode
+
+from .data import left_regular_punctuations, right_regular_punctuations
+from .external.pylatexenc.latex2text import LatexNodes2Text
+from .logger import get_logger
+
+logger = get_logger("clean")
+latex = LatexNodes2Text()
+
+
+joined_left_punctuations = "".join(left_regular_punctuations).replace("]", r"\]")
+joined_right_punctuations = "".join(right_regular_punctuations).replace("[", r"\[")
+
+
+def clean(
+    text: str,
+    ignore_xml: bool = False,
+    ignore_latex: bool = False,
+    remove_brackets: bool = False,
+    convert_to_ascii: bool = False,
+) -> str:
+    if not ignore_xml:
+        text = re.sub(r"<[^>]*>", " ", text)
+        text = html.unescape(text)
+
+    if not ignore_latex:
+        text = text.replace("%", "\\%")  # escape LaTeX comments before parsing as LaTeX
+
+        try:
+            text = latex.latex_to_text(text, tolerant_parsing=True, strict_braces=False)
+            text = text.replace("%s", " ")
+        except:
+            logger.exception("Latex parsing error")
+
+    if convert_to_ascii:
+        text = unicodedata.normalize("NFKD", text)
+
+        try:
+            text.encode("ASCII", errors="strict")
+        except UnicodeEncodeError:
+            text = "".join([c for c in text if not unicodedata.combining(c)])
+            text = unidecode.unidecode(text)
+
+    text = re.sub(
+        r"\b[a-zA-Z](?:[\t ]+[a-zA-Z]\b)+", lambda m: re.sub(r"[\t ]", "", m[0]), text
+    )  # A R T I C L E => ARTICLE
+
+    if remove_brackets:
+        text = re.sub(r"\[[^\]]*\]", " ", text)
+
+    # fix hypens: break- word => break-word
+    text = re.sub(r"(\S)-\s+", r"\1-", text)
+    text = re.sub(r"\s+-(\S)", r"-\1", text)
+
+    # collapse whitespace
+    text = re.sub(r"\s+", " ", text)
+
+    # fix punctuation
+    text = re.sub(rf" ([{joined_left_punctuations}])", r"\1", text)
+    text = re.sub(rf"([{joined_right_punctuations}]) ", r"\1", text)
+
+    text = text.strip()
+
+    return text
--- a/src/great_ai/utilities/config_file/init.py
+++ b/src/great_ai/utilities/config_file/init.py
@ -0,0 +1,2 @@
+from .config_file import ConfigFile
+from .parse_error import ParseError
--- a/src/great_ai/utilities/config_file/config_file.py
+++ b/src/great_ai/utilities/config_file/config_file.py
@ -0,0 +1,87 @@
+import os
+from pathlib import Path
+from typing import Dict, Iterable, Tuple, Union
+
+from ..logger import get_logger
+from .parse_error import ParseError
+from .pattern import pattern
+
+ENVIRONMENT_VARIABLE_KEY_PREFIX = "ENV"
+
+logger = get_logger("ConfigFile")
+
+
+class ConfigFile:
+    def __init__(
+        self, path: Union[Path, str], ignore_missing_environment_variables: bool = False
+    ) -> None:
+        if not isinstance(path, Path):
+            path = Path(path)
+
+        if not path.exists():
+            raise FileNotFoundError(path.absolute())
+
+        self._path = path
+        self._ignore_missing_environment_variables = (
+            ignore_missing_environment_variables
+        )
+        self._key_values: Dict[str, str] = {}
+
+        self._parse()
+
+    def _parse(self):
+        with open(self._path, encoding="utf-8") as f:
+            lines: str = f.read()
+
+        matches = pattern.findall(lines)
+        for key, *values in matches:
+            if key in self._key_values:
+                raise KeyError(
+                    f"Key `{key}` has been already defined and its value is `{self._key_values[key]}`"
+                )
+
+            try:
+                value = next(v for v in values if v)
+            except StopIteration:
+                raise ParseError(
+                    f"Cannot parse config file ({self._path.absolute()}), error at key `{key}`"
+                )
+
+            if value.startswith(f"{ENVIRONMENT_VARIABLE_KEY_PREFIX}:"):
+                _, value = value.split(":")
+                if value not in os.environ:
+                    issue = f'The value of `{key}` contains the "{ENVIRONMENT_VARIABLE_KEY_PREFIX}` prefix but `{value}` is not defined as an environment variable'
+                    if self._ignore_missing_environment_variables:
+                        logger.warning(f"{issue}, defaulting to `None`")
+                    else:
+                        raise KeyError(issue)
+                value = os.environ[value]
+
+            self._key_values[key] = value
+
+    def __getattr__(self, key: str) -> str:
+        if key in self._key_values:
+            return self._key_values[key]
+        raise KeyError(
+            f"Key `{key}` is not found in configuration file ({self._path.absolute()})"
+        )
+
+    __getitem__ = __getattr__
+
+    def __iter__(self) -> Iterable[Tuple[str, str]]:
+        return iter(self._key_values)
+
+    def __len__(self) -> int:
+        return len(self._key_values)
+
+    def keys(self):
+        return self._key_values.keys()
+
+    def values(self):
+        return self._key_values.values()
+
+    def items(self):
+        return self._key_values.items()
+
+    def __repr__(self):
+        return f"{type(self).__name__}(\n  path={self._path},\n  ignore_missing_environment_variables={self._ignore_missing_environment_variables}\n) {{{self._key_values}}}"
--- a/src/great_ai/utilities/config_file/parse_error.py
+++ b/src/great_ai/utilities/config_file/parse_error.py
@ -0,0 +1,2 @@
+class ParseError(Exception):
+    pass
--- a/src/great_ai/utilities/config_file/pattern.py
+++ b/src/great_ai/utilities/config_file/pattern.py
@ -0,0 +1,20 @@
+import re
+
+pattern = re.compile(
+    r"""
+        (?:^|\n)        # new key-value pairs must start on a new line
+        \s*             # leading whitespace is allowed
+        (?!\#)          # the key cannot start with a `#` symbol
+        (\w+?)          # then comes the key
+        \s*=\s*         # the key and value are separated by an equal sign
+        (?:             # then comes the value
+              "([^"]*)"   # the value can be surrounded by quotes: "value"
+            | '([^']*)'   # the value can be surrounded by quotes: 'value'
+            | `([^`]*)`   # the value can be surrounded by quotes: `value`
+            | ([^#\n]*?)  # or it is bare, in that case, the trailing whitespace is ignored
+        )
+        \s*(?:\#.*)?    # comments can be added with the `#` symbol
+        (?=\n|$)        # a key-value pairs are separated by new lines
+    """,
+    flags=re.UNICODE | re.VERBOSE,
+)
--- a/src/great_ai/utilities/data/init.py
+++ b/src/great_ai/utilities/data/init.py
@ -0,0 +1,6 @@
+from .american_spellings import american_spellings
+from .punctuations import (
+    left_regular_punctuations,
+    right_regular_punctuations,
+    sentence_ending_punctuations,
+)
--- a/src/great_ai/utilities/data/american_spellings.py
+++ b/src/great_ai/utilities/data/american_spellings.py
--- a/src/great_ai/utilities/data/punctuations.py
+++ b/src/great_ai/utilities/data/punctuations.py
@ -0,0 +1,22 @@
+sentence_ending_punctuations = [".", "?", "!", ":", ";"]
+
+# punctuations that usually have no space between them and the character to their left
+left_regular_punctuations = [
+    *sentence_ending_punctuations,
+    ",",
+    "'",
+    "%",
+    ")",
+    "}",
+    "]",
+]
+
+# punctuations that usually have no space between them and the character to their right
+right_regular_punctuations = [
+    "(",
+    "{",
+    "[",
+    "$",
+    "€",
+    "#",
+]
--- a/src/great_ai/utilities/evaluate_ranking/init.py
+++ b/src/great_ai/utilities/evaluate_ranking/init.py
@ -0,0 +1,2 @@
+from .draw_f1_iso_lines import draw_f1_iso_lines
+from .evaluate_ranking import evaluate_ranking
--- a/src/great_ai/utilities/evaluate_ranking/draw_f1_iso_lines.py
+++ b/src/great_ai/utilities/evaluate_ranking/draw_f1_iso_lines.py
@ -0,0 +1,28 @@
+from typing import Optional
+
+import matplotlib.pyplot as plt
+import numpy as np
+from matplotlib.axes import Axes
+
+
+def draw_f1_iso_lines(
+    resolution: int = 1000,
+    min: float = 0.2,
+    max: float = 0.8,
+    steps: int = 4,
+    axes: Optional[Axes] = None,
+) -> None:
+    if axes is None:
+        axes = plt.axes()
+
+    for f_score in np.linspace(min, max, num=steps):
+        x = np.linspace(f_score / (2 - f_score), 1, num=resolution)
+        y = f_score * x / (2 * x - f_score)
+
+        axes.plot(x[y >= 0], y[y >= 0], color="gray", alpha=0.2)
+
+        axes.annotate(
+            f"f1={f_score:0.1f}",
+            backgroundcolor="w",
+            xy=(0.9, y[int(resolution * 0.9)] + 0.02),
+        )
--- a/src/great_ai/utilities/evaluate_ranking/evaluate_ranking.py
+++ b/src/great_ai/utilities/evaluate_ranking/evaluate_ranking.py
@ -0,0 +1,90 @@
+from pathlib import Path
+from typing import Dict, List, Optional, Union
+
+import matplotlib
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.metrics import average_precision_score, precision_recall_curve
+
+from ..unique import unique
+from .draw_f1_iso_lines import draw_f1_iso_lines
+
+
+def evaluate_ranking(
+    expected: List[Union[str, float]],
+    actual_scores: List[float],
+    target_recall: float,
+    title: Optional[str] = "",
+    disable_interpolation: bool = False,
+    axes: Optional[plt.Axes] = None,
+    output_svg: Optional[Path] = None,
+    reverse_order: bool = False,
+    plot: bool = True,
+) -> Dict[Union[str, float], float]:
+    assert 0 <= target_recall <= 1
+
+    if plot and axes is None:
+        fig = plt.figure(figsize=(20, 10))
+        fig.patch.set_facecolor("white")
+        ax = plt.axes()
+    else:
+        ax = axes
+
+    classes = sorted(unique(expected), reverse=reverse_order)
+    str_classes = [str(c) for c in classes]
+
+    with matplotlib.rc_context({"font.size": 20}):
+        if plot:
+            ax.set_xmargin(0)
+
+            draw_f1_iso_lines(axes=ax)
+
+        results: Dict[Union[str, float], float] = {}
+        for i in range(len(classes) - 1):
+            binarized_expected = [
+                (v < classes[i]) if reverse_order else (v > classes[i])
+                for v in expected
+            ]
+            precision, recall, _ = precision_recall_curve(
+                binarized_expected, actual_scores
+            )
+
+            if not disable_interpolation:
+                for j in range(1, len(precision)):
+                    precision[j] = max(precision[j - 1], precision[j])
+
+            closest_recall_index = np.argmin(np.abs(recall - target_recall))
+            precision_at_closest_recall = precision[closest_recall_index]
+            average_precision = average_precision_score(
+                binarized_expected, actual_scores
+            )
+            results[classes[i]] = precision_at_closest_recall
+
+            if plot:
+                ax.plot(
+                    recall,
+                    precision,
+                    label=f"{'|'.join(str_classes[:i + 1])} ↔ {'|'.join(str_classes[i+1:])} (P@{target_recall:.2f}={precision_at_closest_recall:.2f}, AP={average_precision:.2f})",
+                )
+
+        if plot:
+            ax.legend(loc="upper right")
+            ax.axvline(x=target_recall, linestyle="--", color="#55c6bb", linewidth=2.0)
+
+            if title is None:
+                title = "Ranking evaluation"
+
+            ax.set_title(f'{title} ({" < ".join(str_classes)})', pad=20)
+
+            ax.set_xlabel("Recall")
+            ax.set_ylabel("Precision")
+
+            ax.set_xticks([target_recall] + sorted(ax.get_xticks()))
+
+        if plot and output_svg is None:
+            if axes is None:
+                plt.show()
+        elif output_svg:
+            plt.savefig(output_svg, format="svg")
+
+    return results
--- a/src/great_ai/utilities/external/init.py
+++ b/src/great_ai/utilities/external/init.py
--- a/src/great_ai/utilities/external/negspacy/README.md
+++ b/src/great_ai/utilities/external/negspacy/README.md
@ -0,0 +1 @@
+https://github.com/jenojp/negspacy
--- a/src/great_ai/utilities/external/negspacy/init.py
+++ b/src/great_ai/utilities/external/negspacy/init.py
--- a/src/great_ai/utilities/external/negspacy/negation.py
+++ b/src/great_ai/utilities/external/negspacy/negation.py
@ -0,0 +1,222 @@
+import logging
+
+from spacy.language import Language
+from spacy.matcher import PhraseMatcher
+from spacy.tokens import Token
+
+from .termsets import termset
+
+default_ts = termset("en").get_patterns()
+
+
+@Language.factory(
+    "negex",
+    default_config={
+        "neg_termset": default_ts,
+        "extension_name": "negex",
+        "chunk_prefix": list(),
+    },
+)
+class Negex:
+    """
+        A spaCy pipeline component which identifies negated tokens in text.
+
+        Based on: NegEx - A Simple Algorithm for Identifying Negated Findings and Diseasesin Discharge Summaries
+    Chapman, Bridewell, Hanbury, Cooper, Buchanan
+
+    Parameters
+    ----------
+    nlp: object
+        spaCy language object
+    termset_lang: str
+        language code, if using default termsets (e.g. "en" for english)
+    extension_name: str
+        defaults to "negex"; whether entity is negated is then available as ent._.negex
+    pseudo_negations: list
+        list of phrases that cancel out a negation, if empty, defaults are used
+    preceding_negations: list
+        negations that appear before an entity, if empty, defaults are used
+    following_negations: list
+        negations that appear after an entity, if empty, defaults are used
+    termination: list
+        phrases that "terminate" a sentence for processing purposes such as "but". If empty, defaults are used
+
+    """
+
+    def __init__(
+        self,
+        nlp: Language,
+        name: str,
+        neg_termset: dict,
+        extension_name: str,
+        chunk_prefix: list,
+    ):
+        if not Token.has_extension(extension_name):
+            Token.set_extension(extension_name, default=False, force=True)
+
+        ts = neg_termset
+        expected_keys = [
+            "pseudo_negations",
+            "preceding_negations",
+            "following_negations",
+            "termination",
+        ]
+        if not set(ts.keys()) == set(expected_keys):
+            raise KeyError(
+                f"Unexpected or missing keys in 'neg_termset', expected: {expected_keys}, instead got: {list(ts.keys())}"
+            )
+
+        self.pseudo_negations = ts["pseudo_negations"]
+        self.preceding_negations = ts["preceding_negations"]
+        self.following_negations = ts["following_negations"]
+        self.termination = ts["termination"]
+
+        self.nlp = nlp
+        self.extension_name = extension_name
+        self.build_patterns()
+        self.chunk_prefix = list(nlp.tokenizer.pipe(chunk_prefix))
+
+    def build_patterns(self):
+        # efficiently build spaCy matcher patterns
+        self.matcher = PhraseMatcher(self.nlp.vocab, attr="LOWER")
+
+        self.pseudo_patterns = list(self.nlp.tokenizer.pipe(self.pseudo_negations))
+        self.matcher.add("pseudo", None, *self.pseudo_patterns)
+
+        self.preceding_patterns = list(
+            self.nlp.tokenizer.pipe(self.preceding_negations)
+        )
+        self.matcher.add("Preceding", None, *self.preceding_patterns)
+
+        self.following_patterns = list(
+            self.nlp.tokenizer.pipe(self.following_negations)
+        )
+        self.matcher.add("Following", None, *self.following_patterns)
+
+        self.termination_patterns = list(self.nlp.tokenizer.pipe(self.termination))
+        self.matcher.add("Termination", None, *self.termination_patterns)
+
+    def process_negations(self, doc):
+        """
+        Find negations in doc and clean candidate negations to remove pseudo negations
+
+        Parameters
+        ----------
+        doc: object
+            spaCy Doc object
+
+        Returns
+        -------
+        preceding: list
+            list of tuples for preceding negations
+        following: list
+            list of tuples for following negations
+        terminating: list
+            list of tuples of terminating phrases
+
+        """
+        ###
+        # does not work properly in spacy 2.1.8. Will incorporate after 2.2.
+        # Relying on user to use NER in meantime
+        # see https://github.com/jenojp/negspacy/issues/7
+        ###
+        # if not doc.is_nered:
+        #     raise ValueError(
+        #         "Negations are evaluated for Named Entities found in text. "
+        #         "Your SpaCy pipeline does not included Named Entity resolution. "
+        #         "Please ensure it is enabled or choose a different language model that includes it."
+        #     )
+        preceding = list()
+        following = list()
+        terminating = list()
+
+        matches = self.matcher(doc)
+        pseudo = [
+            (match_id, start, end)
+            for match_id, start, end in matches
+            if self.nlp.vocab.strings[match_id] == "pseudo"
+        ]
+
+        for match_id, start, end in matches:
+            if self.nlp.vocab.strings[match_id] == "pseudo":
+                continue
+            pseudo_flag = False
+            for p in pseudo:
+                if start >= p[1] and start <= p[2]:
+                    pseudo_flag = True
+                    continue
+            if not pseudo_flag:
+                if self.nlp.vocab.strings[match_id] == "Preceding":
+                    preceding.append((match_id, start, end))
+                elif self.nlp.vocab.strings[match_id] == "Following":
+                    following.append((match_id, start, end))
+                elif self.nlp.vocab.strings[match_id] == "Termination":
+                    terminating.append((match_id, start, end))
+                else:
+                    logging.warnings(
+                        f"phrase {doc[start:end].text} not in one of the expected matcher types."
+                    )
+        return preceding, following, terminating
+
+    def termination_boundaries(self, doc, terminating):
+        """
+        Create sub sentences based on terminations found in text.
+
+        Parameters
+        ----------
+        doc: object
+            spaCy Doc object
+        terminating: list
+            list of tuples with (match_id, start, end)
+
+        returns
+        -------
+        boundaries: list
+            list of tuples with (start, end) of spans
+
+        """
+        sent_starts = [sent.start for sent in doc.sents]
+        terminating_starts = [t[1] for t in terminating]
+        starts = sent_starts + terminating_starts + [len(doc)]
+        starts.sort()
+        boundaries = list()
+        index = 0
+        for i, start in enumerate(starts):
+            if not i == 0:
+                boundaries.append((index, start))
+            index = start
+        return boundaries
+
+    def negex(self, doc):
+        """
+        Negates entities of interest
+
+        Parameters
+        ----------
+        doc: object
+            spaCy Doc object
+
+        """
+        preceding, following, terminating = self.process_negations(doc)
+        boundaries = self.termination_boundaries(doc, terminating)
+        for b in boundaries:
+            sub_preceding = [i for i in preceding if b[0] <= i[1] < b[1]]
+            sub_following = [i for i in following if b[0] <= i[1] < b[1]]
+
+            for e in doc[b[0] : b[1]]:
+                if any(pre < e.i for pre in [i[1] for i in sub_preceding]):
+                    e._.set(self.extension_name, True)
+                    continue
+                if any(fol > e.i for fol in [i[2] for i in sub_following]):
+                    e._.set(self.extension_name, True)
+                    continue
+                if self.chunk_prefix:
+                    if any(
+                        e.text.lower().startswith(c.text.lower())
+                        for c in self.chunk_prefix
+                    ):
+                        e._.set(self.extension_name, True)
+        return doc
+
+    def __call__(self, doc):
+        return self.negex(doc)
--- a/src/great_ai/utilities/external/negspacy/termsets.py
+++ b/src/great_ai/utilities/external/negspacy/termsets.py
@ -0,0 +1,229 @@
+"""
+Default termsets for various languages
+"""
+
+LANGUAGES = dict()
+
+# english termset dictionary
+en = dict()
+pseudo = [
+    "no further",
+    "not able to be",
+    "not certain if",
+    "not certain whether",
+    "not necessarily",
+    "without any further",
+    "without difficulty",
+    "without further",
+    "might not",
+    "not only",
+    "no increase",
+    "no significant change",
+    "no change",
+    "no definite change",
+    "not extend",
+    "not cause",
+]
+en["pseudo_negations"] = pseudo
+
+preceding = [
+    "absence of",
+    "declined",
+    "denied",
+    "denies",
+    "denying",
+    "no sign of",
+    "no signs of",
+    "not",
+    "not demonstrate",
+    "symptoms atypical",
+    "doubt",
+    "negative for",
+    "no",
+    "versus",
+    "without",
+    "doesn't",
+    "doesnt",
+    "don't",
+    "dont",
+    "didn't",
+    "didnt",
+    "wasn't",
+    "wasnt",
+    "weren't",
+    "werent",
+    "isn't",
+    "isnt",
+    "aren't",
+    "arent",
+    "cannot",
+    "can't",
+    "cant",
+    "couldn't",
+    "couldnt",
+    "never",
+]
+en["preceding_negations"] = preceding
+
+following = [
+    "declined",
+    "unlikely",
+    "was not",
+    "were not",
+    "wasn't",
+    "wasnt",
+    "weren't",
+    "werent",
+]
+en["following_negations"] = following
+
+termination = [
+    "although",
+    "apart from",
+    "as there are",
+    "aside from",
+    "but",
+    "except",
+    "however",
+    "involving",
+    "nevertheless",
+    "still",
+    "though",
+    "which",
+    "yet",
+]
+en["termination"] = termination
+
+LANGUAGES["en"] = en
+
+# en_clinical builds upon en
+en_clinical = dict()
+pseudo_clinical = pseudo + [
+    "gram negative",
+    "not rule out",
+    "not ruled out",
+    "not been ruled out",
+    "not drain",
+    "no suspicious change",
+    "no interval change",
+    "no significant interval change",
+]
+en_clinical["pseudo_negations"] = pseudo_clinical
+
+preceding_clinical = preceding + [
+    "patient was not",
+    "without indication of",
+    "without sign of",
+    "without signs of",
+    "without any reactions or signs of",
+    "no complaints of",
+    "no evidence of",
+    "no cause of",
+    "evaluate for",
+    "fails to reveal",
+    "free of",
+    "never developed",
+    "never had",
+    "did not exhibit",
+    "rules out",
+    "rule out",
+    "rule him out",
+    "rule her out",
+    "rule patient out",
+    "rule the patient out",
+    "ruled out",
+    "ruled him out",
+    "ruled her out",
+    "ruled patient out",
+    "ruled the patient out",
+    "r/o",
+    "ro",
+]
+en_clinical["preceding_negations"] = preceding_clinical
+
+following_clinical = following + ["was ruled out", "were ruled out", "free"]
+en_clinical["following_negations"] = following_clinical
+
+termination_clinical = termination + [
+    "cause for",
+    "cause of",
+    "causes for",
+    "causes of",
+    "etiology for",
+    "etiology of",
+    "origin for",
+    "origin of",
+    "origins for",
+    "origins of",
+    "other possibilities of",
+    "reason for",
+    "reason of",
+    "reasons for",
+    "reasons of",
+    "secondary to",
+    "source for",
+    "source of",
+    "sources for",
+    "sources of",
+    "trigger event for",
+]
+en_clinical["termination"] = termination_clinical
+LANGUAGES["en_clinical"] = en_clinical
+
+en_clinical_sensitive = dict()
+
+preceding_clinical_sensitive = preceding_clinical + [
+    "concern for",
+    "supposed",
+    "which causes",
+    "leads to",
+    "h/o",
+    "history of",
+    "instead of",
+    "if you experience",
+    "if you get",
+    "teaching the patient",
+    "taught the patient",
+    "teach the patient",
+    "educated the patient",
+    "educate the patient",
+    "educating the patient",
+    "monitored for",
+    "monitor for",
+    "test for",
+    "tested for",
+]
+en_clinical_sensitive["pseudo_negations"] = pseudo_clinical
+en_clinical_sensitive["preceding_negations"] = preceding_clinical_sensitive
+en_clinical_sensitive["following_negations"] = following_clinical
+en_clinical_sensitive["termination"] = termination_clinical
+
+LANGUAGES["en_clinical_sensitive"] = en_clinical_sensitive
+
+
+class termset:
+    def __init__(self, termset_lang):
+        self.pattern_types = [
+            "pseudo_negations",
+            "preceding_negations",
+            "following_negations",
+            "termination",
+        ]
+        self.terms = LANGUAGES[termset_lang]
+
+    def get_patterns(self):
+        return self.terms
+
+    def remove_patterns(self, pattern_dict):
+        for key, value in pattern_dict.items():
+            if key in self.pattern_types:
+                self.terms[key] = [i for i in self.terms[key] if i not in value]
+            else:
+                raise ValueError(f"Unexpected key: {key} not in {self.pattern_types}")
+
+    def add_patterns(self, pattern_dict):
+        for key, value in pattern_dict.items():
+            if key in self.pattern_types:
+                self.terms[key] = list(set(self.terms[key] + value))
+            else:
+                raise ValueError(f"Unexpected key: {key} not in {self.pattern_types}")
--- a/Show more
+++ b/Show more
				`@ -0,0 +1 @@`
				`from .create_dash_app import create_dash_app`
				`@ -0,0 +1 @@`
				`from .large_file import LargeFile, LargeFileLocal, LargeFileMongo, LargeFileS3`