Move files

great_ai/__init__.py

@@ -0,0 +1,23 @@
+from .context import configure
+from .deploy import GreatAI
+from .exceptions import (
+    ArgumentValidationError,
+    MissingArgumentError,
+    WrongDecoratorOrderError,
+)
+from .models import save_model, use_model
+from .output_views import (
+    ClassificationOutput,
+    MultiLabelClassificationOutput,
+    RegressionOutput,
+)
+from .parameters import log_metric, parameter
+from .persistence import MongodbDriver, ParallelTinyDbDriver, TracingDatabaseDriver
+from .remote import (
+    HttpClient,
+    RemoteCallError,
+    call_remote_great_ai,
+    call_remote_great_ai_async,
+)
+from .tracing import add_ground_truth, delete_ground_truth, query_ground_truth
+from .views import Trace

great_ai/__main__.py

@@ -0,0 +1,208 @@
+#!/usr/bin/env python3
+
+import logging
+import re
+from importlib import import_module, reload
+from pathlib import Path
+from threading import Event
+from typing import Optional
+
+import uvicorn
+from parse_arguments import parse_arguments
+from uvicorn._subprocess import get_subprocess
+from uvicorn.config import LOGGING_CONFIG, Config
+from uvicorn.supervisors.basereload import BaseReload
+from watchdog.events import FileSystemEvent, PatternMatchingEventHandler
+from watchdog.observers import Observer
+
+from great_ai.constants import SERVER_NAME
+from great_ai.context import _is_in_production_mode
+from great_ai.deploy import GreatAI
+from great_ai.exceptions import ArgumentValidationError, MissingArgumentError
+from great_ai.utilities import get_logger
+
+logger = get_logger(SERVER_NAME)
+
+
+GREAT_AI_LOGGING_CONFIG = {
+    **LOGGING_CONFIG,
+    "formatters": {
+        "default": {
+            "()": "great_ai.logger.CustomFormatter",
+            "fmt": "%(asctime)s | %(levelname)8s | %(message)s",
+        },
+        "access": {
+            "()": "great_ai.logger.CustomFormatter",
+            "fmt": "%(asctime)s | %(levelname)8s | %(message)s",  # noqa: E501
+        },
+    },
+}
+
+
+def main() -> None:
+    args = parse_arguments()
+    should_auto_reload = not _is_in_production_mode(logger=None)
+
+    if args.worker_count > 1 and should_auto_reload:
+        raise ArgumentValidationError(
+            "Cannot use auto-reload with multiple worker_count: set the `--worker_count=1` CLI argument,"
+            + "or set the ENVIRONMENT environment variable to `production`."
+        )
+
+    common_config = dict(
+        host=args.host,
+        port=args.port,
+        timeout_keep_alive=args.timeout_keep_alive,
+        workers=args.worker_count,
+        server_header=False,
+        reload=False,
+        log_config=GREAT_AI_LOGGING_CONFIG,
+    )
+
+    if not should_auto_reload:
+        file_name = get_script_name(args.file_name)
+        app = find_app(file_name)
+
+        logger.info(f"Starting uvicorn server with app={app}")
+
+        config = Config(app, **common_config)
+        socket = config.bind_socket()
+        server = GreatAIReload(
+            config, target=uvicorn.Server(config=config).run, sockets=[socket]
+        )
+
+        server.startup()
+        try:
+            Event().wait()
+        finally:
+            server.shutdown()
+            if args.file_name.endswith(".ipynb"):
+                Path(get_script_name_of_notebook(args.file_name)).unlink(
+                    missing_ok=True
+                )
+    else:
+
+        class EventHandler(PatternMatchingEventHandler):
+            def __init__(self) -> None:
+                super().__init__(
+                    patterns=["*.py", "*.ipynb"], ignore_patterns=["__*.py"]
+                )
+                self.server: Optional[GreatAIReload] = None
+                self.restart()
+
+            def on_closed(self, event: FileSystemEvent) -> None:
+                logger.warning(f"File {event.src_path} has triggered a restart")
+                self.restart()
+
+            def restart(self) -> None:
+                file_name = get_script_name(args.file_name)
+                app = find_app(file_name)
+                if app is None:
+                    logger.warning("Auto-reloading skipped")
+                    return
+
+                self.stop_server()
+
+                config = Config(app, **common_config)
+                socket = config.bind_socket()
+                self.server = GreatAIReload(
+                    config, target=uvicorn.Server(config=config).run, sockets=[socket]
+                )
+                self.server.startup()
+
+            def stop_server(self) -> None:
+                if self.server:
+                    self.server.shutdown()
+
+        restart_handler = EventHandler()
+        observer = Observer()
+        observer.schedule(restart_handler, path=".", recursive=True)
+        observer.start()
+
+        try:
+            Event().wait()
+        finally:
+            observer.stop()
+            restart_handler.stop_server()
+            if args.file_name.endswith(".ipynb"):
+                Path(get_script_name_of_notebook(args.file_name)).unlink(
+                    missing_ok=True
+                )
+            observer.join()
+
+
+def get_script_name(file_name_argument: str) -> str:
+    if file_name_argument.endswith(".ipynb"):
+        logger.info("Converting notebook to Python script")
+        from nbconvert import PythonExporter
+
+        exporter = PythonExporter()
+        content, _ = exporter.from_filename(file_name_argument)
+        file_name_argument = get_script_name_of_notebook(file_name_argument)
+
+        with open(file_name_argument, "w", encoding="utf-8") as f:
+            f.write(content)
+
+    return re.sub(r"\.(py|ipynb)$", "", file_name_argument)
+
+
+def get_script_name_of_notebook(notebook_name: str) -> str:
+    base_name = re.sub(r"\.ipynb$", "", notebook_name)
+    return f"__{base_name}__.py"
+
+
+module = None
+
+
+def find_app(file_name: str) -> Optional[str]:
+    global module
+
+    logging.disable(logging.CRITICAL)
+    try:
+        if module is None:
+            module = import_module(file_name)
+        else:
+            module = reload(module)
+    except Exception:
+        logging.disable(logging.NOTSET)
+        logger.exception("Could not load file because of an exception: fix your code")
+        return None
+    finally:
+        logging.disable(logging.NOTSET)
+
+    for name, value in module.__dict__.items():
+        if isinstance(value, GreatAI):
+            app_name = name
+
+    if app_name:
+        logger.info(f"Found `{app_name}` to be the GreatAI app ")
+    else:
+        raise MissingArgumentError(
+            "GreatAI app could not be found, make sure to use `@GreatAI.deploy` on your prediction function"
+        )
+
+    return f"{file_name}:{app_name}.app"
+
+
+class GreatAIReload(BaseReload):
+    def startup(self) -> None:
+        self.process = get_subprocess(
+            config=self.config, target=self.target, sockets=self.sockets
+        )
+        self.process.start()
+
+    def shutdown(self) -> None:
+        self.process.terminate()
+        self.process.join()
+
+        for sock in self.sockets:
+            sock.close()
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except KeyboardInterrupt:
+        exit()
+    except Exception as e:
+        logger.error(e)

great_ai/constants.py

@@ -0,0 +1,31 @@
+from .large_file import LargeFileMongo, LargeFileS3
+from .persistence.mongodb_driver import MongodbDriver
+
+ENV_VAR_KEY = "ENVIRONMENT"
+PRODUCTION_KEY = "production"
+DASHBOARD_PATH = "/dashboard"
+
+MONGO_CONFIG_PATHS = ["mongodb.ini", "mongo.ini", "mongo_db.ini", "mongo-db.ini"]
+DEFAULT_TRACING_DATABASE_CONFIG_PATHS = {
+    MongodbDriver: MONGO_CONFIG_PATHS,
+}
+
+DEFAULT_LARGE_FILE_CONFIG_PATHS = {
+    LargeFileS3: ["s3.ini", "b2.ini"],
+    LargeFileMongo: MONGO_CONFIG_PATHS,
+}
+
+GITHUB_LINK = "https://github.com/ScoutinScience/great_ai"
+
+TRAIN_SPLIT_TAG_NAME = "train"
+TEST_SPLIT_TAG_NAME = "test"
+VALIDATION_SPLIT_TAG_NAME = "validation"
+GROUND_TRUTH_TAG_NAME = "ground_truth"
+PRODUCTION_TAG_NAME = "production"
+DEVELOPMENT_TAG_NAME = "development"
+ONLINE_TAG_NAME = "online"
+
+SERVER_NAME = "GreatAI-Server"
+
+SE4ML_WEBSITE = "https://se-ml.github.io/practices/"
+LIST_ITEM_PREFIX = "  🔩 "

great_ai/context.py

@@ -0,0 +1,191 @@
+import os
+import random
+from logging import DEBUG, Logger
+from pathlib import Path
+from typing import Any, Dict, Optional, Type, cast
+
+from pydantic import BaseModel
+
+from .constants import (
+    DEFAULT_LARGE_FILE_CONFIG_PATHS,
+    DEFAULT_TRACING_DATABASE_CONFIG_PATHS,
+    ENV_VAR_KEY,
+    LIST_ITEM_PREFIX,
+    PRODUCTION_KEY,
+    SE4ML_WEBSITE,
+)
+from .large_file import LargeFileBase, LargeFileLocal
+from .persistence import ParallelTinyDbDriver, TracingDatabaseDriver
+from .utilities import get_logger
+
+
+class Context(BaseModel):
+    tracing_database: TracingDatabaseDriver
+    large_file_implementation: Type[LargeFileBase]
+    is_production: bool
+    logger: Logger
+    should_log_exception_stack: bool
+    prediction_cache_size: int
+    dashboard_table_size: int
+
+    class Config:
+        arbitrary_types_allowed = True
+
+    def to_flat_dict(self) -> Dict[str, Any]:
+        return {
+            "tracing_database": type(self.tracing_database).__name__,
+            "large_file_implementation": self.large_file_implementation.__name__,
+            "is_production": self.is_production,
+            "should_log_exception_stack": self.should_log_exception_stack,
+            "prediction_cache_size": self.prediction_cache_size,
+            "dashboard_table_size": self.dashboard_table_size,
+        }
+
+
+_context: Optional[Context] = None
+
+
+def get_context() -> Context:
+    if _context is None:
+        configure()
+
+    return cast(Context, _context)
+
+
+def configure(
+    *,
+    log_level: int = DEBUG,
+    seed: int = 42,
+    tracing_database: Optional[Type[TracingDatabaseDriver]] = None,
+    large_file_implementation: Optional[Type[LargeFileBase]] = None,
+    should_log_exception_stack: Optional[bool] = None,
+    prediction_cache_size: int = 512,
+    disable_se4ml_banner: bool = False,
+    dashboard_table_size: int = 50,
+) -> None:
+    global _context
+    logger = get_logger("great_ai", level=log_level)
+
+    if _context is not None:
+        logger.warn(
+            "Configuration has been already initialised, overwriting.\n"
+            + "Make sure to call `configure()` before importing your application code."
+        )
+
+    is_production = _is_in_production_mode(logger=logger)
+
+    _set_seed(seed)
+
+    tracing_database = _initialize_tracing_database(tracing_database, logger=logger)()
+
+    if not tracing_database.is_production_ready:
+        if is_production:
+            logger.error(
+                f"The selected tracing database ({type(tracing_database).__name__}) is not recommended for production"
+            )
+        else:
+            logger.warning(
+                f"The selected tracing database ({type(tracing_database).__name__}) is not recommended for production"
+            )
+
+    _context = Context(
+        tracing_database=tracing_database,
+        large_file_implementation=_initialize_large_file(
+            large_file_implementation, logger=logger
+        ),
+        is_production=is_production,
+        logger=logger,
+        should_log_exception_stack=not is_production
+        if should_log_exception_stack is None
+        else should_log_exception_stack,
+        prediction_cache_size=prediction_cache_size,
+        dashboard_table_size=dashboard_table_size,
+    )
+
+    logger.info("Settings: configured ✅")
+    for k, v in get_context().to_flat_dict().items():
+        logger.info(f"{LIST_ITEM_PREFIX}{k}: {v}")
+
+    if not is_production and not disable_se4ml_banner:
+        logger.warning(
+            "You still need to check whether you follow all best practices before trusting your deployment."
+        )
+        logger.warning(f"> Find out more at {SE4ML_WEBSITE}")
+
+
+def _is_in_production_mode(logger: Optional[Logger]) -> bool:
+    environment = os.environ.get(ENV_VAR_KEY)
+
+    if environment is None:
+        if logger:
+            logger.warning(
+                f"Environment variable {ENV_VAR_KEY} is not set, defaulting to development mode ‼️"
+            )
+        is_production = False
+    else:
+        is_production = environment.lower() == PRODUCTION_KEY
+        if logger:
+            if not is_production:
+                logger.info(
+                    f"Value of {ENV_VAR_KEY} is `{environment}` which is not equal to `{PRODUCTION_KEY}`"
+                    + " defaulting to development mode ‼️"
+                )
+            else:
+                logger.info("Running in production mode ✅")
+
+    return is_production
+
+
+def _initialize_tracing_database(
+    selected: Optional[Type[TracingDatabaseDriver]], logger: Logger
+) -> Type[TracingDatabaseDriver]:
+    for tracing_driver, paths in DEFAULT_TRACING_DATABASE_CONFIG_PATHS.items():
+        if selected is None or selected == tracing_driver:
+            if tracing_driver.initialized:
+                logger.warning(
+                    f"{tracing_driver.__name__} has been already configured: skipping initialisation"
+                )
+                return tracing_driver
+            for p in paths:
+                if Path(p).exists():
+                    logger.info(
+                        f"Found credentials file ({Path(p).absolute()}), initialising {tracing_driver.__name__}"
+                    )
+                    tracing_driver.configure_credentials_from_file(p)
+                    return tracing_driver
+    logger.warning(
+        "Cannot find credentials files, defaulting to using ParallelTinyDbDriver"
+    )
+    return ParallelTinyDbDriver
+
+
+def _initialize_large_file(
+    selected: Optional[Type[LargeFileBase]], logger: Logger
+) -> Type[LargeFileBase]:
+    for large_file, paths in DEFAULT_LARGE_FILE_CONFIG_PATHS.items():
+        if selected is None or selected == large_file:
+            if large_file.initialized:
+                logger.warning(
+                    f"{large_file.__name__} has been already configured: skipping initialisation"
+                )
+                return large_file
+            for p in paths:
+                if Path(p).exists():
+                    logger.info(
+                        f"Found credentials file ({Path(p).absolute()}), initialising {large_file.__name__}"
+                    )
+                    large_file.configure_credentials_from_file(p)
+                    return large_file
+    logger.warning("Cannot find credentials files, defaulting to using LargeFileLocal")
+    return LargeFileLocal
+
+
+def _set_seed(seed: int) -> None:
+    random.seed(seed)
+
+    try:
+        import numpy
+
+        numpy.random.seed(seed + 1)
+    except ImportError:
+        pass

great_ai/deploy/__init__.py

@@ -0,0 +1 @@
+from .great_ai import GreatAI

great_ai/deploy/great_ai.py

@@ -0,0 +1,266 @@
+import inspect
+from functools import lru_cache, partial, wraps
+from typing import (
+    Any,
+    Callable,
+    Generic,
+    Iterable,
+    List,
+    Optional,
+    Type,
+    TypeVar,
+    cast,
+    overload,
+)
+
+from fastapi import APIRouter, FastAPI, status
+from pydantic import BaseModel, create_model
+
+from ..constants import DASHBOARD_PATH
+from ..context import get_context
+from ..helper import (
+    freeze_arguments,
+    get_function_metadata_store,
+    snake_case_to_text,
+    use_http_exceptions,
+)
+from ..models import model_versions
+from ..parameters import automatically_decorate_parameters
+from ..tracing.tracing_context import TracingContext
+from ..utilities import parallel_map
+from ..views import ApiMetadata, CacheStatistics, HealthCheckResponse, Trace
+from .routes import (
+    bootstrap_docs_endpoints,
+    bootstrap_feedback_endpoints,
+    bootstrap_trace_endpoints,
+)
+from .routes.bootstrap_dashboard import bootstrap_dashboard
+
+T = TypeVar("T")
+
+
+class GreatAI(Generic[T]):
+    def __init__(self, func: Callable[..., Any], version: str, return_raw_result: bool):
+        is_asynchronous = inspect.iscoroutinefunction(func)
+        func = automatically_decorate_parameters(func)
+        get_function_metadata_store(func).is_finalised = True
+
+        self._func = func
+
+        def func_in_tracing_context_sync(
+            *args: Any, do_not_persist_traces: bool = False, **kwargs: Any
+        ) -> Trace[T]:
+            with TracingContext[T](
+                func.__name__, do_not_persist_traces=do_not_persist_traces
+            ) as t:
+                result = func(*args, **kwargs)
+                output = t.finalise(output=result)
+            return result if return_raw_result else output
+
+        async def func_in_tracing_context_async(
+            *args: Any, do_not_persist_traces: bool = False, **kwargs: Any
+        ) -> Trace[T]:
+            with TracingContext[T](
+                func.__name__, do_not_persist_traces=do_not_persist_traces
+            ) as t:
+                result = await func(*args, **kwargs)
+                output = t.finalise(output=result)
+            return result if return_raw_result else output
+
+        func_in_tracing_context = (
+            func_in_tracing_context_async
+            if is_asynchronous
+            else func_in_tracing_context_sync
+        )
+
+        self._cached_func = lru_cache(get_context().prediction_cache_size)(
+            func_in_tracing_context
+        )  # cannot put decorator on method, because it require the context to be setup
+
+        wraps(func)(self)
+
+        self._version = version
+
+        self.app = FastAPI(
+            title=self.name,
+            version=self.version,
+            description=self.documentation
+            + f"\n\nFind out more in the [dashboard]({DASHBOARD_PATH}).",
+            docs_url=None,
+            redoc_url=None,
+        )
+
+    @overload
+    @staticmethod
+    def create(
+        func: Optional[Callable[..., T]] = None,
+    ) -> "GreatAI[T]":
+        ...
+
+    @overload
+    @staticmethod
+    def create(
+        version: str,
+        return_raw_result: bool,
+        disable_rest_api: bool,
+        disable_docs: bool,
+        disable_dashboard: bool,
+    ) -> Callable[[Callable[..., T]], "GreatAI[T]"]:
+        ...
+
+    @staticmethod
+    def create(
+        func: Optional[Callable[..., T]] = None,
+        *,
+        version: str = "0.0.1",
+        return_raw_result: bool = False,
+        disable_rest_api: bool = False,
+        disable_docs: bool = False,
+        disable_dashboard: bool = False,
+    ):
+        if func is None:
+            return cast(
+                Callable[[Callable[..., T]], GreatAI[T]],
+                partial(
+                    GreatAI.create,
+                    version=version,
+                    return_raw_result=return_raw_result,
+                    disable_rest_api=disable_rest_api,
+                    disable_docs=disable_docs,
+                    disable_dashboard=disable_dashboard,
+                ),
+            )
+
+        instance = GreatAI[T](
+            func, version=version, return_raw_result=return_raw_result
+        )
+
+        if not disable_rest_api:
+            instance._bootstrap_rest_api(
+                disable_docs=disable_docs, disable_dashboard=disable_dashboard
+            )
+
+        return instance
+
+    @freeze_arguments
+    def __call__(self, *args: Any, **kwargs: Any) -> Trace[T]:
+        return self._cached_func(*args, **kwargs)
+
+    def process_batch(
+        self,
+        batch: Iterable[Any],
+        concurrency: Optional[int] = None,
+        do_not_persist_traces: bool = False,
+    ) -> List[Trace[T]]:
+        return list(
+            parallel_map(
+                freeze_arguments(
+                    partial(
+                        self._cached_func, do_not_persist_traces=do_not_persist_traces
+                    )
+                ),
+                batch,
+                concurrency=concurrency,
+            )
+        )
+
+    @property
+    def name(self) -> str:
+        return snake_case_to_text(self._func.__name__)
+
+    @property
+    def version(self) -> str:
+        flat_model_versions = ".".join(f"{k}-v{v}" for k, v in model_versions)
+        if flat_model_versions:
+            flat_model_versions = f"+{flat_model_versions}"
+
+        return f"{self._version}{flat_model_versions}"
+
+    @property
+    def documentation(self) -> str:
+        return (
+            f"GreatAI wrapper for interacting with the `{self._func.__name__}` function.\n\n"
+            + (
+                "\n".join(
+                    line.strip()
+                    for line in (self._func.__doc__ or "").split("\n")
+                    if line.strip()
+                )
+            )
+        )
+
+    def _bootstrap_rest_api(self, disable_docs: bool, disable_dashboard: bool) -> None:
+        self._bootstrap_prediction_endpoint()
+
+        if not disable_docs:
+            bootstrap_docs_endpoints(self.app)
+
+        if not disable_dashboard:
+            bootstrap_dashboard(
+                self.app,
+                function_name=self._func.__name__,
+                documentation=self.documentation,
+            )
+            bootstrap_trace_endpoints(self.app)
+
+        bootstrap_feedback_endpoints(self.app)
+        self._bootstrap_meta_endpoints()
+
+    def _bootstrap_prediction_endpoint(self) -> None:
+        router = APIRouter(
+            tags=["predictions"],
+        )
+
+        schema = self._get_schema()
+
+        @router.post(
+            "/predict", status_code=status.HTTP_200_OK, response_model=Trace[T]
+        )
+        @use_http_exceptions
+        def predict(input_value: schema) -> Trace[T]:  # type: ignore
+            return self(**cast(BaseModel, input_value).dict())
+
+        self.app.include_router(router)
+
+    def _get_schema(self) -> Type[BaseModel]:
+        signature = inspect.signature(self._func)
+        parameters = {
+            p.name: (
+                p.annotation if p.annotation != inspect._empty else Any,
+                p.default if p.default != inspect._empty else ...,
+            )
+            for p in signature.parameters.values()
+            if p.name in get_function_metadata_store(self._func).input_parameter_names
+        }
+
+        schema: Type[BaseModel] = create_model("InputModel", **parameters)  # type: ignore
+        return schema
+
+    def _bootstrap_meta_endpoints(self) -> None:
+        router = APIRouter(
+            tags=["meta"],
+        )
+
+        @router.get("/health", status_code=status.HTTP_200_OK)
+        def check_health() -> HealthCheckResponse:
+            hits, misses, maxsize, cache_size = self._cached_func.cache_info()
+            cache_statistics = CacheStatistics(
+                hits=hits, misses=misses, size=cache_size, max_size=maxsize
+            )
+
+            return HealthCheckResponse(
+                is_healthy=True, cache_statistics=cache_statistics
+            )
+
+        @router.get(
+            "/version", response_model=ApiMetadata, status_code=status.HTTP_200_OK
+        )
+        def get_version() -> ApiMetadata:
+            return ApiMetadata(
+                name=self.name,
+                version=self.version,
+                documentation=self.documentation,
+                configuration=get_context().to_flat_dict(),
+            )
+
+        self.app.include_router(router)

great_ai/deploy/routes/__init__.py

@@ -0,0 +1,4 @@
+from .bootstrap_dashboard import bootstrap_dashboard
+from .bootstrap_docs_endpoints import bootstrap_docs_endpoints
+from .bootstrap_feedback_endpoints import bootstrap_feedback_endpoints
+from .bootstrap_trace_endpoints import bootstrap_trace_endpoints

great_ai/deploy/routes/bootstrap_dashboard.py

@@ -0,0 +1,27 @@
+from pathlib import Path
+
+from fastapi import FastAPI
+from fastapi.middleware.wsgi import WSGIMiddleware
+from fastapi.responses import RedirectResponse
+from fastapi.staticfiles import StaticFiles
+
+from ...constants import DASHBOARD_PATH
+from .dashboard import create_dash_app
+
+PATH = Path(__file__).parent.resolve()
+
+
+def bootstrap_dashboard(app: FastAPI, function_name: str, documentation: str) -> None:
+    dash_app = create_dash_app(function_name, app.version, documentation)
+
+    app.mount(DASHBOARD_PATH, WSGIMiddleware(dash_app))
+
+    @app.get("/", include_in_schema=False)
+    def redirect_to_entrypoint() -> RedirectResponse:
+        return RedirectResponse(DASHBOARD_PATH)
+
+    app.mount(
+        "/assets",
+        StaticFiles(directory=PATH / "dashboard/assets"),
+        name="static",
+    )

great_ai/deploy/routes/bootstrap_docs_endpoints.py

@@ -0,0 +1,14 @@
+from fastapi import FastAPI
+from fastapi.openapi.docs import get_swagger_ui_html
+from fastapi.responses import RedirectResponse
+from starlette.responses import HTMLResponse
+
+
+def bootstrap_docs_endpoints(app: FastAPI) -> None:
+    @app.get("/docs", include_in_schema=False)
+    def custom_swagger_ui_html() -> HTMLResponse:
+        return get_swagger_ui_html(openapi_url="openapi.json", title=app.title)
+
+    @app.get("/docs/index.html", include_in_schema=False)
+    def redirect_to_docs() -> RedirectResponse:
+        return RedirectResponse("/docs")

great_ai/deploy/routes/bootstrap_feedback_endpoints.py

@@ -0,0 +1,44 @@
+from typing import Any
+
+from fastapi import APIRouter, FastAPI, HTTPException, Response, status
+
+from ...context import get_context
+from ...views import EvaluationFeedbackRequest
+
+
+def bootstrap_feedback_endpoints(app: FastAPI) -> None:
+    router = APIRouter(
+        prefix="/traces/{trace_id}/feedback",
+        tags=["feedback"],
+    )
+
+    @router.put("/", status_code=status.HTTP_202_ACCEPTED)
+    def set_feedback(trace_id: str, input: EvaluationFeedbackRequest) -> Response:
+        trace = get_context().tracing_database.get(trace_id)
+        if trace is None:
+            raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
+
+        trace.feedback = input.feedback
+
+        get_context().tracing_database.update(trace_id, trace)
+        return Response(status_code=status.HTTP_202_ACCEPTED)
+
+    @router.get("/", status_code=status.HTTP_200_OK)
+    def get_feedback(trace_id: str) -> Any:
+        trace = get_context().tracing_database.get(trace_id)
+        if trace is None:
+            raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
+        return trace.feedback
+
+    @router.delete("/", status_code=status.HTTP_204_NO_CONTENT)
+    def delete_feedback(trace_id: str) -> Any:
+        trace = get_context().tracing_database.get(trace_id)
+        if trace is None:
+            raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
+
+        trace.feedback = None
+
+        get_context().tracing_database.update(trace_id, trace)
+        return Response(status_code=status.HTTP_204_NO_CONTENT)
+
+    app.include_router(router)

great_ai/deploy/routes/bootstrap_trace_endpoints.py

@@ -0,0 +1,44 @@
+from typing import List
+
+from fastapi import APIRouter, FastAPI, HTTPException, Response, status
+
+from ...context import get_context
+from ...views import Query, Trace
+
+
+def bootstrap_trace_endpoints(app: FastAPI) -> None:
+    router = APIRouter(
+        prefix="/traces",
+        tags=["traces"],
+    )
+
+    @router.post("", status_code=status.HTTP_200_OK, response_model=List[Trace])
+    def query_traces(
+        query: Query,
+        skip: int = 0,
+        take: int = 100,
+    ) -> List[Trace]:
+        return get_context().tracing_database.query(
+            conjunctive_filters=query.filter,
+            conjunctive_tags=query.conjunctive_tags,
+            since=query.since,
+            until=query.until,
+            has_feedback=query.has_feedback,
+            sort_by=query.sort,
+            skip=skip,
+            take=take,
+        )[0]
+
+    @router.get("/{trace_id}", status_code=status.HTTP_200_OK, response_model=Trace)
+    def get_trace(trace_id: str) -> Trace:
+        result = get_context().tracing_database.get(trace_id)
+        if result is None:
+            raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
+        return result
+
+    @router.delete("/{trace_id}", status_code=status.HTTP_204_NO_CONTENT)
+    def delete_trace(trace_id: str) -> Response:
+        get_context().tracing_database.delete(trace_id)
+        return Response(status_code=status.HTTP_204_NO_CONTENT)
+
+    app.include_router(router)

great_ai/deploy/routes/dashboard/__init__.py

@@ -0,0 +1 @@
+from .create_dash_app import create_dash_app

great_ai/deploy/routes/dashboard/assets/index.css

@@ -0,0 +1,232 @@
+:root {
+  --background-color: #edf5f6;
+  --small-padding: 10px;
+  --medium-padding: 20px;
+  --large-padding: 40px;
+  --border-radius: 10px;
+  --shadow: 0 4px 6px -1px rgb(0 0 0 / 10%), 0 2px 4px -1px rgb(0 0 0 / 6%);
+  --disclaimer-width: 180px;
+  --disclaimer-height: 35px;
+}
+
+@media (max-width: 900px) {
+  body {
+    zoom: 0.8;
+  }
+}
+
+@media (max-width: 550px) {
+  :root {
+    --small-padding: 5px;
+    --medium-padding: 10px;
+    --large-padding: 20px;
+    --border-radius: 8px;
+  }
+
+  .environment {
+    margin-top: calc(-1 * var(--large-padding));
+    margin-bottom: var(--large-padding);
+  }
+}
+
+@media (min-width: 551px) {
+  .environment {
+    position: absolute;
+    width: var(--disclaimer-width);
+    height: var(--disclaimer-height);
+    transform: rotate(-45deg);
+    top: calc(
+      var(--disclaimer-width) / 1.4142 - var(--disclaimer-height) / 1.4142
+    );
+    left: calc(-1 * var(--disclaimer-height) / 1.4142);
+    transform-origin: top left;
+    z-index: 100;
+  }
+}
+
+* {
+  margin: 0;
+  box-sizing: border-box;
+  word-break: break-word;
+}
+
+body {
+  background-color: var(--background-color);
+  font-family: Arial, Helvetica, sans-serif;
+}
+
+h1,
+h2,
+h3,
+h4,
+h5,
+h6 {
+  margin: var(--medium-padding) 0 var(--small-padding) 0;
+}
+
+h6 {
+  margin-top: 0;
+  font-size: 3rem;
+}
+
+html,
+body,
+#react-entry-point,
+main {
+  height: 100%;
+}
+
+main {
+  padding-top: var(--large-padding);
+  display: flex;
+  flex-direction: column;
+}
+
+.environment {
+  color: white;
+  text-align: center;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+}
+
+main > header,
+.configuration-container,
+.traces-table-container,
+.parallel-coordinates,
+main > footer {
+  padding: var(--large-padding);
+  flex-shrink: 0;
+  overflow: hidden;
+}
+
+main > header,
+.configuration-container,
+.traces-table-container,
+.parallel-coordinates {
+  margin: 0 var(--large-padding) var(--large-padding) var(--large-padding);
+  border-radius: var(--border-radius);
+  box-shadow: var(--shadow);
+  background-color: white;
+}
+
+main > header {
+  display: flex;
+  align-items: center;
+  flex-wrap: wrap;
+  justify-content: space-between;
+}
+
+main > header > div:nth-child(1) {
+  min-width: 350px;
+  max-width: 450px;
+  flex: 1;
+}
+
+main > header > div > h1 {
+  margin-top: 0;
+}
+
+.version-tag {
+  border-radius: var(--border-radius);
+  background: #ddd;
+  display: inline-block;
+  font-size: 1rem;
+  padding: 3px 6px;
+  margin-left: var(--small-padding)
+}
+
+main > header > *:nth-child(2) {
+  min-width: 250px;
+  max-width: 550px;
+  flex: 1;
+}
+
+main > header .placeholder {
+  opacity: 0.35;
+  font-size: 1.5rem;
+  text-align: center;
+  display: block;
+  min-width: 250px;
+  width: 60%;
+  margin: auto;
+}
+
+.configuration-container {
+  display: flex;
+  justify-content: space-between;
+  flex-wrap: wrap;
+}
+
+.configuration-item {
+  padding-left: var(--small-padding);
+  margin: var(--medium-padding);
+}
+
+.configuration-item h4 {
+  font-weight: bold;
+  margin: 0 0 var(--small-padding) 0;
+}
+
+.traces-table-container {
+  padding: 0;
+}
+
+.traces-table-container header {
+  padding: var(--large-padding);
+}
+
+.traces-table-container header h2 {
+  margin-top: 0;
+}
+
+.dash-filter--case {
+  display: none;
+}
+
+.traces-table-container td > div {
+  white-space: pre !important;
+  max-height: 150px !important;
+  overflow: auto !important;
+  display: inline-block !important;
+  text-align: left !important;
+}
+
+.traces-table-container th > div {
+  text-align: left !important;
+}
+
+.space-filler {
+  flex-grow: 1;
+}
+
+main > footer {
+  opacity: 0.35;
+  margin: 0;
+}
+
+main > footer {
+  display: flex;
+  justify-content: space-between;
+  align-items: center;
+  padding: var(--large-padding);
+  background-color: #ddd;
+  position: relative;
+}
+
+.parallel-coordinates {
+  padding: 0;
+}
+
+a img {
+  display: block;
+  margin-left: var(--large-padding);
+  width: 64px;
+  height: 64px;
+  cursor: pointer;
+  transition: transform 300ms;
+}
+
+a img:hover {
+  transform: scale(1.1);
+}

great_ai/deploy/routes/dashboard/create_dash_app.py

@@ -0,0 +1,259 @@
+from math import ceil
+from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
+
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
+from dash import Dash, dcc, html
+from dash.dependencies import Input, Output
+from flask import Flask
+
+from ....constants import DASHBOARD_PATH, ONLINE_TAG_NAME
+from ....context import get_context
+from ....helper import freeze, snake_case_to_text, text_to_hex_color
+from ....utilities import unique
+from ....views import SortBy, Trace
+from .get_description import get_description
+from .get_filter_from_datatable import get_filter_from_datatable
+from .get_footer import get_footer
+from .get_traces_table import get_traces_table
+
+
+def create_dash_app(function_name: str, version: str, function_docs: str) -> Flask:
+    accent_color = text_to_hex_color(function_name)
+
+    app = Dash(
+        function_name,
+        requests_pathname_prefix=DASHBOARD_PATH + "/",
+        server=Flask(__name__),
+        title=snake_case_to_text(function_name),
+        update_title=None,
+        external_stylesheets=[
+            "/assets/index.css",
+        ],
+    )
+
+    app.layout = html.Main(
+        [
+            html.Div(
+                html.P("PRODUCTION" if get_context().is_production else "DEVELOPMENT"),
+                className="environment",
+                style={"background": accent_color},
+            ),
+            html.Header(
+                [
+                    get_description(
+                        function_name=function_name,
+                        version=version,
+                        function_docs=function_docs,
+                        accent_color=accent_color,
+                    ),
+                    execution_time_histogram_container := html.Div(),
+                ],
+            ),
+            configuration_container := html.Div(
+                className="configuration-container",
+            ),
+            traces_table_container := html.Div(
+                [
+                    html.Header(
+                        [
+                            html.H2("Latest traces"),
+                            html.P(
+                                "Recent traces and aggregated metrics are presented below. Try filtering the table."
+                            ),
+                            html.A(
+                                "Filtering syntax.",
+                                href="https://dash.plotly.com/datatable/filtering",
+                                target="_blank",
+                            ),
+                        ]
+                    ),
+                    table := get_traces_table(),
+                ],
+                className="traces-table-container",
+            ),
+            parallel_coordinates := dcc.Graph(
+                className="parallel-coordinates", config={"displaylogo": False}
+            ),
+            html.Div(className="space-filler"),
+            get_footer(),
+            interval := dcc.Interval(
+                interval=4 * 1000,  # in milliseconds
+            ),
+        ]
+    )
+
+    @app.callback(
+        Output(configuration_container, "children"),
+        Input(interval, "n_intervals"),
+    )
+    def update_configuration(
+        n_intervals: int,
+    ) -> List[html.Div]:
+        config = get_context().to_flat_dict()
+        return [
+            html.Div(
+                [
+                    html.H4(snake_case_to_text(key)),
+                    html.P(str(value)),
+                ],
+                className="configuration-item",
+                style={"border-left": f"2px solid {accent_color}"},
+            )
+            for key, value in config.items()
+        ]
+
+    @app.callback(
+        Output(table, "data"),
+        Output(table, "page_count"),
+        Output(table, "columns"),
+        Output(traces_table_container, "style"),
+        Output(execution_time_histogram_container, "children"),
+        Output(parallel_coordinates, "figure"),
+        Output(parallel_coordinates, "style"),
+        Input(table, "page_current"),
+        Input(table, "page_size"),
+        Input(table, "sort_by"),
+        Input(table, "filter_query"),
+        Input(interval, "n_intervals"),
+    )
+    def update_page(
+        page_current: int,
+        page_size: int,
+        sort_by: List[Dict[str, Union[str, int]]],
+        filter_query: str,
+        n_intervals: int,
+    ) -> Tuple[
+        List[Dict[str, Any]],
+        int,
+        List[Dict[str, Sequence[str]]],
+        Dict[str, Any],
+        Any,
+        go.Figure,
+        Dict[str, Any],
+    ]:
+        conjunctive_filters = (
+            [get_filter_from_datatable(f) for f in filter_query.split(" && ")]
+            if filter_query
+            else []
+        )
+        non_null_conjunctive_filters = [f for f in conjunctive_filters if f is not None]
+
+        elements, count = get_context().tracing_database.query(
+            skip=page_current * page_size,
+            take=page_size,
+            conjunctive_filters=non_null_conjunctive_filters,
+            conjunctive_tags=[ONLINE_TAG_NAME],
+            sort_by=[SortBy.parse_obj(s) for s in sort_by],
+        )
+
+        columns, style = update_layout(elements[0] if elements else None)
+        execution_time_histogram, parallel_coords_fig, style = update_charts(
+            elements=elements, function_name=function_name, accent_color=accent_color
+        )
+
+        return (
+            [
+                {k: str(v) for k, v in e.to_flat_dict(include_original=False).items()}
+                for e in elements
+            ],
+            max(1, ceil(count / page_size)),
+            columns,
+            style,
+            execution_time_histogram,
+            parallel_coords_fig,
+            style,
+        )
+
+    return app.server
+
+
+def update_layout(
+    first_element: Optional[Trace],
+) -> Tuple[List[Dict[str, Sequence[str]]], Dict[str, Any]]:
+
+    if first_element:
+        keys = list(first_element.to_flat_dict(include_original=False).keys())
+        header_height = max(len(i.split(":")) for i in keys)
+        columns = [
+            {
+                "name": [""] * (header_height - len(k.split(":")))
+                + k.replace("_flat", "").split(":"),
+                "id": k,
+            }
+            for k in keys
+        ]
+    else:
+        columns = []
+
+    return (
+        columns,
+        {"display": "none" if first_element is None else "block"},
+    )
+
+
+def update_charts(
+    elements: List[Trace], function_name: str, accent_color: str
+) -> Tuple[Any, go.Figure, Dict[str, Any]]:
+    if not elements:
+        return (
+            html.Span(
+                f"No traces yet: call your function ({function_name}) to create one.",
+                className="placeholder",
+            ),
+            go.Figure(),
+            {"display": "none"},
+        )
+
+    flat_elements = [e.to_flat_dict(include_original=False) for e in elements]
+
+    execution_time_histogram = dcc.Graph(config={"displaylogo": False})
+    df = pd.DataFrame(flat_elements)
+    fig = px.histogram(
+        df,
+        x="original_execution_time_ms",
+        labels={"original_execution_time_ms": "Execution time (ms)"},
+        nbins=20,
+        height=400,
+        log_y=True,
+        color_discrete_sequence=[accent_color],
+    )
+    fig.update_layout(
+        margin=dict(l=0, r=0, b=0, t=0, pad=0),
+    )
+    execution_time_histogram.figure = fig
+
+    parallel_coords_fig = go.Figure(
+        go.Parcoords(
+            dimensions=[
+                get_dimension_descriptor(df, c)
+                for c in df.columns
+                if c not in {"trace_id", "created", "output", "exception", "feedback"}
+                and "_flat" not in c
+            ],
+            line_color=accent_color,
+        )
+    )
+    return execution_time_histogram, parallel_coords_fig, {}
+
+
+def get_dimension_descriptor(df: pd.DataFrame, column: str) -> Dict[str, Any]:
+    dimension: Dict[str, Any] = {
+        "label": snake_case_to_text(column),
+    }
+
+    values = df[column]
+
+    try:
+        dimension["values"] = [float(v) for v in values]
+    except (TypeError, ValueError):
+        MAX_LENGTH = 40
+        unique_values = unique(values, key=freeze)
+        value_mapping = {str(v)[:MAX_LENGTH]: i for i, v in enumerate(unique_values)}
+
+        dimension["values"] = [value_mapping[str(v)[:MAX_LENGTH]] for v in values]
+        dimension["tickvals"] = list(value_mapping.values())
+        dimension["ticktext"] = [k[:MAX_LENGTH] for k in value_mapping.keys()]
+
+    return dimension

great_ai/deploy/routes/dashboard/get_description.py

@@ -0,0 +1,35 @@
+from dash import dcc, html
+
+from ....helper import snake_case_to_text, strip_lines
+
+
+def get_description(
+    function_name: str, version: str, function_docs: str, accent_color: str
+) -> html.Div:
+    return html.Div(
+        [
+            html.H1(
+                [
+                    f"{snake_case_to_text(function_name)} - dashboard",
+                    html.Span(version, className="version-tag"),
+                ],
+                style={"color": accent_color},
+            ),
+            dcc.Markdown(
+                strip_lines(
+                    f"""
+                    > View the live data of your deployment here.
+
+                    ## Using the API
+
+                    You can find the available endpoints at [/docs](/docs).
+
+                    ## Details
+
+                    {function_docs}
+                """
+                ),
+                className="description",
+            ),
+        ]
+    )

great_ai/deploy/routes/dashboard/get_filter_from_datatable.py

@@ -0,0 +1,23 @@
+from typing import Optional, Union
+
+from ....views import Filter, operators
+
+
+def get_filter_from_datatable(description: str) -> Optional[Filter]:
+    for operator in operators:
+        if operator in description:
+            name_part, value_part = description.split(operator, 1)
+            value_part = value_part.strip()
+            name_part = name_part[name_part.find("{") + 1 : name_part.rfind("}")]
+
+            v0 = value_part[0]
+            if v0 == value_part[-1] and v0 in ("'", '"', "`"):
+                value: Union[str, float] = value_part[1:-1].replace("\\" + v0, v0)
+            else:
+                try:
+                    value = float(value_part)
+                except ValueError:
+                    value = value_part
+            return Filter(property=name_part, operator=operator, value=value)
+
+    return None

great_ai/deploy/routes/dashboard/get_footer.py

@@ -0,0 +1,23 @@
+from dash import html
+
+from ....constants import GITHUB_LINK
+
+
+def get_footer() -> html.Footer:
+    return html.Footer(
+        [
+            html.Div(
+                [
+                    html.H6("GreatAI"),
+                    html.P(
+                        "A human-friendly framework for robust end-to-end AI deployments."
+                    ),
+                ]
+            ),
+            html.A(
+                html.Img(src="/assets/github.png"),
+                href=GITHUB_LINK,
+                target="_blank",
+            ),
+        ],
+    )

great_ai/deploy/routes/dashboard/get_traces_table.py

@@ -0,0 +1,34 @@
+from dash import dash_table
+
+from ....context import get_context
+
+
+def get_traces_table() -> dash_table.DataTable:
+    return dash_table.DataTable(
+        page_current=0,
+        page_size=get_context().dashboard_table_size,
+        page_action="custom",
+        filter_action="custom",
+        sort_action="custom",
+        sort_mode="multi",
+        sort_by=[
+            {"column_id": "created", "direction": "desc"},
+        ],
+        style_data={
+            "white-space": "normal",
+            "height": "auto",
+            "max-height": "300px",
+            "overflow": "hidden",
+            "text-overflow": "ellipsis",
+        },
+        style_cell={"padding": "5px"},
+        style_header={
+            "background-color": "white",
+            "font-weight": "bold",
+        },
+        merge_duplicate_headers=True,
+        style_cell_conditional=[
+            {"if": {"column_id": "output"}, "width": 1500},
+        ],
+        style_table={"overflow": "auto"},
+    )

great_ai/exceptions/__init__.py

@@ -0,0 +1,3 @@
+from .argument_validation_error import ArgumentValidationError
+from .missing_argument_error import MissingArgumentError
+from .wrong_decorator_order_error import WrongDecoratorOrderError

great_ai/exceptions/argument_validation_error.py

@@ -0,0 +1,2 @@
+class ArgumentValidationError(Exception):
+    pass

great_ai/exceptions/missing_argument_error.py

@@ -0,0 +1,2 @@
+class MissingArgumentError(Exception):
+    pass

great_ai/exceptions/wrong_decorator_order_error.py

@@ -0,0 +1,2 @@
+class WrongDecoratorOrderError(Exception):
+    pass

great_ai/helper/__init__.py

@@ -0,0 +1,8 @@
+from .freeze_arguments import freeze, freeze_arguments
+from .get_arguments import get_arguments
+from .get_function_metadata_store import get_function_metadata_store
+from .hashable_base_model import HashableBaseModel
+from .snake_case_to_text import snake_case_to_text
+from .strip_lines import strip_lines
+from .text_to_hex_color import text_to_hex_color
+from .use_http_exceptions import use_http_exceptions

great_ai/helper/assert_function_is_not_finalised.py

@@ -0,0 +1,14 @@
+from typing import Any, Callable
+
+from ..exceptions import WrongDecoratorOrderError
+from .get_function_metadata_store import get_function_metadata_store
+
+
+def assert_function_is_not_finalised(func: Callable[..., Any]) -> None:
+    error_message = (
+        "The outer-most (first) decorator has to be `@GreatAI.deploy`. "
+        + f"In the case of `{func.__name__}`, it is not: fix this by moving `@GreatAI.deploy` to the top."
+    )
+
+    if get_function_metadata_store(func).is_finalised:
+        raise WrongDecoratorOrderError(error_message)

great_ai/helper/freeze_arguments.py

@@ -0,0 +1,56 @@
+from functools import wraps
+from typing import Any, Callable, Dict, List, Set, Union
+
+from pydantic import BaseModel
+
+
+class FrozenDict(dict):
+    def __hash__(self) -> int:
+        return hash(frozenset((k, freeze(v)) for k, v in self.items()))
+
+
+class FrozenList(list):
+    def __hash__(self) -> int:
+        return hash(tuple(freeze(i) for i in self))
+
+
+class FrozenSet(set):
+    def __hash__(self) -> int:
+        return hash(frozenset(freeze(i) for i in self))
+
+
+def freeze_arguments(func: Callable[..., Any]) -> Callable[..., Any]:
+    """Transform mutable dictionary
+    Into immutable
+    Useful to be compatible with cache
+    source: https://stackoverflow.com/questions/6358481/using-functools-lru-cache-with-dictionary-arguments
+    """
+
+    @wraps(func)
+    def wrapper(*args: List[Any], **kwargs: Dict[str, Any]) -> Any:
+        args = tuple(freeze(arg) for arg in args)
+        kwargs = {k: freeze(v) for k, v in kwargs.items()}
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+def freeze(value: Union[List[Any], Dict[str, Any], Set[Any]]) -> Any:
+    if isinstance(value, dict):
+        return FrozenDict(value)
+
+    if isinstance(value, list):
+        return FrozenList(value)
+
+    if isinstance(value, set):
+        return FrozenSet(value)
+
+    if isinstance(value, BaseModel):
+
+        class HashableValue(type(value)):
+            def __hash__(self) -> int:
+                return hash(frozenset((k, freeze(v)) for k, v in self.dict().items()))
+
+        return HashableValue(**value.dict())
+
+    return value

great_ai/helper/get_arguments.py

@@ -0,0 +1,24 @@
+import inspect
+from typing import Any, Callable, Dict, Mapping, Sequence
+
+
+def get_arguments(
+    func: Callable[..., Any], args: Sequence[Any], kwargs: Mapping[str, Any]
+) -> Dict[str, Any]:
+    """Return mapping from parameter names to actual argument values"""
+
+    signature = inspect.signature(func)
+
+    defaults = {
+        p.name: p.default
+        for p in signature.parameters.values()
+        if p.default != inspect._empty
+    }
+
+    filter_keys = [
+        param.name
+        for param in signature.parameters.values()
+        if param.kind == param.POSITIONAL_OR_KEYWORD
+    ]
+
+    return {**defaults, **dict(zip(filter_keys, args)), **kwargs}

great_ai/helper/get_function_metadata_store.py

@@ -0,0 +1,12 @@
+from typing import Any, Callable, cast
+
+from ..views.function_metadata import FunctionMetadata
+
+
+def get_function_metadata_store(func: Callable[..., Any]) -> FunctionMetadata:
+    any_func = cast(Any, func)
+
+    if not hasattr(any_func, "_great_ai_metadata"):
+        any_func._great_ai_metadata = FunctionMetadata()
+
+    return any_func._great_ai_metadata

great_ai/helper/hashable_base_model.py

@@ -0,0 +1,6 @@
+from pydantic import BaseModel
+
+
+class HashableBaseModel(BaseModel):
+    def __hash__(self) -> int:
+        return hash((type(self),) + tuple(self.__dict__.values()))

great_ai/helper/snake_case_to_text.py

@@ -0,0 +1,2 @@
+def snake_case_to_text(snake_case: str) -> str:
+    return snake_case.capitalize().replace("_", " ")

great_ai/helper/strip_lines.py

@@ -0,0 +1,2 @@
+def strip_lines(text: str) -> str:
+    return "\n".join(line.strip() for line in text.split("\n"))

great_ai/helper/text_to_hex_color.py

@@ -0,0 +1,13 @@
+import colorsys
+from hashlib import md5
+
+
+def text_to_hex_color(text: str) -> str:
+    ascii_bytes = text.encode("ascii")
+    digest = md5(
+        ascii_bytes
+    ).hexdigest()  # the built-in hash function is salted differently in each process
+    integer = int(digest, 16)
+    hue = integer % 6311 / 6311.0
+    rgb = colorsys.hsv_to_rgb(hue, 0.75, 0.6)
+    return "#" + "".join("%02X" % round(i * 255) for i in rgb)

great_ai/helper/use_http_exceptions.py

@@ -0,0 +1,20 @@
+from functools import wraps
+from typing import Any, Callable, Dict, List, TypeVar, cast
+
+from fastapi import HTTPException, status
+
+F = TypeVar("F", bound=Callable[..., Any])
+
+
+def use_http_exceptions(func: F) -> F:
+    @wraps(func)
+    def wrapper(*args: List[Any], **kwargs: Dict[str, Any]) -> Any:
+        try:
+            return func(*args, **kwargs)
+        except Exception as e:
+            raise HTTPException(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail=f"The following exception has occurred: {type(e).__name__}: {e}",
+            )
+
+    return cast(F, wrapper)

great_ai/large_file/__init__.py

@@ -0,0 +1 @@
+from .large_file import LargeFileBase, LargeFileLocal, LargeFileMongo, LargeFileS3

great_ai/large_file/__main__.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+
+from argparse import Namespace
+from pathlib import Path
+from typing import Mapping, Type
+
+from ..utilities import get_logger
+from .large_file import LargeFileBase, LargeFileLocal, LargeFileMongo, LargeFileS3
+from .parse_arguments import parse_arguments
+
+logger = get_logger("large_file")
+
+
+def main() -> None:
+    parser, args = parse_arguments()
+
+    large_file = get_class(args)
+
+    if not args.cache and not args.push and not args.delete:
+        logger.warning("No action required.")
+        parser.print_help()
+
+    if args.cache:
+        for c in args.cache:
+            split = c.split(":")
+            file_name = split[0]
+            version = None if len(split) == 1 else int(split[1])
+            large_file(file_name, "r", version=version).get()
+
+    if args.push:
+        for p in args.push:
+            path = Path(p)
+            large_file(path.name, "w").push(path)
+
+    if args.delete:
+        for f in args.delete:
+            large_file(f).delete()
+
+
+def get_class(args: Namespace) -> Type[LargeFileBase]:
+    factory: Mapping[str, Type[LargeFileBase]] = {
+        "s3": LargeFileS3,
+        "local": LargeFileLocal,
+        "mongodb": LargeFileMongo,
+    }
+
+    if args.backend not in factory:
+        raise ValueError(
+            f"Backend {args.backend} does not exits, available options: {' ,'.join(factory.keys())}"
+        )
+
+    large_file = factory[args.backend]
+
+    if args.backend != "local":
+        if args.secrets is None:
+            raise ValueError(
+                "Providing a credentials file is required when the backend mode is not `local`."
+            )
+        large_file.configure_credentials_from_file(args.secrets)
+
+    return large_file
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except KeyboardInterrupt:
+        logger.warning("Exiting")
+        exit()
+    except Exception as e:
+        logger.exception(e)

great_ai/large_file/helper/__init__.py

@@ -0,0 +1,2 @@
+from .human_readable_to_byte import human_readable_to_byte
+from .progress_bar import DownloadProgressBar, UploadProgressBar

great_ai/large_file/helper/bytes_to_megabytes.py

@@ -0,0 +1,2 @@
+def bytes_to_megabytes(bytes: int) -> str:
+    return f"{round(bytes / 1000 / 1000, 2):.2f}"

great_ai/large_file/helper/human_readable_to_byte.py

@@ -0,0 +1,31 @@
+import re
+
+
+def human_readable_to_byte(size: str) -> int:
+    """Case is ignored, kb, kB, Kb, and KB are all treated as kilobyte."""
+
+    if size.strip() == "0":
+        return 0
+
+    possible_units = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
+    units_re = "|".join(possible_units)
+    regex = re.compile(
+        rf"""
+        \s*                         # trim
+        (?P<scalar>\d+(.\d+)?)      # get scalar, it might be a float
+        \s*                         # ignore optional whitespace
+        (?P<unit>{units_re})        # capture the unit
+    """,
+        flags=re.VERBOSE | re.IGNORECASE,
+    )
+
+    match = regex.match(size)
+    if not match:
+        raise ValueError(f'Could not find values in "{size}"')
+
+    results = match.groupdict()
+
+    scalar = float(results["scalar"])
+    idx = possible_units.index(results["unit"].upper())
+    factor = 1024**idx
+    return round(scalar * factor)

great_ai/large_file/helper/progress_bar.py

@@ -0,0 +1,51 @@
+import os
+import threading
+from logging import Logger
+from pathlib import Path
+
+from .bytes_to_megabytes import bytes_to_megabytes
+
+
+class ProgressBar:
+    min_progress_percentage_change = 10
+
+    def __init__(self, file_size: int, logger: Logger, prefix: str):
+        self._file_size = file_size
+        self._logger = logger
+        self._prefix = prefix
+
+        self._last_percentage: float = 0
+        self._seen_so_far = 0
+        self._lock = threading.Lock()
+
+    def __call__(self, bytes_amount: int) -> None:
+        with self._lock:
+            self._seen_so_far += bytes_amount
+            percentage = (self._seen_so_far / float(self._file_size)) * 100
+
+            if (
+                percentage != 100
+                and percentage - self._last_percentage
+                < self.min_progress_percentage_change
+            ):
+                return
+
+            self._last_percentage += self.min_progress_percentage_change
+
+            file_size_mb = bytes_to_megabytes(self._file_size)
+            seen_so_far_mb = bytes_to_megabytes(self._seen_so_far)
+            progress = seen_so_far_mb.rjust(len(file_size_mb))
+            self._logger.info(
+                f"{self._prefix} {progress}/{file_size_mb} MB ({percentage:.1f}%)"
+            )
+
+
+class DownloadProgressBar(ProgressBar):
+    def __init__(self, name: str, size: int, logger: Logger):
+        super().__init__(file_size=size, logger=logger, prefix=f"Downloading {name}")
+
+
+class UploadProgressBar(ProgressBar):
+    def __init__(self, path: Path, logger: Logger):
+        size = os.path.getsize(path)
+        super().__init__(file_size=size, logger=logger, prefix=f"Uploading {path.name}")

great_ai/large_file/large_file/__init__.py

@@ -0,0 +1,4 @@
+from .large_file_base import LargeFileBase
+from .large_file_local import LargeFileLocal
+from .large_file_mongo import LargeFileMongo
+from .large_file_s3 import LargeFileS3

great_ai/large_file/large_file/large_file_base.py

@@ -0,0 +1,316 @@
+import os
+import shutil
+import tempfile
+from abc import ABC, abstractmethod
+from functools import lru_cache
+from pathlib import Path
+from types import TracebackType
+from typing import IO, Any, List, Optional, Type, Union, cast
+
+from great_ai.utilities import ConfigFile, get_logger
+
+from ..helper import human_readable_to_byte
+from ..models import DataInstance
+
+logger = get_logger("large_file")
+
+
+CACHE_NAME_VERSION_SEPARATOR = "-"
+COMPRESSION_ALGORITHM = "gztar"
+ARCHIVE_EXTENSION = ".tar.gz"
+
+
+class LargeFileBase(ABC):
+    """
+    Store large files remotely. Use local cache for speed up.
+
+    Examples:
+
+    ```
+    with LargeFile("test.txt", "w", keep_last_n=3) as f:
+        for i in range(1000000):
+            f.write('test\n')
+
+    with LargeFile("test.txt", "r") as f:
+        print(f.readlines()[0])
+
+    path_to_cached_text_file = LargeFile("test.txt", version=0).get()
+    ```
+
+    By default, files are stored in the ".cache" folder and the
+    least recently use is deleted after the overall size reaches 30 GBs.
+
+    Change it with the following properties.
+
+    ```
+    LargeFile.cache_path = Path(".cache")
+    LargeFile.max_cache_size = "30GB"
+    ```
+    """
+
+    initialized = False
+    cache_path = Path(".cache")
+    max_cache_size: Optional[str] = "30GB"
+
+    def __init__(
+        self,
+        name: str,
+        mode: str = "r",
+        *,
+        buffering: int = -1,
+        encoding: Optional[str] = None,
+        errors: Optional[str] = None,
+        newline: Optional[str] = None,
+        version: Optional[int] = None,
+        keep_last_n: Optional[int] = None,
+        cache_only_mode: bool = False,
+    ):
+        self._name = name
+        self._version = version
+        self._mode = mode
+        self._keep_last_n = keep_last_n
+        self._cache_only_mode = cache_only_mode
+
+        self._buffering = buffering
+        self._encoding = encoding
+        self._errors = errors
+        self._newline = newline
+
+        LargeFileBase.cache_path.mkdir(parents=True, exist_ok=True)
+
+        self._find_instances()
+        self._check_mode_and_set_version()
+
+    @classmethod
+    def configure_credentials_from_file(
+        cls,
+        secrets_path: Union[Path, str],
+    ) -> None:
+        cls.configure_credentials(**ConfigFile(secrets_path))
+
+    @classmethod
+    def configure_credentials(
+        cls,
+    ) -> None:
+        cls.initialized = True
+
+    def __enter__(self) -> IO:
+        self._file: IO[Any] = (
+            tempfile.NamedTemporaryFile(
+                mode=self._mode,
+                buffering=self._buffering,
+                encoding=self._encoding,
+                newline=self._newline,
+                errors=self._errors,
+                delete=False,
+                prefix="large_file-",
+            )
+            if "w" in self._mode
+            else open(
+                self.get(),
+                mode=self._mode,
+                buffering=self._buffering,
+                encoding=self._encoding,
+                newline=self._newline,
+                errors=self._errors,
+            )
+        )
+
+        return self._file
+
+    def __exit__(
+        self,
+        type: Optional[Type[BaseException]],
+        exc: Optional[BaseException],
+        traceback: Optional[TracebackType],
+    ) -> bool:
+        self._file.close()
+
+        if type is None:
+            if "w" in self._mode:
+                self.push(Path(self._file.name))
+                os.unlink(self._file.name)
+        else:
+            logger.exception("Could not finish operation.")
+
+        return True
+
+    @property
+    def _local_name(self) -> str:
+        return f"{self._name}{CACHE_NAME_VERSION_SEPARATOR}{self.version}"
+
+    @property
+    def version(self) -> int:
+        return cast(int, self._version)
+
+    @lru_cache(1)
+    def get(self, hide_progress: bool = False) -> Path:
+        remote_path = next(
+            i.remote_path for i in self._instances if i.version == self._version
+        )
+
+        destination = self.cache_path / self._local_name
+        if not destination.exists():
+            logger.info(f"File {self._local_name} does not exist locally")
+
+            with tempfile.TemporaryDirectory() as tmp:
+                local_root_path = Path(tmp)
+                tmp_file_archive = (
+                    local_root_path / f"{self._local_name}{ARCHIVE_EXTENSION}"
+                )
+                self._download(
+                    remote_path, tmp_file_archive, hide_progress=hide_progress
+                )
+
+                logger.info(f"Decompressing {self._local_name}")
+                shutil.unpack_archive(str(tmp_file_archive), tmp, COMPRESSION_ALGORITHM)
+                shutil.move(str(local_root_path / self._local_name), str(destination))
+        else:
+            logger.info(f"File {self._local_name} found in cache")
+
+        return destination
+
+    def push(self, path: Union[Path, str], hide_progress: bool = False) -> None:
+        if isinstance(path, str):
+            path = Path(path)
+
+        with tempfile.TemporaryDirectory() as tmp:
+            if path.is_file():
+                logger.info(f"Copying file for {self._local_name}")
+                copy: Any = shutil.copy
+            else:
+                logger.info(f"Copying directory for {self._local_name}")
+                copy = shutil.copytree
+
+            try:
+                # Make local copy in the cache
+                shutil.rmtree(self.cache_path / self._local_name, ignore_errors=True)
+                copy(str(path), str(self.cache_path / self._local_name))
+            except shutil.SameFileError:
+                pass  # No worries
+
+            copy(str(path), str(Path(tmp) / self._local_name))
+
+            with tempfile.TemporaryDirectory() as tmp2:
+                # A directory has to be zipped and it cannot contain the output of the zipping
+                logger.info(f"Compressing {self._local_name}")
+                shutil.make_archive(
+                    str(Path(tmp2) / self._local_name),
+                    COMPRESSION_ALGORITHM,
+                    tmp,
+                )
+
+                file_to_be_uploaded = (
+                    Path(tmp2) / f"{self._local_name}{ARCHIVE_EXTENSION}"
+                )
+                self._upload(file_to_be_uploaded, hide_progress=hide_progress)
+
+        self.clean_up()
+
+    def delete(self) -> None:
+        self._keep_last_n = 0
+        self._delete_old_remote_versions()
+
+    def _find_instances(self) -> None:
+        if self._cache_only_mode:
+            self._instances = self._find_instances_from_cache()
+        else:
+            self._instances = self._find_remote_instances()
+
+        self._instances = sorted(self._instances, key=lambda i: i.version)
+
+    def _find_instances_from_cache(self) -> List[DataInstance]:
+        logger.info(f"Fetching cached versions of {self._name}")
+
+        candidates = [
+            DataInstance(
+                name=CACHE_NAME_VERSION_SEPARATOR.join(
+                    f.name.split(CACHE_NAME_VERSION_SEPARATOR)[:-1]
+                ),
+                version=int(f.name.split(CACHE_NAME_VERSION_SEPARATOR)[-1]),
+                remote_path=f,
+            )
+            for f in self.cache_path.glob(
+                f"{self._name}{CACHE_NAME_VERSION_SEPARATOR}*"
+            )
+        ]
+
+        return [c for c in candidates if c.name == self._name]
+
+    def _check_mode_and_set_version(self) -> None:
+        if "+" in self._mode:
+            raise ValueError(
+                f"File mode `{self._mode}` is not allowed3, remove the `+`."
+            )
+
+        if "w" in self._mode:
+            if self._version is not None:
+                raise ValueError("Providing a version is not allowed in write mode.")
+
+            self._version = self._instances[-1].version + 1 if self._instances else 0
+
+        elif "r" in self._mode:
+            if not self._instances:
+                raise FileNotFoundError(
+                    f"File {self._name} not found. No versions are available."
+                )
+
+            if self._version is None:
+                self._version = self._instances[-1].version
+                logger.info(
+                    f"Latest version of {self._name} is {self._version} "
+                    + f"(from versions: {self.versions_pretty})"
+                )
+            elif self._version not in [i.version for i in self._instances]:
+                raise FileNotFoundError(
+                    f"File {self._name} not found with version {self._version}. "
+                    + f"(from versions: {self.versions_pretty})"
+                )
+        else:
+            raise ValueError("Unsupported file mode.")
+
+    @property
+    def versions_pretty(self) -> str:
+        return ", ".join((str(i.version) for i in self._instances))
+
+    def clean_up(self) -> None:
+        self._delete_old_remote_versions()
+        self._prune_cache()
+
+    def _prune_cache(self) -> None:
+        self.cache_path.mkdir(parents=True, exist_ok=True)
+
+        if self.max_cache_size is None:
+            return
+
+        allowed_size = human_readable_to_byte(self.max_cache_size)
+        assert allowed_size >= 0
+
+        least_recently_read = sorted(
+            [f for f in self.cache_path.glob("*")], key=lambda f: f.stat().st_atime
+        )
+
+        while sum(os.path.getsize(f) for f in least_recently_read) > allowed_size:
+            file = least_recently_read.pop(0)
+            logger.info(
+                f"Deleting file from cache to meet quota (max_cache_size={self.max_cache_size}): {file}"
+            )
+            os.unlink(file)
+
+    @abstractmethod
+    def _find_remote_instances(self) -> List[DataInstance]:
+        pass
+
+    @abstractmethod
+    def _download(
+        self, remote_path: Any, local_path: Path, hide_progress: bool
+    ) -> None:
+        pass
+
+    @abstractmethod
+    def _upload(self, local_path: Path, hide_progress: bool) -> None:
+        pass
+
+    @abstractmethod
+    def _delete_old_remote_versions(self) -> None:
+        pass

great_ai/large_file/large_file/large_file_local.py

@@ -0,0 +1,58 @@
+from pathlib import Path
+from typing import Any, List, Optional
+
+from ...utilities import get_logger
+from ..models import DataInstance
+from .large_file_base import LargeFileBase
+
+logger = get_logger("large_file")
+
+
+class LargeFileLocal(LargeFileBase):
+    def __init__(
+        self,
+        name: str,
+        mode: str = "r",
+        *,
+        buffering: int = -1,
+        encoding: Optional[str] = None,
+        errors: Optional[str] = None,
+        newline: Optional[str] = None,
+        version: Optional[int] = None,
+        keep_last_n: Optional[int] = None,
+    ):
+        super().__init__(
+            name,
+            mode,
+            buffering=buffering,
+            encoding=encoding,
+            errors=errors,
+            newline=newline,
+            version=version,
+            keep_last_n=keep_last_n,
+            cache_only_mode=True,
+        )
+        super().configure_credentials()
+
+    def _find_remote_instances(self) -> List[DataInstance]:
+        return []
+
+    def _download(
+        self, remote_path: Any, local_path: Path, hide_progress: bool
+    ) -> None:
+        raise NotImplementedError()
+
+    def _upload(self, local_path: Path, hide_progress: bool) -> None:
+        pass  # the "upload" is already done py the parent's caching mechanism
+
+    def _delete_old_remote_versions(self) -> None:
+        if self._keep_last_n is not None:
+            for i in (
+                self._instances[: -self._keep_last_n]
+                if self._keep_last_n > 0
+                else self._instances
+            ):
+                logger.info(
+                    f"Removing old version (keep_last_n={self._keep_last_n}): {i.remote_path}"
+                )
+                i.remote_path.unlink()

great_ai/large_file/large_file/large_file_mongo.py

@@ -0,0 +1,121 @@
+import re
+from functools import cached_property
+from pathlib import Path
+from typing import Any, List, Mapping
+
+from gridfs import DEFAULT_CHUNK_SIZE, Database, GridFSBucket
+from pymongo import MongoClient
+
+from ...utilities import get_logger
+from ..helper import DownloadProgressBar, UploadProgressBar
+from ..models import DataInstance
+from .large_file_base import LargeFileBase
+
+logger = get_logger("large_file")
+
+
+MONGO_NAME_VERSION_SEPARATOR = "_"
+
+
+class LargeFileMongo(LargeFileBase):
+    mongo_connection_string = None
+    mongo_database = None
+
+    @classmethod
+    def configure_credentials(  # type: ignore
+        cls,
+        *,
+        mongo_connection_string: str,
+        mongo_database: str,
+        **_: Mapping[str, Any],
+    ) -> None:
+        cls.mongo_connection_string = mongo_connection_string
+        cls.mongo_database = mongo_database
+        super().configure_credentials()
+
+    @cached_property
+    def _client(self) -> GridFSBucket:
+        if self.mongo_connection_string is None or self.mongo_database is None:
+            raise ValueError(
+                "Please configure the MongoDB access options by calling LargeFileMongo.configure_credentials or set offline_mode=True in the constructor."
+            )
+
+        db: Database = MongoClient(self.mongo_connection_string)[self.mongo_database]
+        return GridFSBucket(db)
+
+    def _find_remote_instances(self) -> List[DataInstance]:
+        logger.debug(f"Fetching Mongo (GridFS) versions of {self._name}")
+
+        return [
+            DataInstance(
+                name=MONGO_NAME_VERSION_SEPARATOR.join(
+                    f.name.split(MONGO_NAME_VERSION_SEPARATOR)[:-1]
+                ),
+                version=int(f.name.split(MONGO_NAME_VERSION_SEPARATOR)[-1]),
+                remote_path=(f._id, f.length),
+                origin="mongodb",
+            )
+            for f in self._client.find(
+                {
+                    "filename": re.compile(
+                        re.escape(self._name + MONGO_NAME_VERSION_SEPARATOR) + ".*"
+                    )
+                }
+            )
+        ]
+
+    def _download(
+        self, remote_path: Any, local_path: Path, hide_progress: bool
+    ) -> None:
+        logger.info(f"Downloading {remote_path[0]} from Mongo (GridFS)")
+
+        progress = (
+            DownloadProgressBar(
+                name=str(remote_path[0]), size=remote_path[1], logger=logger
+            )
+            if not hide_progress
+            else None
+        )
+        with self._client.open_download_stream(remote_path[0]) as stream:
+            with open(local_path, "wb") as f:
+                while True:
+                    content = stream.read(DEFAULT_CHUNK_SIZE)
+                    f.write(content)
+
+                    if progress:
+                        progress(len(content))
+                    if len(content) < DEFAULT_CHUNK_SIZE:
+                        break
+
+    def _upload(self, local_path: Path, hide_progress: bool) -> None:
+        logger.info(f"Uploading {local_path} to Mongo (GridFS)")
+
+        progress = (
+            UploadProgressBar(path=local_path, logger=logger)
+            if not hide_progress
+            else None
+        )
+        with self._client.open_upload_stream(
+            f"{self._name}{MONGO_NAME_VERSION_SEPARATOR}{self.version}"
+        ) as stream:
+            with open(local_path, "rb") as f:
+                while True:
+                    content = f.read(DEFAULT_CHUNK_SIZE)
+                    stream.write(content)
+
+                    if progress:
+                        progress(len(content))
+                    if len(content) < DEFAULT_CHUNK_SIZE:
+                        break
+
+    def _delete_old_remote_versions(self) -> None:
+        if self._keep_last_n is not None:
+            for i in (
+                self._instances[: -self._keep_last_n]
+                if self._keep_last_n > 0
+                else self._instances
+            ):
+                logger.info(
+                    f"Removing old version from MongoDB (GridFS) (keep_last_n={self._keep_last_n}): {i.name}{MONGO_NAME_VERSION_SEPARATOR}{i.version}"
+                )
+                self._client.delete(i.remote_path[0])

great_ai/large_file/large_file/large_file_s3.py

@@ -0,0 +1,151 @@
+from functools import cached_property
+from pathlib import Path
+from typing import Any, List, Mapping, Optional
+
+import boto3
+
+from ...utilities import get_logger
+from ..helper import DownloadProgressBar, UploadProgressBar
+from ..models import DataInstance
+from .large_file_base import LargeFileBase
+
+logger = get_logger("large_file")
+
+
+S3_NAME_VERSION_SEPARATOR = "/"
+
+
+class LargeFileS3(LargeFileBase):
+    """
+    Store large files in S3. Use local cache for speed up.
+
+    Examples:
+
+    ```
+    with LargeFile("test.txt", "w", keep_last_n=3) as f:
+        for i in range(1000000):
+            f.write('test\n')
+
+    with LargeFile("test.txt", "r") as f:
+        print(f.readlines()[0])
+
+    path_to_cached_text_file = LargeFile("test.txt", version=0).get()
+    ```
+
+    By default, files are stored in the ".cache" folder and the
+    least recently use is deleted after the overall size reaches 30 GBs.
+
+    Change it with the following properties.
+
+    ```
+    LargeFile.cache_path = Path(".cache")
+    LargeFile.max_cache_size = "30GB"
+    ```
+    """
+
+    region_name = None
+    access_key_id = None
+    secret_access_key = None
+    bucket_name = None
+    endpoint_url = None
+
+    @classmethod
+    def configure_credentials(  # type: ignore
+        cls,
+        *,
+        aws_region_name: str,
+        aws_access_key_id: str,
+        aws_secret_access_key: str,
+        large_files_bucket_name: str,
+        aws_endpoint_url: Optional[str] = None,
+        **_: Mapping[str, Any],
+    ) -> None:
+        cls.region_name = aws_region_name
+        cls.access_key_id = aws_access_key_id
+        cls.secret_access_key = aws_secret_access_key
+        cls.bucket_name = large_files_bucket_name
+        cls.endpoint_url = aws_endpoint_url
+        super().configure_credentials()
+
+    @cached_property
+    def _client(self) -> boto3.client:
+        if (
+            self.region_name is None
+            or self.access_key_id is None
+            or self.secret_access_key is None
+            or self.bucket_name is None
+        ):
+            raise ValueError(
+                "Please configure the S3 access options by calling LargeFileS3.configure_credentials or set offline_mode=True in the constructor."
+            )
+
+        return boto3.client(
+            "s3",
+            aws_access_key_id=self.access_key_id,
+            aws_secret_access_key=self.secret_access_key,
+            region_name=self.region_name,
+            endpoint_url=self.endpoint_url,
+        )
+
+    def _find_remote_instances(self) -> List[DataInstance]:
+        logger.debug(f"Fetching S3 versions of {self._name}")
+
+        found_objects = self._client.list_objects_v2(
+            Bucket=self.bucket_name, Prefix=self._name
+        )
+        return (
+            [
+                DataInstance(
+                    name=o["Key"].split(S3_NAME_VERSION_SEPARATOR)[0],
+                    version=int(o["Key"].split(S3_NAME_VERSION_SEPARATOR)[-1]),
+                    remote_path=o["Key"],
+                )
+                for o in found_objects["Contents"]
+                if o["Key"].split(S3_NAME_VERSION_SEPARATOR)[0] == self._name
+            ]
+            if "Contents" in found_objects
+            else []
+        )
+
+    def _download(
+        self, remote_path: Any, local_path: Path, hide_progress: bool
+    ) -> None:
+        logger.info(f"Downloading {remote_path} from S3")
+
+        size = self._client.head_object(Bucket=self.bucket_name, Key=remote_path)[
+            "ContentLength"
+        ]
+
+        self._client.download_file(
+            Bucket=self.bucket_name,
+            Key=remote_path,
+            Filename=str(local_path),
+            Callback=None
+            if hide_progress
+            else DownloadProgressBar(name=str(remote_path), size=size, logger=logger),
+        )
+
+    def _upload(self, local_path: Path, hide_progress: bool) -> None:
+        key = f"{self._name}/{self.version}"
+        logger.info(f"Uploading {self._local_name} to S3 as {key}")
+
+        self._client.upload_file(
+            Filename=str(local_path),
+            Bucket=self.bucket_name,
+            Key=key,
+            Callback=None
+            if hide_progress
+            else UploadProgressBar(path=local_path, logger=logger),
+        )
+
+    def _delete_old_remote_versions(self) -> None:
+        if self._keep_last_n is not None:
+            for i in (
+                self._instances[: -self._keep_last_n]
+                if self._keep_last_n > 0
+                else self._instances
+            ):
+                logger.info(
+                    f"Removing old version from S3 (keep_last_n={self._keep_last_n}): {i.remote_path}"
+                )
+                self._client.delete_object(Bucket=self.bucket_name, Key=i.remote_path)

great_ai/large_file/models/__init__.py

@@ -0,0 +1 @@
+from .data_instance import DataInstance

great_ai/large_file/models/data_instance.py

@@ -0,0 +1,9 @@
+from typing import Any
+
+from pydantic import BaseModel
+
+
+class DataInstance(BaseModel):
+    name: str
+    version: int
+    remote_path: Any

great_ai/large_file/parse_arguments.py

@@ -0,0 +1,56 @@
+from argparse import ArgumentParser, Namespace
+from typing import Tuple
+
+
+def parse_arguments() -> Tuple[ArgumentParser, Namespace]:
+    parser = ArgumentParser(
+        description="Store and version large files in S3; open them like regular files. Caching included.",
+    )
+
+    parser.add_argument(
+        "-b",
+        "--backend",
+        type=str,
+        help="choose which backend to use, available options: `local`, `s3`, `mongodb`",
+        required=True,
+    )
+
+    parser.add_argument(
+        "-s",
+        "--secrets",
+        type=str,
+        help="path to an .ini configuration file with your S3 credentials",
+        required=False,
+    )
+
+    parser.add_argument(
+        "-c",
+        "--cache",
+        nargs="+",
+        type=str,
+        help="download file into local cache, example: file_name.txt:version",
+        required=False,
+    )
+
+    parser.add_argument(
+        "-p",
+        "--push",
+        nargs="+",
+        type=str,
+        help="push a local file into S3 and set it as the most recent version",
+        required=False,
+    )
+
+    parser.add_argument(
+        "-d",
+        "--delete",
+        nargs="+",
+        type=str,
+        help="delete every version of file from S3",
+        required=False,
+    )
+
+    parser.print_usage = parser.print_help  # type: ignore
+    args = parser.parse_args()
+
+    return parser, args

great_ai/models/__init__.py

@@ -0,0 +1,2 @@
+from .save_model import save_model
+from .use_model import model_versions, use_model

great_ai/models/save_model.py

@@ -0,0 +1,24 @@
+from pathlib import Path
+from typing import Optional, Union
+
+from dill import dump
+
+from ..context import get_context
+
+
+def save_model(
+    model: Union[Path, str, object], key: str, *, keep_last_n: Optional[int] = None
+) -> str:
+    file = get_context().large_file_implementation(
+        name=key, mode="wb", keep_last_n=keep_last_n
+    )
+
+    if isinstance(model, Path) or isinstance(model, str):
+        file.push(model)
+    else:
+        with file as f:
+            dump(model, f)
+
+    get_context().logger.info(f"Model {key} uploaded with version {file.version}")
+
+    return f"{key}:{file.version}"

great_ai/models/use_model.py

@@ -0,0 +1,73 @@
+from functools import wraps
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    List,
+    Literal,
+    Optional,
+    Set,
+    Tuple,
+    TypeVar,
+    Union,
+    cast,
+)
+
+from dill import load
+
+from ..context import get_context
+from ..helper import get_function_metadata_store
+from ..helper.assert_function_is_not_finalised import assert_function_is_not_finalised
+from ..tracing.tracing_context import TracingContext
+from ..views import Model
+
+F = TypeVar("F", bound=Callable[..., Any])
+
+
+def use_model(
+    key: str,
+    *,
+    version: Union[int, Literal["latest"]] = "latest",
+    model_kwarg_name: str = "model",
+) -> Callable[[F], F]:
+    assert (
+        isinstance(version, int) or version == "latest"
+    ), "Only integers or the string literal `latest` is allowed as a version"
+
+    model, actual_version = _load_model(
+        key=key,
+        version=None if version == "latest" else version,
+    )
+
+    def decorator(func: F) -> F:
+        assert_function_is_not_finalised(func)
+
+        store = get_function_metadata_store(func)
+        store.model_parameter_names.append(model_kwarg_name)
+
+        @wraps(func)
+        def wrapper(*args: List[Any], **kwargs: Dict[str, Any]) -> Any:
+            tracing_context = TracingContext.get_current_tracing_context()
+            if tracing_context:
+                tracing_context.log_model(Model(key=key, version=actual_version))
+            return func(*args, **kwargs, **{model_kwarg_name: model})
+
+        return cast(F, wrapper)
+
+    return decorator
+
+
+model_versions: Set[Tuple[str, int]] = set()
+
+
+def _load_model(key: str, version: Optional[int] = None) -> Tuple[Any, int]:
+    file = get_context().large_file_implementation(name=key, mode="rb", version=version)
+    path = file.get()
+
+    model_versions.add((key, file.version))
+
+    if path.is_dir():
+        return path, file.version
+
+    with file as f:
+        return load(f), file.version

great_ai/output_views/__init__.py

@@ -0,0 +1,3 @@
+from .classification_output import ClassificationOutput
+from .multi_label_classification_output import MultiLabelClassificationOutput
+from .regression_output import RegressionOutput

great_ai/output_views/classification_output.py

@@ -0,0 +1,9 @@
+from typing import Any, Optional, Union
+
+from ..helper import HashableBaseModel
+
+
+class ClassificationOutput(HashableBaseModel):
+    label: Union[str, int]
+    confidence: float
+    explanation: Optional[Any]

great_ai/output_views/multi_label_classification_output.py

@@ -0,0 +1,8 @@
+from typing import List
+
+from ..helper import HashableBaseModel
+from .classification_output import ClassificationOutput
+
+
+class MultiLabelClassificationOutput(HashableBaseModel):
+    labels: List[ClassificationOutput] = []

great_ai/output_views/regression_output.py

@@ -0,0 +1,8 @@
+from typing import Any, Optional, Union
+
+from ..helper import HashableBaseModel
+
+
+class RegressionOutput(HashableBaseModel):
+    value: Union[int, float]
+    explanation: Optional[Any]

great_ai/output_views/sequence_labeling_output.py

@@ -0,0 +1 @@
+# todo

great_ai/parameters/__init__.py

@@ -0,0 +1,3 @@
+from .automatically_decorate_parameters import automatically_decorate_parameters
+from .log_metric import log_metric
+from .parameter import parameter

great_ai/parameters/automatically_decorate_parameters.py

@@ -0,0 +1,27 @@
+import inspect
+from typing import Any, Callable, TypeVar
+
+from ..helper.get_function_metadata_store import get_function_metadata_store
+from .parameter import parameter
+
+F = TypeVar("F", bound=Callable[..., Any])
+
+
+def automatically_decorate_parameters(func: F) -> F:
+    signature = inspect.signature(func)
+    parameter_names = [
+        param.name
+        for param in signature.parameters.values()
+        if param.kind == param.POSITIONAL_OR_KEYWORD
+    ]
+
+    metadata = get_function_metadata_store(func)
+
+    for name in parameter_names:
+        if (
+            name not in metadata.model_parameter_names
+            and name not in metadata.input_parameter_names
+        ):
+            func = parameter(name)(func)
+
+    return func

great_ai/parameters/log_metric.py

@@ -0,0 +1,15 @@
+import inspect
+from typing import Any
+
+from ..context import get_context
+from ..tracing import TracingContext
+
+
+def log_metric(argument_name: str, value: Any) -> None:
+    tracing_context = TracingContext.get_current_tracing_context()
+    caller = inspect.stack()[1].function
+    actual_name = f"metric:{caller}:{argument_name}"
+    if tracing_context:
+        tracing_context.log_value(name=actual_name, value=value)
+
+    get_context().logger.info(f"{actual_name}={value}")

great_ai/parameters/parameter.py

@@ -0,0 +1,52 @@
+from functools import wraps
+from typing import Any, Callable, Dict, TypeVar, cast
+
+from typeguard import check_type
+
+from ..exceptions import ArgumentValidationError
+from ..helper import get_arguments, get_function_metadata_store
+from ..helper.assert_function_is_not_finalised import assert_function_is_not_finalised
+from ..tracing.tracing_context import TracingContext
+
+T = TypeVar("T")
+F = TypeVar("F", bound=Callable[..., Any])
+
+
+def parameter(
+    parameter_name: str,
+    *,
+    validator: Callable[[T], bool] = lambda _: True,
+    disable_logging: bool = False,
+) -> Callable[[F], F]:
+    def decorator(func: F) -> F:
+        get_function_metadata_store(func).input_parameter_names.append(parameter_name)
+        assert_function_is_not_finalised(func)
+
+        actual_name = f"arg:{parameter_name}"
+
+        @wraps(func)
+        def wrapper(*args: Any, **kwargs: Dict[str, Any]) -> Any:
+            arguments = get_arguments(func, args, kwargs)
+            argument = arguments.get(parameter_name)
+
+            expected_type = func.__annotations__.get(parameter_name)
+
+            if expected_type is not None:
+                check_type(parameter_name, argument, expected_type)
+
+            if not validator(argument):
+                raise ArgumentValidationError(
+                    f"Argument {parameter_name} in {func.__name__} did not pass validation"
+                )
+
+            context = TracingContext.get_current_tracing_context()
+            if context and not disable_logging:
+                context.log_value(name=f"{actual_name}:value", value=argument)
+                if isinstance(argument, str):
+                    context.log_value(name=f"{actual_name}:length", value=len(argument))
+
+            return func(*args, **kwargs)
+
+        return cast(F, wrapper)
+
+    return decorator

great_ai/parse_arguments.py

@@ -0,0 +1,51 @@
+from argparse import ArgumentParser, Namespace
+
+
+def parse_arguments() -> Namespace:
+    parser = ArgumentParser(
+        description="GreatAI-Server for deploying you AI applications with ease.",
+    )
+
+    parser.add_argument(
+        "file_name",
+        type=str,
+        help="the name of the file containing your to-be-served function such as `main.py`\n",
+    )
+
+    default_host = "0.0.0.0"
+    parser.add_argument(
+        "--host",
+        type=str,
+        help=f"it is passed to uvicorn which starts a server listening on this address (default: {default_host})",
+        default=default_host,
+        required=False,
+    )
+
+    default_port = 6060
+    parser.add_argument(
+        "--port",
+        type=int,
+        help=f"it is passed to uvicorn which starts a server listening on this port (default: {default_port})",
+        default=default_port,
+        required=False,
+    )
+
+    default_timeout_keep_alive = 600
+    parser.add_argument(
+        "--timeout_keep_alive",
+        type=int,
+        help=f"it is passed to uvicorn which uses it for timing out requests taking longer than this many seconds (default: {default_timeout_keep_alive})",
+        default=600,
+        required=False,
+    )
+
+    default_worker_count = 1
+    parser.add_argument(
+        "--worker_count",
+        type=int,
+        help=f"it is passed to uvicorn which starts this many server processes (default: {default_worker_count})",
+        default=default_worker_count,
+        required=False,
+    )
+
+    return parser.parse_args()

great_ai/persistence/__init__.py

@@ -0,0 +1,3 @@
+from .mongodb_driver import MongodbDriver
+from .parallel_tinydb_driver import ParallelTinyDbDriver
+from .tracing_database_driver import TracingDatabaseDriver

great_ai/persistence/mongodb_driver.py

@@ -0,0 +1,137 @@
+from datetime import datetime
+from typing import Any, List, Mapping, Optional, Sequence, Tuple
+
+from pymongo import MongoClient
+
+from ..views import Filter, SortBy, Trace
+from .tracing_database_driver import TracingDatabaseDriver
+
+operator_mapping = {
+    "=": "$eq",
+    "!=": "$ne",
+    "<": "$lt",
+    "<=": "$lte",
+    ">": "$gt",
+    ">=": "$gte",
+    "contains": "$regex",
+}
+
+
+class MongodbDriver(TracingDatabaseDriver):
+    is_production_ready = True
+
+    def __init__(self) -> None:
+        super().__init__()
+        if self.mongo_connection_string is None or self.mongo_database is None:
+            raise ValueError(
+                "Please configure the MongoDB access options by calling MongodbDriver.configure_credentials"
+            )
+
+    @classmethod
+    def configure_credentials(  # type: ignore
+        cls,
+        *,
+        mongo_connection_string: str,
+        mongo_database: str,
+        **_: Mapping[str, Any],
+    ) -> None:
+        cls.mongo_connection_string = mongo_connection_string
+        cls.mongo_database = mongo_database
+        super().configure_credentials()
+
+    def save(self, trace: Trace) -> str:
+        serialized = trace.to_flat_dict()
+        serialized["_id"] = trace.trace_id
+
+        with MongoClient(self.mongo_connection_string) as client:
+            return client[self.mongo_database].traces.insert_one(serialized)
+
+    def save_batch(self, documents: List[Trace]) -> List[str]:
+        serialized = [d.to_flat_dict() for d in documents]
+        for s in serialized:
+            s["_id"] = s["trace_id"]
+
+        with MongoClient(self.mongo_connection_string) as client:
+            return client[self.mongo_database].traces.insert_many(
+                serialized, ordered=False
+            )
+
+    def get(self, id: str) -> Optional[Trace]:
+        with MongoClient(self.mongo_connection_string) as client:
+            value = client[self.mongo_database].traces.find_one(id)
+
+        if value:
+            value = Trace.parse_obj(value)
+
+        return value
+
+    def _get_operator(self, filter: Filter) -> str:
+        if filter.operator == "contains" and not isinstance(filter.value, str):
+            return operator_mapping["="]
+        return operator_mapping[filter.operator]
+
+    def query(
+        self,
+        *,
+        skip: int = 0,
+        take: Optional[int] = None,
+        conjunctive_filters: Sequence[Filter] = [],
+        conjunctive_tags: Sequence[str] = [],
+        since: Optional[datetime] = None,
+        until: Optional[datetime] = None,
+        has_feedback: Optional[bool] = None,
+        sort_by: Sequence[SortBy] = [],
+    ) -> Tuple[List[Trace], int]:
+
+        query = {
+            "filter": {
+                "$and": [{"tags": tag} for tag in conjunctive_tags]
+                + [
+                    {f.property: {self._get_operator(f): f.value}}
+                    for f in conjunctive_filters
+                ]
+                + [{}]
+            },
+            "sort": [
+                (col.column_id, 1 if col.direction == "asc" else -1) for col in sort_by
+            ],
+        }
+
+        if skip:
+            query["skip"] = skip
+
+        if take:
+            query["limit"] = take
+
+        if since:
+            query["filter"]["$and"].append({"created": {"$gte": since}})
+
+        if until:
+            query["filter"]["$and"].append({"created": {"$lte": until}})
+
+        if has_feedback is not None:
+            query["filter"]["$and"].append(
+                {"feedback": {"$ne": None}} if has_feedback else {"feedback": None}
+            )
+
+        with MongoClient(self.mongo_connection_string) as client:
+            values = client[self.mongo_database].traces.find(**query)
+            documents = [Trace.parse_obj(t) for t in values]
+
+        return documents, len(documents)
+
+    def update(self, id: str, new_version: Trace) -> None:
+        serialized = new_version.to_flat_dict()
+        serialized["_id"] = new_version.trace_id
+        with MongoClient(self.mongo_connection_string) as client:
+            client[self.mongo_database].traces.update_one(id, new_version)
+
+    def delete(self, id: str) -> None:
+        with MongoClient(self.mongo_connection_string) as client:
+            client[self.mongo_database].traces.delete_one(id)
+
+    def delete_batch(self, ids: List[str]) -> List[str]:
+        delete_filter = {"_id": {"$in": ids}}
+
+        with MongoClient(self.mongo_connection_string) as client:
+            return client[self.mongo_database].traces.delete_many(delete_filter)

great_ai/persistence/parallel_tinydb_driver.py

@@ -0,0 +1,112 @@
+from datetime import datetime
+from multiprocessing import Lock
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, cast
+
+import pandas as pd
+from tinydb import TinyDB
+
+from ..views import Filter, SortBy, Trace
+from .tracing_database_driver import TracingDatabaseDriver
+
+DEFAULT_TRACING_DB_FILENAME = "tracing_database.json"
+lock = Lock()
+
+
+operator_mapping = {"=": "eq", "!=": "ne", "<": "lt", "<=": "le", ">": "gt", ">=": "ge"}
+
+
+class ParallelTinyDbDriver(TracingDatabaseDriver):
+    is_production_ready = False
+    path_to_db = Path(DEFAULT_TRACING_DB_FILENAME)
+
+    def save(self, trace: Trace) -> str:
+        return self._safe_execute(lambda db: db.insert(trace.dict()))
+
+    def save_batch(self, documents: List[Trace]) -> List[str]:
+        traces = [d.dict() for d in documents]
+        return self._safe_execute(lambda db: db.insert_multiple(traces))
+
+    def get(self, id: str) -> Optional[Trace]:
+        value = self._safe_execute(lambda db: db.get(lambda d: d["trace_id"] == id))
+        if value:
+            value = Trace.parse_obj(value)
+        return value
+
+    def query(
+        self,
+        *,
+        skip: int = 0,
+        take: Optional[int] = None,
+        conjunctive_filters: Sequence[Filter] = [],
+        conjunctive_tags: Sequence[str] = [],
+        since: Optional[datetime] = None,
+        until: Optional[datetime] = None,
+        has_feedback: Optional[bool] = None,
+        sort_by: Sequence[SortBy] = []
+    ) -> Tuple[List[Trace], int]:
+        def does_match(d: Dict[str, Any]) -> bool:
+            return (
+                not set(conjunctive_tags) - set(d["tags"])
+                and (
+                    since is None
+                    or cast(datetime, datetime.fromisoformat(d["created"])) >= since
+                )
+                and (
+                    until is None
+                    or cast(datetime, datetime.fromisoformat(d["created"])) <= until
+                )
+                and (
+                    has_feedback is None or has_feedback == (d["feedback"] is not None)
+                )
+            )
+
+        documents: List[Trace] = [
+            Trace.parse_obj(t)
+            for t in self._safe_execute(lambda db: db.search(does_match))
+        ]
+
+        if not documents:
+            return [], 0
+
+        df = pd.DataFrame([d.to_flat_dict() for d in documents])
+
+        for f in conjunctive_filters:
+            operator = f.operator.lower()
+            if operator in operator_mapping:
+                df = df.loc[
+                    getattr(df[f.property], operator_mapping[f.operator])(f.value)
+                ]
+            elif operator == "contains":
+                df = df.loc[df[f.property].str.contains(f.value, case=False)]
+
+        if sort_by:
+            df.sort_values(
+                [col.column_id for col in sort_by],
+                ascending=[col.direction == "asc" for col in sort_by],
+                inplace=True,
+            )
+
+        count = len(df)
+        result = df.iloc[skip:] if take is None else df.iloc[skip : skip + take]
+        return [
+            next(d for d in documents if d.trace_id == trace_id)
+            for trace_id in result["trace_id"]
+        ], count
+
+    def update(self, id: str, new_version: Trace) -> None:
+        self._safe_execute(
+            lambda db: db.update(new_version.dict(), lambda d: d["trace_id"] == id)
+        )
+
+    def delete(self, id: str) -> None:
+        self._safe_execute(lambda db: db.remove(lambda d: d["trace_id"] == id))
+
+    def delete_batch(self, ids: List[str]) -> List[str]:
+        for i in ids:
+            self.delete(i)
+
+    def _safe_execute(self, func: Callable[[TinyDB], Any]) -> Any:
+        with lock:
+            with TinyDB(self.path_to_db) as db:
+                return func(db)

great_ai/persistence/tracing_database_driver.py

@@ -0,0 +1,70 @@
+from abc import ABC, abstractmethod
+from datetime import datetime
+from pathlib import Path
+from typing import List, Optional, Sequence, Tuple, Union
+
+from ..utilities import ConfigFile
+from ..views import Filter, SortBy, Trace
+
+
+class TracingDatabaseDriver(ABC):
+    is_production_ready: bool
+    initialized: bool = False
+
+    @classmethod
+    def configure_credentials_from_file(
+        cls,
+        secrets_path: Union[Path, str],
+    ) -> None:
+        cls.configure_credentials(**ConfigFile(secrets_path))
+
+    @classmethod
+    def configure_credentials(
+        cls,
+    ) -> None:
+        cls.initialized = True
+
+    @abstractmethod
+    def save(self, document: Trace) -> str:
+        pass
+
+    @abstractmethod
+    def save_batch(
+        self,
+        documents: List[Trace],
+    ) -> List[str]:
+        pass
+
+    @abstractmethod
+    def get(self, id: str) -> Optional[Trace]:
+        pass
+
+    @abstractmethod
+    def query(
+        self,
+        *,
+        skip: int = 0,
+        take: Optional[int] = None,
+        conjunctive_filters: Sequence[Filter] = [],
+        conjunctive_tags: Sequence[str] = [],
+        until: Optional[datetime] = None,
+        since: Optional[datetime] = None,
+        has_feedback: Optional[bool] = None,
+        sort_by: Sequence[SortBy] = []
+    ) -> Tuple[List[Trace], int]:
+        pass
+
+    @abstractmethod
+    def update(self, id: str, new_version: Trace) -> None:
+        pass
+
+    @abstractmethod
+    def delete(self, id: str) -> None:
+        pass
+
+    @abstractmethod
+    def delete_batch(
+        self,
+        ids: List[str],
+    ) -> None:
+        pass

great_ai/remote/__init__.py

@@ -0,0 +1,4 @@
+from .call_remote_great_ai import call_remote_great_ai
+from .call_remote_great_ai_async import call_remote_great_ai_async
+from .http_client import HttpClient
+from .remote_call_error import RemoteCallError

great_ai/remote/call_remote_great_ai.py

@@ -0,0 +1,34 @@
+import asyncio
+from typing import Any, Mapping, Optional, Type, TypeVar
+
+from pydantic import BaseModel
+
+from great_ai.utilities import get_logger
+
+from ..views import Trace
+from .call_remote_great_ai_async import call_remote_great_ai_async
+
+logger = get_logger("call_remote_great_ai")
+
+T = TypeVar("T", bound=BaseModel)
+
+
+def call_remote_great_ai(
+    base_uri: str,
+    data: Mapping[str, Any],
+    retry_count: int = 4,
+    model_class: Optional[Type[T]] = None,
+) -> Trace[T]:
+    try:
+        asyncio.get_running_loop()
+        raise Exception(
+            f"Already running in an event loop, you have to call `{call_remote_great_ai_async.__name__}`"
+        )
+    except RuntimeError:
+        pass
+
+    future = call_remote_great_ai_async(
+        base_uri=base_uri, data=data, retry_count=retry_count, model_class=model_class
+    )
+
+    return asyncio.run(future)

great_ai/remote/call_remote_great_ai_async.py

@@ -0,0 +1,37 @@
+from typing import Any, Mapping, Optional, Type, TypeVar
+
+from pydantic import BaseModel
+
+from ..views import Trace
+from .http_client import HttpClient
+from .remote_call_error import RemoteCallError
+
+http: Optional[HttpClient] = None
+
+T = TypeVar("T", bound=BaseModel)
+
+
+async def call_remote_great_ai_async(
+    base_uri: str,
+    data: Mapping[str, Any],
+    retry_count: int = 4,
+    model_class: Optional[Type[T]] = None,
+) -> Trace[T]:
+    global http
+    if http is None:
+        http = HttpClient()
+
+    if base_uri.endswith("/"):
+        base_uri = base_uri[:-1]
+
+    url = f"{base_uri}/predict/"
+    response = await http.post(
+        url=url, data=data, retry_count=retry_count, expected_status=200
+    )
+
+    try:
+        if model_class is not None:
+            response["output"] = model_class.parse_obj(response["output"])
+        return Trace.parse_obj(response)
+    except Exception:
+        raise RemoteCallError("Could not parse response")

great_ai/remote/http_client.py

@@ -0,0 +1,60 @@
+import logging
+from asyncio import sleep
+from typing import Any, Mapping, Optional
+
+import aiohttp
+
+from .remote_call_error import RemoteCallError
+
+logger = logging.getLogger("http")
+
+
+class HttpClient:
+    timeout_seconds: int = 600
+    wait_between_retries_seconds: float = 5
+
+    def __init__(
+        self,
+    ) -> None:
+        timeout = aiohttp.ClientTimeout(total=self.timeout_seconds)
+
+        self._session = aiohttp.ClientSession(
+            raise_for_status=False,
+            timeout=timeout,
+        )
+
+    async def post(
+        self,
+        url: str,
+        data: Mapping[str, Any],
+        retry_count: int = 0,
+        expected_status: Optional[int] = None,
+        **kwargs: Any,
+    ) -> Any:
+        for i in range(retry_count + 1):
+            try:
+                async with self._session.post(url, json=data, **kwargs) as r:
+                    if (
+                        expected_status is not None and r.status != expected_status
+                    ) or r.status >= 500:
+                        response_text = await r.text()
+                        raise ValueError(
+                            f"Found not-expected status code: {r.status}, response is: {response_text}"
+                        )
+                    try:
+                        return await r.json()
+                    except Exception:
+                        raise RemoteCallError(
+                            "JSON parsing failed",
+                        )
+            except Exception as e:
+                if retry_count - i > 1:
+                    logger.warning(
+                        f"Request failed ({e}), {retry_count - i - 1} retries left",
+                    )
+                    await sleep(self.wait_between_retries_seconds)
+
+        raise RemoteCallError(f"Request has failed too many ({retry_count + 1}) times")
+
+    async def close(self) -> None:
+        await self._session.close()

great_ai/remote/remote_call_error.py

@@ -0,0 +1,2 @@
+class RemoteCallError(Exception):
+    pass

great_ai/tracing/__init__.py

@@ -0,0 +1,4 @@
+from .add_ground_truth import add_ground_truth
+from .delete_ground_truth import delete_ground_truth
+from .query_ground_truth import query_ground_truth
+from .tracing_context import TracingContext

great_ai/tracing/add_ground_truth.py

@@ -0,0 +1,67 @@
+from datetime import datetime
+from math import ceil
+from random import shuffle
+from typing import Any, Iterable, List, TypeVar
+
+from ..constants import (
+    GROUND_TRUTH_TAG_NAME,
+    TEST_SPLIT_TAG_NAME,
+    TRAIN_SPLIT_TAG_NAME,
+    VALIDATION_SPLIT_TAG_NAME,
+)
+from ..context import get_context
+from ..views import Trace
+
+T = TypeVar("T")
+
+
+def add_ground_truth(
+    inputs: Iterable[Any],
+    expected_outputs: Iterable[T],
+    *,
+    tags: List[str] = [],
+    train_split_ratio: float = 1,
+    test_split_ratio: float = 0,
+    validation_split_ratio: float = 0
+) -> None:
+    get_context()  # this resets the seed
+
+    inputs = list(inputs)
+    expected_outputs = list(expected_outputs)
+    assert len(inputs) == len(
+        expected_outputs
+    ), "The length of the inputs and expected_outputs must be equal"
+
+    sum_ratio = train_split_ratio + test_split_ratio + validation_split_ratio
+    assert sum_ratio > 0, "The sum of the split ratios must be a positive number"
+
+    train_split_ratio /= sum_ratio
+    test_split_ratio /= sum_ratio
+    validation_split_ratio /= sum_ratio
+
+    values = list(zip(inputs, expected_outputs))
+    shuffle(values)
+
+    split_tags = (
+        [TRAIN_SPLIT_TAG_NAME] * ceil(train_split_ratio * len(inputs))
+        + [TEST_SPLIT_TAG_NAME] * ceil(test_split_ratio * len(inputs))
+        + [VALIDATION_SPLIT_TAG_NAME] * ceil(validation_split_ratio * len(inputs))
+    )
+    shuffle(split_tags)
+
+    created = datetime.utcnow().isoformat()
+    traces = [
+        Trace(
+            created=created,
+            original_execution_time_ms=0,
+            logged_values=X if isinstance(X, dict) else {"input": X},
+            models=[],
+            output=y,
+            feedback=y,
+            exception=None,
+            tags=[GROUND_TRUTH_TAG_NAME, split_tag, *tags],
+        )
+        for ((X, y), split_tag) in zip(values, split_tags)
+    ]
+
+    get_context().tracing_database.save_batch(traces)

great_ai/tracing/delete_ground_truth.py

@@ -0,0 +1,22 @@
+from datetime import datetime
+from typing import List, Optional, Union
+
+from ..context import get_context
+
+
+def delete_ground_truth(
+    conjunctive_tags: Union[List[str], str] = [],
+    *,
+    until: Optional[datetime] = None,
+    since: Optional[datetime] = None,
+) -> None:
+    tags = (
+        conjunctive_tags if isinstance(conjunctive_tags, list) else [conjunctive_tags]
+    )
+    db = get_context().tracing_database
+
+    items, length = db.query(
+        conjunctive_tags=tags, until=until, since=since, has_feedback=True
+    )
+
+    db.delete_batch([i.trace_id for i in items])

great_ai/tracing/query_ground_truth.py

@@ -0,0 +1,22 @@
+from datetime import datetime
+from typing import List, Optional, Union
+
+from ..context import get_context
+from ..views import Trace
+
+
+def query_ground_truth(
+    conjunctive_tags: Union[List[str], str] = [],
+    *,
+    since: Optional[datetime] = None,
+    return_max_count: Optional[int] = None
+) -> List[Trace]:
+    tags = (
+        conjunctive_tags if isinstance(conjunctive_tags, list) else [conjunctive_tags]
+    )
+    db = get_context().tracing_database
+
+    items, length = db.query(
+        conjunctive_tags=tags, since=since, take=return_max_count, has_feedback=True
+    )
+    return items

great_ai/tracing/tracing_context.py

@@ -0,0 +1,86 @@
+from contextvars import ContextVar
+from datetime import datetime
+from types import TracebackType
+from typing import Any, Dict, Generic, List, Literal, Optional, Type, TypeVar
+
+from ..constants import DEVELOPMENT_TAG_NAME, ONLINE_TAG_NAME, PRODUCTION_TAG_NAME
+from ..context import get_context
+from ..views import Model, Trace
+
+T = TypeVar("T")
+
+
+class TracingContext(Generic[T]):
+    def __init__(self, function_name: str, do_not_persist_traces: bool) -> None:
+        self._do_not_persist_traces = do_not_persist_traces
+        self._models: List[Model] = []
+        self._values: Dict[str, Any] = {}
+        self._trace: Optional[Trace[T]] = None
+        self._start_time = datetime.utcnow()
+        self._name = function_name
+
+    def log_value(self, name: str, value: Any) -> None:
+        self._values[name] = value
+
+    def log_model(self, model: Model) -> None:
+        self._models.append(model)
+
+    def finalise(self, output: T = None, exception: BaseException = None) -> Trace[T]:
+        assert self._trace is None, "has been already finalised"
+
+        delta_time = (datetime.utcnow() - self._start_time).microseconds / 1000
+        self._trace = Trace(
+            created=self._start_time.isoformat(),
+            original_execution_time_ms=delta_time,
+            logged_values=self._values,
+            models=self._models,
+            output=output,
+            exception=None
+            if exception is None
+            else f"{type(exception).__name__}: {exception}",
+            tags=[
+                self._name,
+                ONLINE_TAG_NAME,
+                PRODUCTION_TAG_NAME
+                if get_context().is_production
+                else DEVELOPMENT_TAG_NAME,
+            ],
+        )
+
+        return self._trace
+
+    @staticmethod
+    def get_current_tracing_context() -> Optional["TracingContext"]:
+        return _current_tracing_context.get()
+
+    def __enter__(self) -> "TracingContext":
+        _current_tracing_context.set(self)
+        return self
+
+    def __exit__(
+        self,
+        type: Optional[Type[BaseException]],
+        exception: Optional[BaseException],
+        traceback: Optional[TracebackType],
+    ) -> Literal[False]:
+        _current_tracing_context.set(None)
+
+        if exception is not None and type is not None:
+            self.finalise(exception=exception)
+            if get_context().should_log_exception_stack:
+                get_context().logger.exception("Could not finish operation")
+            else:
+                get_context().logger.error(
+                    f"Could not finish operation because of {type.__name__}: {exception}"
+                )
+
+        assert self._trace is not None
+        if not self._do_not_persist_traces:
+            get_context().tracing_database.save(self._trace)
+
+        return False
+
+
+_current_tracing_context: ContextVar[Optional[TracingContext]] = ContextVar(
+    "_current_tracing_context"
+)

great_ai/utilities/__init__.py

@@ -0,0 +1,11 @@
+from .chunk import chunk
+from .clean import clean
+from .config_file import ConfigFile, ParseError
+from .evaluate_ranking import evaluate_ranking
+from .get_sentences import get_sentences
+from .language import english_name_of_language, is_english, predict_language
+from .logger import get_logger
+from .match_names import match_names
+from .parallel_map import WorkerException, parallel_map, threaded_parallel_map
+from .unchunk import unchunk
+from .unique import unique

great_ai/utilities/chunk.py

@@ -0,0 +1,17 @@
+from typing import Iterable, List, TypeVar
+
+T = TypeVar("T")
+
+
+def chunk(values: Iterable[T], chunk_size: int) -> Iterable[T]:
+    assert chunk_size >= 1
+
+    result: List[T] = []
+    for v in values:
+        result.append(v)
+        if len(result) == chunk_size:
+            yield result
+            result = []
+
+    if len(result) > 0:
+        yield result

great_ai/utilities/clean.py

@@ -0,0 +1,68 @@
+import html
+import re
+import unicodedata
+
+import unidecode
+
+from .data import left_regular_punctuations, right_regular_punctuations
+from .external.pylatexenc.latex2text import LatexNodes2Text
+from .logger import get_logger
+
+logger = get_logger("clean")
+latex = LatexNodes2Text()
+
+
+joined_left_punctuations = "".join(left_regular_punctuations).replace("]", r"\]")
+joined_right_punctuations = "".join(right_regular_punctuations).replace("[", r"\[")
+
+
+def clean(
+    text: str,
+    ignore_xml: bool = False,
+    ignore_latex: bool = False,
+    remove_brackets: bool = False,
+    convert_to_ascii: bool = False,
+) -> str:
+    if not ignore_xml:
+        text = re.sub(r"<[^>]*>", " ", text)
+        text = html.unescape(text)
+
+    if not ignore_latex:
+        text = text.replace("%", "\\%")  # escape LaTeX comments before parsing as LaTeX
+
+        try:
+            text = latex.latex_to_text(text, tolerant_parsing=True, strict_braces=False)
+            text = text.replace("%s", " ")
+        except:
+            logger.exception("Latex parsing error")
+
+    if convert_to_ascii:
+        text = unicodedata.normalize("NFKD", text)
+
+        try:
+            text.encode("ASCII", errors="strict")
+        except UnicodeEncodeError:
+            text = "".join([c for c in text if not unicodedata.combining(c)])
+            text = unidecode.unidecode(text)
+
+    text = re.sub(
+        r"\b[a-zA-Z](?:[\t ]+[a-zA-Z]\b)+", lambda m: re.sub(r"[\t ]", "", m[0]), text
+    )  # A R T I C L E => ARTICLE
+
+    if remove_brackets:
+        text = re.sub(r"\[[^\]]*\]", " ", text)
+
+    # fix hypens: break- word => break-word
+    text = re.sub(r"(\S)-\s+", r"\1-", text)
+    text = re.sub(r"\s+-(\S)", r"-\1", text)
+
+    # collapse whitespace
+    text = re.sub(r"\s+", " ", text)
+
+    # fix punctuation
+    text = re.sub(rf" ([{joined_left_punctuations}])", r"\1", text)
+    text = re.sub(rf"([{joined_right_punctuations}]) ", r"\1", text)
+
+    text = text.strip()
+
+    return text

great_ai/utilities/config_file/__init__.py

@@ -0,0 +1,2 @@
+from .config_file import ConfigFile
+from .parse_error import ParseError

great_ai/utilities/config_file/config_file.py

@@ -0,0 +1,91 @@
+import os
+from pathlib import Path
+from typing import Dict, Iterable, Tuple, Union
+
+from ..logger import get_logger
+from .parse_error import ParseError
+from .pattern import pattern
+
+ENVIRONMENT_VARIABLE_KEY_PREFIX = "ENV"
+
+logger = get_logger("ConfigFile")
+
+
+class ConfigFile:
+    def __init__(self, path: Union[Path, str]) -> None:
+        if not isinstance(path, Path):
+            path = Path(path)
+
+        if not path.exists():
+            raise FileNotFoundError(path.absolute())
+
+        self._path = path
+        self._key_values: Dict[str, str] = {}
+
+        self._parse()
+
+    def _parse(self):
+        with open(self._path, encoding="utf-8") as f:
+            lines: str = f.read()
+
+        matches = pattern.findall(lines)
+        for key, *values in matches:
+            try:
+                value = next(v for v in values if v)
+            except StopIteration:
+                raise ParseError(
+                    f"Cannot parse config file ({self._path.absolute()}), error at key `{key}`"
+                )
+
+            already_exists = key in self._key_values
+            if already_exists and not value.startswith(
+                f"{ENVIRONMENT_VARIABLE_KEY_PREFIX}:"
+            ):
+                raise KeyError(
+                    f"Key `{key}` has been already defined and its value is `{self._key_values[key]}`"
+                )
+
+            if value.startswith(f"{ENVIRONMENT_VARIABLE_KEY_PREFIX}:"):
+                _, value = value.split(":")
+                if value not in os.environ:
+                    issue = f'The value of `{key}` contains the "{ENVIRONMENT_VARIABLE_KEY_PREFIX}` prefix but `{value}` is not defined as an environment variable'
+                    if already_exists:
+                        logger.warning(
+                            f"{issue}, using the default value defined above (`{self._key_values[key]}`)"
+                        )
+                        continue
+                    else:
+                        raise KeyError(
+                            f"{issue} and no default value has been provided"
+                        )
+                else:
+                    value = os.environ[value]
+
+            self._key_values[key] = value
+
+    def __getattr__(self, key: str) -> str:
+        if key in self._key_values:
+            return self._key_values[key]
+        raise KeyError(
+            f"Key `{key}` is not found in configuration file ({self._path.absolute()})"
+        )
+
+    __getitem__ = __getattr__
+
+    def __iter__(self) -> Iterable[Tuple[str, str]]:
+        return iter(self._key_values)
+
+    def __len__(self) -> int:
+        return len(self._key_values)
+
+    def keys(self):
+        return self._key_values.keys()
+
+    def values(self):
+        return self._key_values.values()
+
+    def items(self):
+        return self._key_values.items()
+
+    def __repr__(self):
+        return f"{type(self).__name__}(path={self._path}) {self._key_values}"

great_ai/utilities/config_file/parse_error.py

@@ -0,0 +1,2 @@
+class ParseError(Exception):
+    pass

great_ai/utilities/config_file/pattern.py

@@ -0,0 +1,20 @@
+import re
+
+pattern = re.compile(
+    r"""
+        (?:^|\n)        # new key-value pairs must start on a new line
+        \s*             # leading whitespace is allowed
+        (?!\#)          # the key cannot start with a `#` symbol
+        (\w+?)          # then comes the key
+        \s*=\s*         # the key and value are separated by an equal sign
+        (?:             # then comes the value
+              "([^"]*)"   # the value can be surrounded by quotes: "value"
+            | '([^']*)'   # the value can be surrounded by quotes: 'value'
+            | `([^`]*)`   # the value can be surrounded by quotes: `value`
+            | ([^#\n]*?)  # or it is bare, in that case, the trailing whitespace is ignored
+        )
+        \s*(?:\#.*)?    # comments can be added with the `#` symbol
+        (?=\n|$)        # a key-value pairs are separated by new lines
+    """,
+    flags=re.UNICODE | re.VERBOSE,
+)

great_ai/utilities/data/__init__.py

@@ -0,0 +1,6 @@
+from .american_spellings import american_spellings
+from .punctuations import (
+    left_regular_punctuations,
+    right_regular_punctuations,
+    sentence_ending_punctuations,
+)

great_ai/utilities/data/punctuations.py

@@ -0,0 +1,22 @@
+sentence_ending_punctuations = [".", "?", "!", ":", ";"]
+
+# punctuations that usually have no space between them and the character to their left
+left_regular_punctuations = [
+    *sentence_ending_punctuations,
+    ",",
+    "'",
+    "%",
+    ")",
+    "}",
+    "]",
+]
+
+# punctuations that usually have no space between them and the character to their right
+right_regular_punctuations = [
+    "(",
+    "{",
+    "[",
+    "$",
+    "€",
+    "#",
+]

great_ai/utilities/evaluate_ranking/__init__.py

@@ -0,0 +1,2 @@
+from .draw_f1_iso_lines import draw_f1_iso_lines
+from .evaluate_ranking import evaluate_ranking

great_ai/utilities/evaluate_ranking/draw_f1_iso_lines.py

@@ -0,0 +1,28 @@
+from typing import Optional
+
+import matplotlib.pyplot as plt
+import numpy as np
+from matplotlib.axes import Axes
+
+
+def draw_f1_iso_lines(
+    resolution: int = 1000,
+    min: float = 0.2,
+    max: float = 0.8,
+    steps: int = 4,
+    axes: Optional[Axes] = None,
+) -> None:
+    if axes is None:
+        axes = plt.axes()
+
+    for f_score in np.linspace(min, max, num=steps):
+        x = np.linspace(f_score / (2 - f_score), 1, num=resolution)
+        y = f_score * x / (2 * x - f_score)
+
+        axes.plot(x[y >= 0], y[y >= 0], color="gray", alpha=0.2)
+
+        axes.annotate(
+            f"f1={f_score:0.1f}",
+            backgroundcolor="w",
+            xy=(0.9, y[int(resolution * 0.9)] + 0.02),
+        )

great_ai/utilities/evaluate_ranking/evaluate_ranking.py

@@ -0,0 +1,90 @@
+from pathlib import Path
+from typing import Dict, List, Optional, Union
+
+import matplotlib
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.metrics import average_precision_score, precision_recall_curve
+
+from ..unique import unique
+from .draw_f1_iso_lines import draw_f1_iso_lines
+
+
+def evaluate_ranking(
+    expected: List[Union[str, float]],
+    actual_scores: List[float],
+    target_recall: float,
+    title: Optional[str] = "",
+    disable_interpolation: bool = False,
+    axes: Optional[plt.Axes] = None,
+    output_svg: Optional[Path] = None,
+    reverse_order: bool = False,
+    plot: bool = True,
+) -> Dict[Union[str, float], float]:
+    assert 0 <= target_recall <= 1
+
+    if plot and axes is None:
+        fig = plt.figure(figsize=(20, 10))
+        fig.patch.set_facecolor("white")
+        ax = plt.axes()
+    else:
+        ax = axes
+
+    classes = sorted(unique(expected), reverse=reverse_order)
+    str_classes = [str(c) for c in classes]
+
+    with matplotlib.rc_context({"font.size": 20}):
+        if plot:
+            ax.set_xmargin(0)
+
+            draw_f1_iso_lines(axes=ax)
+
+        results: Dict[Union[str, float], float] = {}
+        for i in range(len(classes) - 1):
+            binarized_expected = [
+                (v < classes[i]) if reverse_order else (v > classes[i])
+                for v in expected
+            ]
+            precision, recall, _ = precision_recall_curve(
+                binarized_expected, actual_scores
+            )
+
+            if not disable_interpolation:
+                for j in range(1, len(precision)):
+                    precision[j] = max(precision[j - 1], precision[j])
+
+            closest_recall_index = np.argmin(np.abs(recall - target_recall))
+            precision_at_closest_recall = precision[closest_recall_index]
+            average_precision = average_precision_score(
+                binarized_expected, actual_scores
+            )
+            results[classes[i]] = precision_at_closest_recall
+
+            if plot:
+                ax.plot(
+                    recall,
+                    precision,
+                    label=f"{'|'.join(str_classes[:i + 1])} ↔ {'|'.join(str_classes[i+1:])} (P@{target_recall:.2f}={precision_at_closest_recall:.2f}, AP={average_precision:.2f})",
+                )
+
+        if plot:
+            ax.legend(loc="upper right")
+            ax.axvline(x=target_recall, linestyle="--", color="#55c6bb", linewidth=2.0)
+
+            if title is None:
+                title = "Ranking evaluation"
+
+            ax.set_title(f'{title} ({" < ".join(str_classes)})', pad=20)
+
+            ax.set_xlabel("Recall")
+            ax.set_ylabel("Precision")
+
+            ax.set_xticks([target_recall] + sorted(ax.get_xticks()))
+
+        if plot and output_svg is None:
+            if axes is None:
+                plt.show()
+        elif output_svg:
+            plt.savefig(output_svg, format="svg")
+
+    return results

great_ai/utilities/external/pylatexenc/README.md

@@ -0,0 +1 @@
+https://github.com/phfaist/pylatexenc

great_ai/utilities/external/pylatexenc/__init__.py

@@ -0,0 +1,37 @@
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2015 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+
+"""
+Utilities for LaTeX to/from Unicode Text conversion.
+
+Main Site:
+
+    https://github.com/phfaist/pylatexenc/
+
+"""
+
+from .version import version_str as _version_str
+
+__version__ = _version_str

great_ai/utilities/external/pylatexenc/_util.py

@@ -0,0 +1,161 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2019 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+
+# Internal module. Internal API may move, disappear or otherwise change at any
+# time and without notice.
+
+
+try:
+    # Python >= 3.3
+    from collections.abc import MutableMapping
+except ImportError:
+    from collections import MutableMapping
+
+import bisect
+import warnings
+
+# ------------------------------------------------------------------------------
+
+
+def pylatexenc_deprecated_ver(ver, msg, stacklevel=2):
+    warnings.warn(
+        "Deprecated (pylatexenc {}): {} ".format(ver, msg.strip()),
+        DeprecationWarning,
+        stacklevel=stacklevel + 1,
+    )
+
+
+def pylatexenc_deprecated_2(msg, stacklevel=2):
+    warnings.warn(
+        (
+            "Deprecated (pylatexenc 2.0): {} "
+            "[see https://pylatexenc.readthedocs.io/en/latest/new-in-pylatexenc-2/]"
+        ).format(msg.strip()),
+        DeprecationWarning,
+        stacklevel=stacklevel + 1,
+    )
+
+
+# ------------------------------------------------------------------------------
+
+
+class LazyDict(MutableMapping):
+    r"""
+    A lazy dictionary that loads its data when it is first queried.
+
+    This is used to store the legacy
+    :py:data:`pylatexenc.latexwalker.default_macro_dict` as well as
+    :py:data:`pylatexenc.latex2text.default_macro_dict` etc.  Such that these
+    "dictionaries" are still exposed at the module-level, but the data is loaded
+    only if they are actually queried.
+    """
+
+    def __init__(self, generate_dict_fn):
+        self._full_dict = None
+        self._generate_dict_fn = generate_dict_fn
+
+    def _ensure_instance(self):
+        if self._full_dict is not None:
+            return
+        self._full_dict = self._generate_dict_fn()
+
+    def __getitem__(self, key):
+        self._ensure_instance()
+        return self._full_dict.__getitem__(key)
+
+    def __setitem__(self, key, val):
+        self._ensure_instance()
+        return self._full_dict.__setitem__(key, val)
+
+    def __delitem__(self, key):
+        self._ensure_instance()
+        return self._full_dict.__delitem__(key)
+
+    def __iter__(self):
+        self._ensure_instance()
+        return iter(self._full_dict)
+
+    def __len__(self):
+        self._ensure_instance()
+        return len(self._full_dict)
+
+    def copy(self):
+        self._ensure_instance()
+        return self._full_dict.copy()
+
+    def clear(self):
+        self._ensure_instance()
+        return self._full_dict.clear()
+
+
+# ------------------------------------------------------------------------------
+
+
+class LineNumbersCalculator(object):
+    r"""
+    Utility to calculate line numbers.
+    """
+
+    def __init__(self, s):
+        super(LineNumbersCalculator, self).__init__()
+
+        def find_all_new_lines(x):
+            # first line starts at the beginning of the string
+            yield 0
+            k = 0
+            while k < len(x):
+                k = x.find("\n", k)
+                if k == -1:
+                    return
+                k += 1
+                # s[k] is the character after the newline, i.e., the 0-th column
+                # of the new line
+                yield k
+
+        self._pos_new_lines = list(find_all_new_lines(s))
+
+    def pos_to_lineno_colno(self, pos, as_dict=False):
+        r"""
+        Return the line and column number corresponding to the given `pos`.
+
+        Return a tuple `(lineno, colno)` giving line number and column number.
+        Line numbers start at 1 and column number start at zero, i.e., the
+        beginning of the document (`pos=0`) has line and column number `(1,0)`.
+        If `as_dict=True`, then a dictionary with keys 'lineno', 'colno' is
+        returned instead of a tuple.
+        """
+
+        # find line number in list
+
+        # line_no is the index of the last item in self._pos_new_lines that is <= pos.
+        line_no = bisect.bisect_right(self._pos_new_lines, pos) - 1
+        assert line_no >= 0 and line_no < len(self._pos_new_lines)
+
+        col_no = pos - self._pos_new_lines[line_no]
+        # 1+... so that line and column numbers start at 1
+        if as_dict:
+            return {"lineno": 1 + line_no, "colno": col_no}
+        return (1 + line_no, col_no)

great_ai/utilities/external/pylatexenc/latex2text/__main__.py

@@ -0,0 +1,325 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2018 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+import argparse
+import fileinput
+import logging
+import sys
+
+from .. import latexwalker
+from ..latex2text import LatexNodes2Text, _strict_latex_spaces_predef
+from ..version import version_str
+
+
+def main(argv=None):
+
+    if argv is None:
+        argv = sys.argv[1:]
+
+    parser = argparse.ArgumentParser(prog="latex2text", add_help=False)
+
+    codegroup = parser.add_argument_group("Input options")
+
+    codegroup.add_argument(
+        "--code",
+        "-c",
+        action="store",
+        default=None,
+        metavar="LATEX_CODE",
+        help="Convert the given LATEX_CODE to unicode text instead of reading "
+        "from FILE or standard input.  You cannot specify FILEs if you use this "
+        "option, and any standard input is ignored.",
+    )
+
+    codegroup.add_argument(
+        "files",
+        metavar="FILE",
+        nargs="*",
+        help="Input files to read LaTeX code from. If no FILE(s) is/are specified, "
+        "LaTeX code is read from standard input unless --code is specified",
+    )
+
+    group = parser.add_argument_group("LatexWalker options")
+
+    group.add_argument(
+        "--parser-keep-inline-math",
+        action="store_const",
+        const=True,
+        dest="parser_keep_inline_math",
+        default=None,
+        help=argparse.SUPPRESS,
+    )
+    group.add_argument(
+        "--no-parser-keep-inline-math",
+        action="store_const",
+        const=False,
+        dest="parser_keep_inline_math",
+        help=argparse.SUPPRESS,
+    )
+
+    group.add_argument(
+        "--tolerant-parsing",
+        action="store_const",
+        const=True,
+        dest="tolerant_parsing",
+        default=True,
+    )
+    group.add_argument(
+        "--no-tolerant-parsing",
+        action="store_const",
+        const=False,
+        dest="tolerant_parsing",
+        help="Tolerate syntax errors when parsing, and attempt to continue (default yes)",
+    )
+
+    # I'm not sure this flag is useful and if it should be exposed at all.
+    # Accept it, but make it hidden.
+    parser.add_argument(
+        "--strict-braces",
+        action="store_const",
+        const=True,
+        dest="strict_braces",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
+    parser.add_argument(
+        "--no-strict-braces",
+        action="store_const",
+        const=False,
+        dest="strict_braces",
+        # help="Report errors for mismatching LaTeX braces (default no)"
+        help=argparse.SUPPRESS,
+    )
+
+    group = parser.add_argument_group("LatexNodes2Text options")
+
+    group.add_argument(
+        "--text-keep-inline-math",
+        action="store_const",
+        const=True,
+        dest="text_keep_inline_math",
+        default=None,
+        help=argparse.SUPPRESS,
+    )
+    group.add_argument(
+        "--no-text-keep-inline-math",
+        action="store_const",
+        const=False,
+        dest="text_keep_inline_math",
+        help=argparse.SUPPRESS,
+    )
+
+    group.add_argument(
+        "--math-mode",
+        action="store",
+        dest="math_mode",
+        choices=["text", "with-delimiters", "verbatim", "remove"],
+        default="text",
+        help="How to handle chunks of math mode LaTeX code. 'text' = convert "
+        "to text like the rest; 'with-delimiters' = same as 'text' but retain "
+        "the original math mode delimiters; 'verbatim' = keep verbatim LaTeX code; "
+        "'remove' = remove from input entirely",
+    )
+
+    group.add_argument(
+        "--fill-text",
+        dest="fill_text",
+        action="store",
+        nargs="?",
+        default=-1,
+        help="Attempt to wrap text to the given width, or 80 columns if option is "
+        "specified with no argument",
+    )
+
+    group.add_argument(
+        "--keep-comments",
+        action="store_const",
+        const=True,
+        dest="keep_comments",
+        default=False,
+    )
+    group.add_argument(
+        "--no-keep-comments",
+        action="store_const",
+        const=False,
+        dest="keep_comments",
+        help="Keep LaTeX comments in text output (default no)",
+    )
+
+    class ListWithHiddenItems(list):
+        def __init__(self, thelist, hiddenitems):
+            super(ListWithHiddenItems, self).__init__(thelist)
+            self.hiddenitems = hiddenitems
+
+        def __contains__(self, value):
+            return (
+                super(ListWithHiddenItems, self).__contains__(value)
+                or value in self.hiddenitems
+            )
+
+    strict_latex_spaces_choices = ListWithHiddenItems(
+        # the list
+        ["off", "on"]
+        + list(k for k in _strict_latex_spaces_predef.keys() if k != "default"),
+        # hidden items: Value is accepted, but not shown in list of choices
+        ["default"],
+    )
+    group.add_argument(
+        "--strict-latex-spaces",
+        choices=strict_latex_spaces_choices,
+        dest="strict_latex_spaces",
+        default="macros",
+        help="How to handle whitespace. See documentation for the class "
+        "LatexNodes2Text().",
+    )
+
+    group.add_argument(
+        "--keep-braced-groups",
+        action="store_const",
+        const=True,
+        dest="keep_braced_groups",
+        default=False,
+    )
+    group.add_argument(
+        "--no-keep-braced-groups",
+        action="store_const",
+        const=False,
+        dest="keep_braced_groups",
+        help="Keep LaTeX {braced groups} in text output (default no)",
+    )
+
+    group.add_argument(
+        "--keep-braced-groups-minlen",
+        type=int,
+        default=2,
+        dest="keep_braced_groups_minlen",
+        help="Only apply --keep-braced-groups to groups that contain at least "
+        "this many characters",
+    )
+
+    group = parser.add_argument_group("General options")
+
+    group.add_argument(
+        "-q",
+        "--quiet",
+        dest="logging_level",
+        action="store_const",
+        const=logging.ERROR,
+        default=logging.INFO,
+        help="Suppress warning messages",
+    )
+    group.add_argument(
+        "-v",
+        "--verbose",
+        dest="logging_level",
+        action="store_const",
+        const=logging.DEBUG,
+        help="Verbose output",
+    )
+    group.add_argument(
+        "--version",
+        action="version",
+        version="pylatexenc {}".format(version_str),
+        help="Show version information and exit",
+    )
+    group.add_argument(
+        "--help", action="help", help="Show this help information and exit"
+    )
+
+    args = parser.parse_args(argv)
+
+    logging.basicConfig()
+    logging.getLogger().setLevel(args.logging_level)
+    logger = logging.getLogger(__name__)
+
+    if (
+        args.parser_keep_inline_math is not None
+        or args.text_keep_inline_math is not None
+    ):
+        logger.warning(
+            "Options --parser-keep-inline-math and --text-keep-inline-math are "
+            "deprecated and no longer have any effect.  Please use "
+            "--math-mode=... instead."
+        )
+
+    latex = ""
+    if args.code:
+        if args.files:
+            logger.error(
+                "Cannot specify both FILEs and --code option. "
+                "Use --help option for more information."
+            )
+            sys.exit(1)
+        latex = args.code
+    else:
+        for line in fileinput.input(files=args.files):
+            latex += line
+
+    if args.fill_text != -1:
+        if args.fill_text is not None and len(args.fill_text):
+            fill_text = int(args.fill_text)
+        else:
+            fill_text = True
+    else:
+        fill_text = None
+
+    lw = latexwalker.LatexWalker(
+        latex, tolerant_parsing=args.tolerant_parsing, strict_braces=args.strict_braces
+    )
+
+    (nodelist, pos, len_) = lw.get_latex_nodes()
+
+    ln2t = LatexNodes2Text(
+        math_mode=args.math_mode,
+        keep_comments=args.keep_comments,
+        strict_latex_spaces=args.strict_latex_spaces,
+        keep_braced_groups=args.keep_braced_groups,
+        keep_braced_groups_minlen=args.keep_braced_groups_minlen,
+        fill_text=fill_text,
+    )
+
+    print(ln2t.nodelist_to_text(nodelist))
+
+
+def run_main():
+
+    try:
+
+        main()
+
+    except SystemExit:
+        raise
+    except:  # lgtm [py/catch-base-exception]
+        import pdb
+        import traceback
+
+        traceback.print_exc()
+        pdb.post_mortem()
+
+
+if __name__ == "__main__":
+
+    main()
+    # run_main() # debug

great_ai/utilities/external/pylatexenc/latexencode/__init__.py

@@ -0,0 +1,325 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2018 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+r"""
+The `latexencode` module provides a set of routines that allows you to
+convert a unicode string to LaTeX escape sequences.
+
+For basic usage you can use the :py:func:`unicode_to_latex()` function
+directly::
+
+  >>> print(unicode_to_latex('À votre santé'))
+  \`A votre sant\'e
+  >>> print(unicode_to_latex('The length of samples #3 & #4 is 3μm'))
+  The length of samples \#3 \& \#4 is 3\ensuremath{\mu}m
+
+The conversion is handled by the class :py:class:`UnicodeToLatexEncoder`.  If
+you are converting multiple strings, you may create an instance with the flags
+you like and invoke its method
+:py:meth:`~UnicodeToLatexEncoder.unicode_to_latex()` as many times as necessary::
+
+  >>> u = UnicodeToLatexEncoder(unknown_char_policy='replace')
+  >>> print(u.unicode_to_latex('À votre santé'))
+  \`A votre sant\'e
+  >>> print(u.unicode_to_latex('The length of samples #3 & #4 is 3μm'))
+  The length of samples \#3 \& \#4 is 3\ensuremath{\mu}m
+  >>> print(u.unicode_to_latex('À votre santé: 乾杯'))
+  \`A votre sant\'e: {\bfseries ?}{\bfseries ?}
+
+Example using custom conversion rules::
+  >>> import re
+  >>> u = UnicodeToLatexEncoder(
+  ...     conversion_rules=[
+  ...         UnicodeToLatexConversionRule(rule_type=RULE_REGEX, rule=[
+  ...             (re.compile(r'-->'), r'\\textrightarrow'),
+  ...             (re.compile(r'<--'), r'\\textleftarrow'),
+  ...         ]),
+  ...         'defaults'
+  ...     ]
+  ... )
+  >>> print(u.unicode_to_latex("Cheers --> À votre santé"))
+  Cheers {\textrightarrow} \`A votre sant\'e
+
+See :py:class:`UnicodeToLatexEncoder` and
+:py:class:`UnicodeToLatexConversionRule`.  Note for regex rules, the replacement
+text is expanded like the second argument of `re.sub()` and backslashes need to
+be escaped even inside raw strings.
+
+.. versionadded:: 2.0
+
+   The class :py:class:`UnicodeToLatexEncoder` along with its helper functions
+   and classes were introduced in `pylatexenc 2.0`.
+
+   The earlier function :py:func:`utf8tolatex()` that was available in
+   `pylatexenc 1.x` is still provided unchanged, so code written for `pylatexenc
+   1.x` should work without changes.  New code is however strongly encouraged to
+   employ the new API.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import functools
+import itertools
+import logging
+import sys
+import unicodedata
+
+if sys.version_info.major > 2:
+    unicode = str  # need to support unicode() w/ no arguments
+    basestring = str
+    # use MappingProxyType for keeping
+    # inspect function argument names
+    from inspect import getfullargspec
+    from types import MappingProxyType as _MappingProxyType
+else:
+    _MappingProxyType = dict
+    # inspect function argument names -- simulate getfullargspec with getargspec (argh...)
+    from inspect import getargspec as getfullargspec
+
+logger = logging.getLogger(__name__)
+
+
+from .. import _util
+from ._partial_latex_encoder import PartialLatexToLatexEncoder
+from ._unicode_to_latex_encoder import (
+    RULE_CALLABLE,
+    RULE_DICT,
+    RULE_REGEX,
+    UnicodeToLatexConversionRule,
+    UnicodeToLatexEncoder,
+    get_builtin_conversion_rules,
+    get_builtin_uni2latex_dict,
+)
+
+# ------------------------------------------------
+
+
+# ------------------------------------------------
+
+
+# ------------------------------------------------
+
+
+_u2l_obj_cache = {}
+
+
+def unicode_to_latex(
+    s,
+    non_ascii_only=False,
+    replacement_latex_protection="braces",
+    unknown_char_policy="keep",
+    unknown_char_warning=True,
+):
+    r"""
+    Shorthand for constructing a :py:class:`UnicodeToLatexEncoder` instance and
+    calling its :py:meth:`~UnicodeToLatexEncoder.unicode_to_latex()` method.
+
+    The :py:class:`UnicodeToLatexEncoder` instances for given option settings
+    are cached, making repeated calls to :py:func:`unicode_to_latex()` possible
+    without creating a new instance upon each call.
+
+    The parameters `non_ascii_only`, `replacement_latex_protection`,
+    `unknown_char_policy`, and `unknown_char_warning` are directly passed on to
+    the :py:class:`UnicodeToLatexEncoder` constructor.  See the class doc for
+    :py:class:`UnicodeToLatexEncoder` for more information about what they do.
+
+    You may only use arguments to this function that are python hashable (like
+    `True`, `False`, or simple strings) to help us keep a cache of previously
+    constructed :py:class:`UnicodeToLatexEncoder` instances.  For instance, it
+    is not possible to provide a callable to `unknown_char_policy`.  It is also
+    not possible to specify custom conversion rules with this helper function.
+    If you need any of these features, simply create a
+    :py:class:`UnicodeToLatexEncoder` instance directly.
+    """
+
+    key = (
+        non_ascii_only,
+        replacement_latex_protection,
+        unknown_char_policy,
+        unknown_char_warning,
+    )
+
+    if key in _u2l_obj_cache:
+        u = _u2l_obj_cache[key]
+    else:
+        u = UnicodeToLatexEncoder(
+            non_ascii_only=non_ascii_only,
+            replacement_latex_protection=replacement_latex_protection,
+            unknown_char_policy=unknown_char_policy,
+            unknown_char_warning=unknown_char_warning,
+        )
+        _u2l_obj_cache[key] = u
+
+    return u.unicode_to_latex(s)
+
+
+# ------------------------------------------------------------------------------
+
+# Don't change pylatexenc 1.x function:
+
+
+def _get_deprecated_utf82latex():
+    #
+    # Don't issue a deprecation warning, because utf8tolatex() uses the
+    # `utf82latex` dict even if it isn't modified by the user.
+    #
+    #     _util.pylatexenc_deprecated_2(
+    #         "The module-level dictionary `pylatexenc.latexencode.utf82latex` is deprecated "
+    #         "and might be removed in a future version of `pylatexenc`.",
+    #     )
+
+    # return a copy of the dict so that the user can modify the module-level
+    # `utf82latex` dict without influencing the behavior of the new
+    # `unicode_to_latex()` routines. (E.g., if two python modules use
+    # pylatexenc.latexencode, we don't want one python module's use of
+    # `utf2tolatex()` to influence the behavior of another module's use of
+    # `unicode_to_latex()`.  If both modules use `utf8tolatex()`, we can't avoid
+    # this influence.)
+    from ._uni2latexmap import uni2latex as _uni2latex
+
+    return _uni2latex.copy()
+
+
+utf82latex = _util.LazyDict(generate_dict_fn=_get_deprecated_utf82latex)
+"""
+.. deprecated:: 2.0
+
+   Pylatexenc 1.x exposed the module-level dictionary `utf82latex` that could be
+   modified to alter the behavior of `utf8tolatex()`.
+
+   If you would like to obtain a copy of the built-in unicode to text
+   dictionary, see :py:func:`get_builtin_uni2latex_dict()`.  If you would like
+   to alter the behavior of :py:func:`utf8tolatex()`, you should use
+   :py:class:`UnicodeToLatexEncoder` which provides a rich interface for
+   specifying rules how to convert chars to LaTeX escapes.
+
+   For backwards compatibility, you can still modify the module-level dictionary
+   `utf82latex` (but you can't assign a new object to it) and this will directly
+   modify the global built-in dictionary of known latex escapes.  This is not
+   recommended however, and the `utf82latex` module-level dictionary might be
+   removed in the future.
+
+   .. warning::
+
+      Modifying the `utf82latex` module-level dictionary is not recommended.
+      Doing so will alter the behavior of the `utf8tolatex()` function also for
+      all other modules that also use `pylatexenc`!
+"""
+
+
+def utf8tolatex(
+    s,
+    non_ascii_only=False,
+    brackets=True,
+    substitute_bad_chars=False,
+    fail_bad_chars=False,
+):
+    """
+    .. note::
+
+       Since `pylatexenc 2.0`, it is recommended to use the the
+       :py:func:`unicode_to_latex()` function or the
+       :py:class:`UnicodeToLatexEncoder` class instead of the earlier function
+       `utf8tolatex()`.
+
+       The new routines provide much more flexibility and versatility.  For
+       instance, you can specify custom escape sequences for certain characters.
+       Some cheap benchmarks seem to indicate that the new routines are not
+       significantly slower than the `utf8tolatex()` function.  Also, the name
+       `utf8tolatex()` was poorly chosen, since the argument is in fact not
+       'utf-8'-encoded but rather a Python unicode string object.
+
+       The function `utf8tolatex()` is still provided unchanged from `pylatexenc
+       1.x`.  We do not plan to remove this function in the near future so it is
+       not (yet) considered as deprecated and we will continue to provide it in
+       near future versions of `pylatexenc`.  Bug reports, improvements, and new
+       features will however be directed to :py:func:`UnicodeToLatexEncoder()`.
+
+    Encode a UTF-8 string to a LaTeX snippet.
+
+    If `non_ascii_only` is set to `True`, then usual (ascii) characters such as ``#``,
+    ``{``, ``}`` etc. will not be escaped.  If set to `False` (the default), they are
+    escaped to their respective LaTeX escape sequences.
+
+    If `brackets` is set to `True` (the default), then LaTeX macros are enclosed in
+    brackets.  For example, ``sant\N{LATIN SMALL LETTER E WITH ACUTE}`` is replaced by
+    ``sant{\\'e}`` if `brackets=True` and by ``sant\\'e`` if `brackets=False`.
+
+    .. warning::
+        Using `brackets=False` might give you an invalid LaTeX string, so avoid
+        it! (for instance, ``ma\N{LATIN SMALL LETTER I WITH CIRCUMFLEX}tre`` will be
+        replaced incorrectly by ``ma\\^\\itre`` resulting in an unknown macro ``\\itre``).
+
+    If `substitute_bad_chars=True`, then any non-ascii character for which no LaTeX escape
+    sequence is known is replaced by a question mark in boldface. Otherwise (by default),
+    the character is left as it is.
+
+    If `fail_bad_chars=True`, then a `ValueError` is raised if we cannot find a
+    character substitution for any non-ascii character.
+
+    .. versionchanged:: 1.3
+
+        Added `fail_bad_chars` switch
+    """
+
+    s = unicode(s)  # make sure s is unicode
+    s = unicodedata.normalize("NFC", s)
+
+    if not s:
+        return ""
+
+    result = ""
+    for ch in s:
+        # logger.longdebug("Encoding char %r", ch)
+        if non_ascii_only and ord(ch) < 127:
+            result += ch
+        else:
+            # use the `utf82latex` dict -- not `_uni2latex` which should NOT be
+            # modified externally even for backwards-compatible code
+            lch = utf82latex.get(ord(ch), None)
+            if lch is not None:
+                # add brackets if needed, i.e. if we have a substituting macro.
+                # note: in condition, beware, that lch might be of zero length.
+                result += "{" + lch + "}" if brackets and lch[0:1] == "\\" else lch
+            elif (ord(ch) >= 32 and ord(ch) <= 127) or (ch in "\n\r\t"):
+                # ordinary printable ascii char, just add it
+                result += ch
+            else:
+                # non-ascii char
+                msg = "Character cannot be encoded into LaTeX: U+%04X - `%s'" % (
+                    ord(ch),
+                    ch,
+                )
+                if fail_bad_chars:
+                    raise ValueError(msg)
+
+                logger.warning(msg)
+                if substitute_bad_chars:
+                    result += r"{\bfseries ?}"
+                else:
+                    # keep unescaped char
+                    result += ch
+
+    return result

great_ai/utilities/external/pylatexenc/latexencode/__main__.py

@@ -0,0 +1,148 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2019 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+
+import argparse
+import fileinput
+import logging
+import sys
+
+from ..latexencode import unicode_to_latex
+from ..version import version_str
+
+
+def main(argv=None):
+
+    if argv is None:
+        argv = sys.argv[1:]
+
+    parser = argparse.ArgumentParser(prog="latexencode", add_help=False)
+    parser.add_argument(
+        "files",
+        metavar="FILE",
+        nargs="*",
+        help="Input files (if none specified, read from stdandard input)",
+    )
+
+    parser.add_argument(
+        "--non-ascii-only",
+        action="store_const",
+        const=True,
+        dest="non_ascii_only",
+        default=False,
+    )
+    parser.add_argument(
+        "--no-non-ascii-only",
+        action="store_const",
+        const=False,
+        dest="non_ascii_only",
+        help="The option --non-ascii-only specifies that only non-ascii characters "
+        "are to be encoded into LaTeX sequences, and not characters like '$' "
+        "even though they might have a special LaTeX meaning.",
+    )
+
+    parser.add_argument(
+        "--replacement-latex-protection",
+        choices=(
+            "braces",
+            "braces-all",
+            "braces-almost-all",
+            "braces-after-macro",
+            "none",
+        ),
+        dest="replacement_latex_protection",
+        default="braces",
+        help=r"How to protect replacement latex code from producing invalid latex code "
+        r"when concatenated in a longer string.  One of 'braces', 'braces-all', "
+        r"'braces-almost-all', 'braces-after-macro', 'none'.  Example: using "
+        r"choice 'braces' we avoid the invalid replacement 'a→b' -> 'a\tob' "
+        r"with instead 'a{\to}b'.",
+    )
+
+    parser.add_argument(
+        "--unknown-char-policy",
+        choices=("keep", "replace", "ignore", "fail"),
+        dest="unknown_char_policy",
+        default="keep",
+        help="How to deal with nonascii characters with no known latex code equivalent.",
+    )
+
+    parser.add_argument(
+        "-q",
+        "--quiet",
+        dest="logging_level",
+        action="store_const",
+        const=logging.ERROR,
+        default=logging.INFO,
+        help="Suppress warning messages",
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version="pylatexenc {}".format(version_str),
+        help="Show version information and exit",
+    )
+    parser.add_argument(
+        "--help", action="help", help="Show this help information and exit"
+    )
+
+    args = parser.parse_args(argv)
+
+    logging.basicConfig()
+    logging.getLogger().setLevel(args.logging_level)
+
+    latex = ""
+    for line in fileinput.input(files=args.files):
+        latex += line
+
+    result = unicode_to_latex(
+        latex,
+        non_ascii_only=args.non_ascii_only,
+        replacement_latex_protection=args.replacement_latex_protection,
+        unknown_char_policy=args.unknown_char_policy,
+    )
+
+    sys.stdout.write(result)
+
+
+def run_main():
+    try:
+
+        main()
+
+    except SystemExit:
+        raise
+    except:  # lgtm [py/catch-base-exception]
+        import pdb
+        import traceback
+
+        traceback.print_exc()
+        pdb.post_mortem()
+
+
+if __name__ == "__main__":
+
+    # run_main()  ## DEBUG
+    main()

great_ai/utilities/external/pylatexenc/latexencode/_partial_latex_encoder.py

@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2021 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+# import sys
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+from ._unicode_to_latex_encoder import (
+    RULE_CALLABLE,
+    UnicodeToLatexConversionRule,
+    UnicodeToLatexEncoder,
+)
+
+# if sys.version_info.major == 2:
+#     bytes = str
+#     str = unicode
+
+
+class PartialLatexToLatexEncoder(UnicodeToLatexEncoder):
+    r"""
+    Encode a string while preserving some (fuzzily detected) LaTeX constructs
+    that the input string already has (e.g. accent macros or inline math modes).
+
+    Sometimes you need to fully LaTeX-encode a string that already has some
+    LaTeX constructs.  For instance, titles of bibliographic entries might
+    include some inline math or accents, but they might also include unicode
+    characters that need to be encoded.  Using a
+    :py:class:`UnicodeToLatexEncoder` on such strings would result in ugly
+    doubly-escaped strings such as ``\textbackslash{}'\{e\}``.  Instead,
+    constructs such as ``\'{e}`` should be preserved while other characters
+    and/or constructs (say '&' or '%') as well as unicode characters should be
+    encoded.
+
+    This class offers a simple partial solution: Characters are encoded as per
+    the given `conversion_rules` (or the default conversion rules of
+    :py:class:`UnicodeToLatexEncoder` objects), except that the characters in
+    `keep_latex_chars` are to be interpreted as LaTeX and are not to be further
+    encoded.
+
+    .. versionadded: 2.10
+    """
+
+    def __init__(
+        self,
+        # keyword arguments:
+        keep_latex_chars=r"\${}^_",
+        conversion_rules=None,
+        **kwargs
+    ):
+
+        base_conversion_rules = conversion_rules
+        if base_conversion_rules is None:
+            base_conversion_rules = ["defaults"]
+
+        super(PartialLatexToLatexEncoder, self).__init__(
+            # only a single rule, our own special method that tries to parse
+            # partial latex.
+            conversion_rules=[
+                UnicodeToLatexConversionRule(
+                    rule_type=RULE_CALLABLE,
+                    rule=self._do_partial_latex_encode_step,
+                    replacement_latex_protection="none",
+                )
+            ]
+            + base_conversion_rules,
+            **kwargs
+        )
+
+        self.keep_latex_chars = keep_latex_chars
+
+    def _do_partial_latex_encode_step(self, s, pos):
+        r"""
+        This method is used as a "callable rule" for the
+        :py:class:`UnicodeToLatexEncoder` object.
+
+        The strategy is to see if we have something that looks like a LaTeX char
+        we want to keep.  If so, keep it as is; if not, return `None` so that
+        further rules can be considered by the base unicode encoder.
+        """
+
+        from ..latexwalker import LatexWalker
+
+        if s[pos] in self.keep_latex_chars:
+            # Read a token and if it is a macro, keep the full macro!
+            lw = LatexWalker(s, tolerant_parsing=False)
+
+            tok = lw.get_token(pos, environments=False)
+
+            tok_as_latex = tok.pre_space + s[tok.pos : tok.pos + tok.len]
+
+            # keep the LaTeX token as-is
+            return (tok.pos + tok.len - pos, tok_as_latex)
+
+        return None