Move files

src/great_ai/__init__.py

@@ -1,3 +1,23 @@
-from .great_ai import *
-from .large_file import *
-from .utilities import *
+from .context import configure
+from .deploy import GreatAI
+from .exceptions import (
+    ArgumentValidationError,
+    MissingArgumentError,
+    WrongDecoratorOrderError,
+)
+from .models import save_model, use_model
+from .output_views import (
+    ClassificationOutput,
+    MultiLabelClassificationOutput,
+    RegressionOutput,
+)
+from .parameters import log_metric, parameter
+from .persistence import MongodbDriver, ParallelTinyDbDriver, TracingDatabaseDriver
+from .remote import (
+    HttpClient,
+    RemoteCallError,
+    call_remote_great_ai,
+    call_remote_great_ai_async,
+)
+from .tracing import add_ground_truth, delete_ground_truth, query_ground_truth
+from .views import Trace

src/great_ai/__main__.py

@@ -1,208 +0,0 @@
-#!/usr/bin/env python3
-
-import logging
-import re
-from importlib import import_module, reload
-from pathlib import Path
-from threading import Event
-from typing import Optional
-
-import uvicorn
-from uvicorn._subprocess import get_subprocess
-from uvicorn.config import LOGGING_CONFIG, Config
-from uvicorn.supervisors.basereload import BaseReload
-from watchdog.events import FileSystemEvent, PatternMatchingEventHandler
-from watchdog.observers import Observer
-
-from .great_ai.constants import SERVER_NAME
-from .great_ai.context import _is_in_production_mode
-from .great_ai.deploy import GreatAI
-from .great_ai.exceptions import ArgumentValidationError, MissingArgumentError
-from .parse_arguments import parse_arguments
-from .utilities import get_logger
-
-logger = get_logger(SERVER_NAME)
-
-
-GREAT_AI_LOGGING_CONFIG = {
-    **LOGGING_CONFIG,
-    "formatters": {
-        "default": {
-            "()": "great_ai.logger.CustomFormatter",
-            "fmt": "%(asctime)s | %(levelname)8s | %(message)s",
-        },
-        "access": {
-            "()": "great_ai.logger.CustomFormatter",
-            "fmt": "%(asctime)s | %(levelname)8s | %(message)s",  # noqa: E501
-        },
-    },
-}
-
-
-def main() -> None:
-    args = parse_arguments()
-    should_auto_reload = not _is_in_production_mode(logger=None)
-
-    if args.worker_count > 1 and should_auto_reload:
-        raise ArgumentValidationError(
-            "Cannot use auto-reload with multiple worker_count: set the `--worker_count=1` CLI argument,"
-            + "or set the ENVIRONMENT environment variable to `production`."
-        )
-
-    common_config = dict(
-        host=args.host,
-        port=args.port,
-        timeout_keep_alive=args.timeout_keep_alive,
-        workers=args.worker_count,
-        server_header=False,
-        reload=False,
-        log_config=GREAT_AI_LOGGING_CONFIG,
-    )
-
-    if not should_auto_reload:
-        file_name = get_script_name(args.file_name)
-        app = find_app(file_name)
-
-        logger.info(f"Starting uvicorn server with app={app}")
-
-        config = Config(app, **common_config)
-        socket = config.bind_socket()
-        server = GreatAIReload(
-            config, target=uvicorn.Server(config=config).run, sockets=[socket]
-        )
-
-        server.startup()
-        try:
-            Event().wait()
-        finally:
-            server.shutdown()
-            if args.file_name.endswith(".ipynb"):
-                Path(get_script_name_of_notebook(args.file_name)).unlink(
-                    missing_ok=True
-                )
-    else:
-
-        class EventHandler(PatternMatchingEventHandler):
-            def __init__(self) -> None:
-                super().__init__(
-                    patterns=["*.py", "*.ipynb"], ignore_patterns=["__*.py"]
-                )
-                self.server: Optional[GreatAIReload] = None
-                self.restart()
-
-            def on_closed(self, event: FileSystemEvent) -> None:
-                logger.warning(f"File {event.src_path} has triggered a restart")
-                self.restart()
-
-            def restart(self) -> None:
-                file_name = get_script_name(args.file_name)
-                app = find_app(file_name)
-                if app is None:
-                    logger.warning("Auto-reloading skipped")
-                    return
-
-                self.stop_server()
-
-                config = Config(app, **common_config)
-                socket = config.bind_socket()
-                self.server = GreatAIReload(
-                    config, target=uvicorn.Server(config=config).run, sockets=[socket]
-                )
-                self.server.startup()
-
-            def stop_server(self) -> None:
-                if self.server:
-                    self.server.shutdown()
-
-        restart_handler = EventHandler()
-        observer = Observer()
-        observer.schedule(restart_handler, path=".", recursive=True)
-        observer.start()
-
-        try:
-            Event().wait()
-        finally:
-            observer.stop()
-            restart_handler.stop_server()
-            if args.file_name.endswith(".ipynb"):
-                Path(get_script_name_of_notebook(args.file_name)).unlink(
-                    missing_ok=True
-                )
-            observer.join()
-
-
-def get_script_name(file_name_argument: str) -> str:
-    if file_name_argument.endswith(".ipynb"):
-        logger.info("Converting notebook to Python script")
-        from nbconvert import PythonExporter
-
-        exporter = PythonExporter()
-        content, _ = exporter.from_filename(file_name_argument)
-        file_name_argument = get_script_name_of_notebook(file_name_argument)
-
-        with open(file_name_argument, "w", encoding="utf-8") as f:
-            f.write(content)
-
-    return re.sub(r"\.(py|ipynb)$", "", file_name_argument)
-
-
-def get_script_name_of_notebook(notebook_name: str) -> str:
-    base_name = re.sub(r"\.ipynb$", "", notebook_name)
-    return f"__{base_name}__.py"
-
-
-module = None
-
-
-def find_app(file_name: str) -> Optional[str]:
-    global module
-
-    logging.disable(logging.CRITICAL)
-    try:
-        if module is None:
-            module = import_module(file_name)
-        else:
-            module = reload(module)
-    except Exception:
-        logging.disable(logging.NOTSET)
-        logger.exception("Could not load file because of an exception: fix your code")
-        return None
-    finally:
-        logging.disable(logging.NOTSET)
-
-    for name, value in module.__dict__.items():
-        if isinstance(value, GreatAI):
-            app_name = name
-
-    if app_name:
-        logger.info(f"Found `{app_name}` to be the GreatAI app ")
-    else:
-        raise MissingArgumentError(
-            "GreatAI app could not be found, make sure to use `@GreatAI.deploy` on your prediction function"
-        )
-
-    return f"{file_name}:{app_name}.app"
-
-
-class GreatAIReload(BaseReload):
-    def startup(self) -> None:
-        self.process = get_subprocess(
-            config=self.config, target=self.target, sockets=self.sockets
-        )
-        self.process.start()
-
-    def shutdown(self) -> None:
-        self.process.terminate()
-        self.process.join()
-
-        for sock in self.sockets:
-            sock.close()
-
-
-if __name__ == "__main__":
-    try:
-        main()
-    except KeyboardInterrupt:
-        exit()
-    except Exception as e:
-        logger.error(e)

src/great_ai/constants.py

@@ -1,4 +1,5 @@
-from ..large_file import LargeFileMongo, LargeFileS3
+from large_file import LargeFileMongo, LargeFileS3
+
 from .persistence.mongodb_driver import MongodbDriver
 
 ENV_VAR_KEY = "ENVIRONMENT"

src/great_ai/context.py

@@ -6,8 +6,9 @@ from typing import Any, Dict, Optional, Type, cast
 
 from pydantic import BaseModel
 
-from ..large_file import LargeFile, LargeFileLocal
-from ..utilities import get_logger
+from large_file import LargeFileBase, LargeFileLocal
+from utilities import get_logger
+
 from .constants import (
     DEFAULT_LARGE_FILE_CONFIG_PATHS,
     DEFAULT_TRACING_DATABASE_CONFIG_PATHS,
@@ -21,7 +22,7 @@ from .persistence import ParallelTinyDbDriver, TracingDatabaseDriver
 
 class Context(BaseModel):
     tracing_database: TracingDatabaseDriver
-    large_file_implementation: Type[LargeFile]
+    large_file_implementation: Type[LargeFileBase]
     is_production: bool
     logger: Logger
     should_log_exception_stack: bool
@@ -57,7 +58,7 @@ def configure(
     log_level: int = DEBUG,
     seed: int = 42,
     tracing_database: Optional[Type[TracingDatabaseDriver]] = None,
-    large_file_implementation: Optional[Type[LargeFile]] = None,
+    large_file_implementation: Optional[Type[LargeFileBase]] = None,
     should_log_exception_stack: Optional[bool] = None,
     prediction_cache_size: int = 512,
     disable_se4ml_banner: bool = False,
@@ -160,8 +161,8 @@ def _initialize_tracing_database(
 
 
 def _initialize_large_file(
-    selected: Optional[Type[LargeFile]], logger: Logger
-) -> Type[LargeFile]:
+    selected: Optional[Type[LargeFileBase]], logger: Logger
+) -> Type[LargeFileBase]:
     for large_file, paths in DEFAULT_LARGE_FILE_CONFIG_PATHS.items():
         if selected is None or selected == large_file:
             if large_file.initialized:

src/great_ai/deploy/great_ai.py

@@ -16,7 +16,8 @@ from typing import (
 from fastapi import APIRouter, FastAPI, status
 from pydantic import BaseModel, create_model
 
-from ...utilities import parallel_map
+from utilities import parallel_map
+
 from ..constants import DASHBOARD_PATH
 from ..context import get_context
 from ..helper import (

src/great_ai/deploy/routes/dashboard/create_dash_app.py

@@ -8,7 +8,8 @@ from dash import Dash, dcc, html
 from dash.dependencies import Input, Output
 from flask import Flask
 
-from .....utilities import unique
+from utilities import unique
+
 from ....constants import DASHBOARD_PATH, ONLINE_TAG_NAME
 from ....context import get_context
 from ....helper import freeze, snake_case_to_text, text_to_hex_color

src/great_ai/great_ai/__init__.py

@@ -1,23 +0,0 @@
-from .context import configure
-from .deploy import GreatAI
-from .exceptions import (
-    ArgumentValidationError,
-    MissingArgumentError,
-    WrongDecoratorOrderError,
-)
-from .models import save_model, use_model
-from .output_views import (
-    ClassificationOutput,
-    MultiLabelClassificationOutput,
-    RegressionOutput,
-)
-from .parameters import log_metric, parameter
-from .persistence import MongodbDriver, ParallelTinyDbDriver, TracingDatabaseDriver
-from .remote import (
-    HttpClient,
-    RemoteCallError,
-    call_remote_great_ai,
-    call_remote_great_ai_async,
-)
-from .tracing import add_ground_truth, delete_ground_truth, query_ground_truth
-from .views import Trace

src/great_ai/large_file/__init__.py

@@ -1 +0,0 @@
-from .large_file import LargeFile, LargeFileLocal, LargeFileMongo, LargeFileS3

src/great_ai/large_file/__main__.py

@@ -1,71 +0,0 @@
-#!/usr/bin/env python3
-
-from argparse import Namespace
-from pathlib import Path
-from typing import Mapping, Type
-
-from ..utilities import get_logger
-from .large_file import LargeFile, LargeFileLocal, LargeFileMongo, LargeFileS3
-from .parse_arguments import parse_arguments
-
-logger = get_logger("large_file")
-
-
-def main() -> None:
-    parser, args = parse_arguments()
-
-    large_file = get_class(args)
-
-    if not args.cache and not args.push and not args.delete:
-        logger.warning("No action required.")
-        parser.print_help()
-
-    if args.cache:
-        for c in args.cache:
-            split = c.split(":")
-            file_name = split[0]
-            version = None if len(split) == 1 else int(split[1])
-            large_file(file_name, "r", version=version).get()
-
-    if args.push:
-        for p in args.push:
-            path = Path(p)
-            large_file(path.name, "w").push(path)
-
-    if args.delete:
-        for f in args.delete:
-            large_file(f).delete()
-
-
-def get_class(args: Namespace) -> Type[LargeFile]:
-    factory: Mapping[str, Type[LargeFile]] = {
-        "s3": LargeFileS3,
-        "local": LargeFileLocal,
-        "mongodb": LargeFileMongo,
-    }
-
-    if args.backend not in factory:
-        raise ValueError(
-            f"Backend {args.backend} does not exits, available options: {' ,'.join(factory.keys())}"
-        )
-
-    large_file = factory[args.backend]
-
-    if args.backend != "local":
-        if args.secrets is None:
-            raise ValueError(
-                "Providing a credentials file is required when the backend mode is not `local`."
-            )
-        large_file.configure_credentials_from_file(args.secrets)
-
-    return large_file
-
-
-if __name__ == "__main__":
-    try:
-        main()
-    except KeyboardInterrupt:
-        logger.warning("Exiting")
-        exit()
-    except Exception as e:
-        logger.exception(e)

src/great_ai/large_file/helper/__init__.py

@@ -1,2 +0,0 @@
-from .human_readable_to_byte import human_readable_to_byte
-from .progress_bar import DownloadProgressBar, UploadProgressBar

src/great_ai/large_file/helper/bytes_to_megabytes.py

@@ -1,2 +0,0 @@
-def bytes_to_megabytes(bytes: int) -> str:
-    return f"{round(bytes / 1000 / 1000, 2):.2f}"

src/great_ai/large_file/helper/human_readable_to_byte.py

@@ -1,31 +0,0 @@
-import re
-
-
-def human_readable_to_byte(size: str) -> int:
-    """Case is ignored, kb, kB, Kb, and KB are all treated as kilobyte."""
-
-    if size.strip() == "0":
-        return 0
-
-    possible_units = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
-    units_re = "|".join(possible_units)
-    regex = re.compile(
-        rf"""
-        \s*                         # trim
-        (?P<scalar>\d+(.\d+)?)      # get scalar, it might be a float
-        \s*                         # ignore optional whitespace
-        (?P<unit>{units_re})        # capture the unit
-    """,
-        flags=re.VERBOSE | re.IGNORECASE,
-    )
-
-    match = regex.match(size)
-    if not match:
-        raise ValueError(f'Could not find values in "{size}"')
-
-    results = match.groupdict()
-
-    scalar = float(results["scalar"])
-    idx = possible_units.index(results["unit"].upper())
-    factor = 1024**idx
-    return round(scalar * factor)

src/great_ai/large_file/helper/progress_bar.py

@@ -1,51 +0,0 @@
-import os
-import threading
-from logging import Logger
-from pathlib import Path
-
-from .bytes_to_megabytes import bytes_to_megabytes
-
-
-class ProgressBar:
-    min_progress_percentage_change = 10
-
-    def __init__(self, file_size: int, logger: Logger, prefix: str):
-        self._file_size = file_size
-        self._logger = logger
-        self._prefix = prefix
-
-        self._last_percentage: float = 0
-        self._seen_so_far = 0
-        self._lock = threading.Lock()
-
-    def __call__(self, bytes_amount: int) -> None:
-        with self._lock:
-            self._seen_so_far += bytes_amount
-            percentage = (self._seen_so_far / float(self._file_size)) * 100
-
-            if (
-                percentage != 100
-                and percentage - self._last_percentage
-                < self.min_progress_percentage_change
-            ):
-                return
-
-            self._last_percentage += self.min_progress_percentage_change
-
-            file_size_mb = bytes_to_megabytes(self._file_size)
-            seen_so_far_mb = bytes_to_megabytes(self._seen_so_far)
-            progress = seen_so_far_mb.rjust(len(file_size_mb))
-            self._logger.info(
-                f"{self._prefix} {progress}/{file_size_mb} MB ({percentage:.1f}%)"
-            )
-
-
-class DownloadProgressBar(ProgressBar):
-    def __init__(self, name: str, size: int, logger: Logger):
-        super().__init__(file_size=size, logger=logger, prefix=f"Downloading {name}")
-
-
-class UploadProgressBar(ProgressBar):
-    def __init__(self, path: Path, logger: Logger):
-        size = os.path.getsize(path)
-        super().__init__(file_size=size, logger=logger, prefix=f"Uploading {path.name}")

src/great_ai/large_file/large_file/__init__.py

@@ -1,4 +0,0 @@
-from .large_file import LargeFile
-from .large_file_local import LargeFileLocal
-from .large_file_mongo import LargeFileMongo
-from .large_file_s3 import LargeFileS3

src/great_ai/large_file/large_file/large_file.py

@@ -1,315 +0,0 @@
-import os
-import shutil
-import tempfile
-from abc import ABC, abstractmethod
-from functools import lru_cache
-from pathlib import Path
-from types import TracebackType
-from typing import IO, Any, List, Optional, Type, Union, cast
-
-from ...utilities import ConfigFile, get_logger
-from ..helper import human_readable_to_byte
-from ..models import DataInstance
-
-logger = get_logger("large_file")
-
-
-CACHE_NAME_VERSION_SEPARATOR = "-"
-COMPRESSION_ALGORITHM = "gztar"
-ARCHIVE_EXTENSION = ".tar.gz"
-
-
-class LargeFile(ABC):
-    """
-    Store large files remotely. Use local cache for speed up.
-
-    Examples:
-
-    ```
-    with LargeFile("test.txt", "w", keep_last_n=3) as f:
-        for i in range(1000000):
-            f.write('test\n')
-
-    with LargeFile("test.txt", "r") as f:
-        print(f.readlines()[0])
-
-    path_to_cached_text_file = LargeFile("test.txt", version=0).get()
-    ```
-
-    By default, files are stored in the ".cache" folder and the
-    least recently use is deleted after the overall size reaches 30 GBs.
-
-    Change it with the following properties.
-
-    ```
-    LargeFile.cache_path = Path(".cache")
-    LargeFile.max_cache_size = "30GB"
-    ```
-    """
-
-    initialized = False
-    cache_path = Path(".cache")
-    max_cache_size: Optional[str] = "30GB"
-
-    def __init__(
-        self,
-        name: str,
-        mode: str = "r",
-        *,
-        buffering: int = -1,
-        encoding: Optional[str] = None,
-        errors: Optional[str] = None,
-        newline: Optional[str] = None,
-        version: Optional[int] = None,
-        keep_last_n: Optional[int] = None,
-        cache_only_mode: bool = False,
-    ):
-        self._name = name
-        self._version = version
-        self._mode = mode
-        self._keep_last_n = keep_last_n
-        self._cache_only_mode = cache_only_mode
-
-        self._buffering = buffering
-        self._encoding = encoding
-        self._errors = errors
-        self._newline = newline
-
-        LargeFile.cache_path.mkdir(parents=True, exist_ok=True)
-
-        self._find_instances()
-        self._check_mode_and_set_version()
-
-    @classmethod
-    def configure_credentials_from_file(
-        cls,
-        secrets_path: Union[Path, str],
-    ) -> None:
-        cls.configure_credentials(**ConfigFile(secrets_path))
-
-    @classmethod
-    def configure_credentials(
-        cls,
-    ) -> None:
-        cls.initialized = True
-
-    def __enter__(self) -> IO:
-        self._file: IO[Any] = (
-            tempfile.NamedTemporaryFile(
-                mode=self._mode,
-                buffering=self._buffering,
-                encoding=self._encoding,
-                newline=self._newline,
-                errors=self._errors,
-                delete=False,
-                prefix="large_file-",
-            )
-            if "w" in self._mode
-            else open(
-                self.get(),
-                mode=self._mode,
-                buffering=self._buffering,
-                encoding=self._encoding,
-                newline=self._newline,
-                errors=self._errors,
-            )
-        )
-
-        return self._file
-
-    def __exit__(
-        self,
-        type: Optional[Type[BaseException]],
-        exc: Optional[BaseException],
-        traceback: Optional[TracebackType],
-    ) -> bool:
-        self._file.close()
-
-        if type is None:
-            if "w" in self._mode:
-                self.push(Path(self._file.name))
-                os.unlink(self._file.name)
-        else:
-            logger.exception("Could not finish operation.")
-
-        return True
-
-    @property
-    def _local_name(self) -> str:
-        return f"{self._name}{CACHE_NAME_VERSION_SEPARATOR}{self.version}"
-
-    @property
-    def version(self) -> int:
-        return cast(int, self._version)
-
-    @lru_cache(1)
-    def get(self, hide_progress: bool = False) -> Path:
-        remote_path = next(
-            i.remote_path for i in self._instances if i.version == self._version
-        )
-
-        destination = self.cache_path / self._local_name
-        if not destination.exists():
-            logger.info(f"File {self._local_name} does not exist locally")
-
-            with tempfile.TemporaryDirectory() as tmp:
-                local_root_path = Path(tmp)
-                tmp_file_archive = (
-                    local_root_path / f"{self._local_name}{ARCHIVE_EXTENSION}"
-                )
-                self._download(
-                    remote_path, tmp_file_archive, hide_progress=hide_progress
-                )
-
-                logger.info(f"Decompressing {self._local_name}")
-                shutil.unpack_archive(str(tmp_file_archive), tmp, COMPRESSION_ALGORITHM)
-                shutil.move(str(local_root_path / self._local_name), str(destination))
-        else:
-            logger.info(f"File {self._local_name} found in cache")
-
-        return destination
-
-    def push(self, path: Union[Path, str], hide_progress: bool = False) -> None:
-        if isinstance(path, str):
-            path = Path(path)
-
-        with tempfile.TemporaryDirectory() as tmp:
-            if path.is_file():
-                logger.info(f"Copying file for {self._local_name}")
-                copy: Any = shutil.copy
-            else:
-                logger.info(f"Copying directory for {self._local_name}")
-                copy = shutil.copytree
-
-            try:
-                # Make local copy in the cache
-                shutil.rmtree(self.cache_path / self._local_name, ignore_errors=True)
-                copy(str(path), str(self.cache_path / self._local_name))
-            except shutil.SameFileError:
-                pass  # No worries
-
-            copy(str(path), str(Path(tmp) / self._local_name))
-
-            with tempfile.TemporaryDirectory() as tmp2:
-                # A directory has to be zipped and it cannot contain the output of the zipping
-                logger.info(f"Compressing {self._local_name}")
-                shutil.make_archive(
-                    str(Path(tmp2) / self._local_name),
-                    COMPRESSION_ALGORITHM,
-                    tmp,
-                )
-
-                file_to_be_uploaded = (
-                    Path(tmp2) / f"{self._local_name}{ARCHIVE_EXTENSION}"
-                )
-                self._upload(file_to_be_uploaded, hide_progress=hide_progress)
-
-        self.clean_up()
-
-    def delete(self) -> None:
-        self._keep_last_n = 0
-        self._delete_old_remote_versions()
-
-    def _find_instances(self) -> None:
-        if self._cache_only_mode:
-            self._instances = self._find_instances_from_cache()
-        else:
-            self._instances = self._find_remote_instances()
-
-        self._instances = sorted(self._instances, key=lambda i: i.version)
-
-    def _find_instances_from_cache(self) -> List[DataInstance]:
-        logger.info(f"Fetching cached versions of {self._name}")
-
-        candidates = [
-            DataInstance(
-                name=CACHE_NAME_VERSION_SEPARATOR.join(
-                    f.name.split(CACHE_NAME_VERSION_SEPARATOR)[:-1]
-                ),
-                version=int(f.name.split(CACHE_NAME_VERSION_SEPARATOR)[-1]),
-                remote_path=f,
-            )
-            for f in self.cache_path.glob(
-                f"{self._name}{CACHE_NAME_VERSION_SEPARATOR}*"
-            )
-        ]
-
-        return [c for c in candidates if c.name == self._name]
-
-    def _check_mode_and_set_version(self) -> None:
-        if "+" in self._mode:
-            raise ValueError(
-                f"File mode `{self._mode}` is not allowed3, remove the `+`."
-            )
-
-        if "w" in self._mode:
-            if self._version is not None:
-                raise ValueError("Providing a version is not allowed in write mode.")
-
-            self._version = self._instances[-1].version + 1 if self._instances else 0
-
-        elif "r" in self._mode:
-            if not self._instances:
-                raise FileNotFoundError(
-                    f"File {self._name} not found. No versions are available."
-                )
-
-            if self._version is None:
-                self._version = self._instances[-1].version
-                logger.info(
-                    f"Latest version of {self._name} is {self._version} "
-                    + f"(from versions: {self.versions_pretty})"
-                )
-            elif self._version not in [i.version for i in self._instances]:
-                raise FileNotFoundError(
-                    f"File {self._name} not found with version {self._version}. "
-                    + f"(from versions: {self.versions_pretty})"
-                )
-        else:
-            raise ValueError("Unsupported file mode.")
-
-    @property
-    def versions_pretty(self) -> str:
-        return ", ".join((str(i.version) for i in self._instances))
-
-    def clean_up(self) -> None:
-        self._delete_old_remote_versions()
-        self._prune_cache()
-
-    def _prune_cache(self) -> None:
-        self.cache_path.mkdir(parents=True, exist_ok=True)
-
-        if self.max_cache_size is None:
-            return
-
-        allowed_size = human_readable_to_byte(self.max_cache_size)
-        assert allowed_size >= 0
-
-        least_recently_read = sorted(
-            [f for f in self.cache_path.glob("*")], key=lambda f: f.stat().st_atime
-        )
-
-        while sum(os.path.getsize(f) for f in least_recently_read) > allowed_size:
-            file = least_recently_read.pop(0)
-            logger.info(
-                f"Deleting file from cache to meet quota (max_cache_size={self.max_cache_size}): {file}"
-            )
-            os.unlink(file)
-
-    @abstractmethod
-    def _find_remote_instances(self) -> List[DataInstance]:
-        pass
-
-    @abstractmethod
-    def _download(
-        self, remote_path: Any, local_path: Path, hide_progress: bool
-    ) -> None:
-        pass
-
-    @abstractmethod
-    def _upload(self, local_path: Path, hide_progress: bool) -> None:
-        pass
-
-    @abstractmethod
-    def _delete_old_remote_versions(self) -> None:
-        pass

src/great_ai/large_file/large_file/large_file_local.py

@@ -1,58 +0,0 @@
-from pathlib import Path
-from typing import Any, List, Optional
-
-from ...utilities import get_logger
-from ..models import DataInstance
-from .large_file import LargeFile
-
-logger = get_logger("large_file")
-
-
-class LargeFileLocal(LargeFile):
-    def __init__(
-        self,
-        name: str,
-        mode: str = "r",
-        *,
-        buffering: int = -1,
-        encoding: Optional[str] = None,
-        errors: Optional[str] = None,
-        newline: Optional[str] = None,
-        version: Optional[int] = None,
-        keep_last_n: Optional[int] = None,
-    ):
-        super().__init__(
-            name,
-            mode,
-            buffering=buffering,
-            encoding=encoding,
-            errors=errors,
-            newline=newline,
-            version=version,
-            keep_last_n=keep_last_n,
-            cache_only_mode=True,
-        )
-        super().configure_credentials()
-
-    def _find_remote_instances(self) -> List[DataInstance]:
-        return []
-
-    def _download(
-        self, remote_path: Any, local_path: Path, hide_progress: bool
-    ) -> None:
-        raise NotImplementedError()
-
-    def _upload(self, local_path: Path, hide_progress: bool) -> None:
-        pass  # the "upload" is already done py the parent's caching mechanism
-
-    def _delete_old_remote_versions(self) -> None:
-        if self._keep_last_n is not None:
-            for i in (
-                self._instances[: -self._keep_last_n]
-                if self._keep_last_n > 0
-                else self._instances
-            ):
-                logger.info(
-                    f"Removing old version (keep_last_n={self._keep_last_n}): {i.remote_path}"
-                )
-                i.remote_path.unlink()

src/great_ai/large_file/large_file/large_file_mongo.py

@@ -1,121 +0,0 @@
-import re
-from functools import cached_property
-from pathlib import Path
-from typing import Any, List, Mapping
-
-from gridfs import DEFAULT_CHUNK_SIZE, Database, GridFSBucket
-from pymongo import MongoClient
-
-from ...utilities import get_logger
-from ..helper import DownloadProgressBar, UploadProgressBar
-from ..models import DataInstance
-from .large_file import LargeFile
-
-logger = get_logger("large_file")
-
-
-MONGO_NAME_VERSION_SEPARATOR = "_"
-
-
-class LargeFileMongo(LargeFile):
-    mongo_connection_string = None
-    mongo_database = None
-
-    @classmethod
-    def configure_credentials(  # type: ignore
-        cls,
-        *,
-        mongo_connection_string: str,
-        mongo_database: str,
-        **_: Mapping[str, Any],
-    ) -> None:
-        cls.mongo_connection_string = mongo_connection_string
-        cls.mongo_database = mongo_database
-        super().configure_credentials()
-
-    @cached_property
-    def _client(self) -> GridFSBucket:
-        if self.mongo_connection_string is None or self.mongo_database is None:
-            raise ValueError(
-                "Please configure the MongoDB access options by calling LargeFileMongo.configure_credentials or set offline_mode=True in the constructor."
-            )
-
-        db: Database = MongoClient(self.mongo_connection_string)[self.mongo_database]
-        return GridFSBucket(db)
-
-    def _find_remote_instances(self) -> List[DataInstance]:
-        logger.debug(f"Fetching Mongo (GridFS) versions of {self._name}")
-
-        return [
-            DataInstance(
-                name=MONGO_NAME_VERSION_SEPARATOR.join(
-                    f.name.split(MONGO_NAME_VERSION_SEPARATOR)[:-1]
-                ),
-                version=int(f.name.split(MONGO_NAME_VERSION_SEPARATOR)[-1]),
-                remote_path=(f._id, f.length),
-                origin="mongodb",
-            )
-            for f in self._client.find(
-                {
-                    "filename": re.compile(
-                        re.escape(self._name + MONGO_NAME_VERSION_SEPARATOR) + ".*"
-                    )
-                }
-            )
-        ]
-
-    def _download(
-        self, remote_path: Any, local_path: Path, hide_progress: bool
-    ) -> None:
-        logger.info(f"Downloading {remote_path[0]} from Mongo (GridFS)")
-
-        progress = (
-            DownloadProgressBar(
-                name=str(remote_path[0]), size=remote_path[1], logger=logger
-            )
-            if not hide_progress
-            else None
-        )
-        with self._client.open_download_stream(remote_path[0]) as stream:
-            with open(local_path, "wb") as f:
-                while True:
-                    content = stream.read(DEFAULT_CHUNK_SIZE)
-                    f.write(content)
-
-                    if progress:
-                        progress(len(content))
-                    if len(content) < DEFAULT_CHUNK_SIZE:
-                        break
-
-    def _upload(self, local_path: Path, hide_progress: bool) -> None:
-        logger.info(f"Uploading {local_path} to Mongo (GridFS)")
-
-        progress = (
-            UploadProgressBar(path=local_path, logger=logger)
-            if not hide_progress
-            else None
-        )
-        with self._client.open_upload_stream(
-            f"{self._name}{MONGO_NAME_VERSION_SEPARATOR}{self.version}"
-        ) as stream:
-            with open(local_path, "rb") as f:
-                while True:
-                    content = f.read(DEFAULT_CHUNK_SIZE)
-                    stream.write(content)
-
-                    if progress:
-                        progress(len(content))
-                    if len(content) < DEFAULT_CHUNK_SIZE:
-                        break
-
-    def _delete_old_remote_versions(self) -> None:
-        if self._keep_last_n is not None:
-            for i in (
-                self._instances[: -self._keep_last_n]
-                if self._keep_last_n > 0
-                else self._instances
-            ):
-                logger.info(
-                    f"Removing old version from MongoDB (GridFS) (keep_last_n={self._keep_last_n}): {i.name}{MONGO_NAME_VERSION_SEPARATOR}{i.version}"
-                )
-                self._client.delete(i.remote_path[0])

src/great_ai/large_file/large_file/large_file_s3.py

@@ -1,151 +0,0 @@
-from functools import cached_property
-from pathlib import Path
-from typing import Any, List, Mapping, Optional
-
-import boto3
-
-from ...utilities import get_logger
-from ..helper import DownloadProgressBar, UploadProgressBar
-from ..models import DataInstance
-from .large_file import LargeFile
-
-logger = get_logger("large_file")
-
-
-S3_NAME_VERSION_SEPARATOR = "/"
-
-
-class LargeFileS3(LargeFile):
-    """
-    Store large files in S3. Use local cache for speed up.
-
-    Examples:
-
-    ```
-    with LargeFile("test.txt", "w", keep_last_n=3) as f:
-        for i in range(1000000):
-            f.write('test\n')
-
-    with LargeFile("test.txt", "r") as f:
-        print(f.readlines()[0])
-
-    path_to_cached_text_file = LargeFile("test.txt", version=0).get()
-    ```
-
-    By default, files are stored in the ".cache" folder and the
-    least recently use is deleted after the overall size reaches 30 GBs.
-
-    Change it with the following properties.
-
-    ```
-    LargeFile.cache_path = Path(".cache")
-    LargeFile.max_cache_size = "30GB"
-    ```
-    """
-
-    region_name = None
-    access_key_id = None
-    secret_access_key = None
-    bucket_name = None
-    endpoint_url = None
-
-    @classmethod
-    def configure_credentials(  # type: ignore
-        cls,
-        *,
-        aws_region_name: str,
-        aws_access_key_id: str,
-        aws_secret_access_key: str,
-        large_files_bucket_name: str,
-        aws_endpoint_url: Optional[str] = None,
-        **_: Mapping[str, Any],
-    ) -> None:
-        cls.region_name = aws_region_name
-        cls.access_key_id = aws_access_key_id
-        cls.secret_access_key = aws_secret_access_key
-        cls.bucket_name = large_files_bucket_name
-        cls.endpoint_url = aws_endpoint_url
-        super().configure_credentials()
-
-    @cached_property
-    def _client(self) -> boto3.client:
-        if (
-            self.region_name is None
-            or self.access_key_id is None
-            or self.secret_access_key is None
-            or self.bucket_name is None
-        ):
-            raise ValueError(
-                "Please configure the S3 access options by calling LargeFileS3.configure_credentials or set offline_mode=True in the constructor."
-            )
-
-        return boto3.client(
-            "s3",
-            aws_access_key_id=self.access_key_id,
-            aws_secret_access_key=self.secret_access_key,
-            region_name=self.region_name,
-            endpoint_url=self.endpoint_url,
-        )
-
-    def _find_remote_instances(self) -> List[DataInstance]:
-        logger.debug(f"Fetching S3 versions of {self._name}")
-
-        found_objects = self._client.list_objects_v2(
-            Bucket=self.bucket_name, Prefix=self._name
-        )
-        return (
-            [
-                DataInstance(
-                    name=o["Key"].split(S3_NAME_VERSION_SEPARATOR)[0],
-                    version=int(o["Key"].split(S3_NAME_VERSION_SEPARATOR)[-1]),
-                    remote_path=o["Key"],
-                )
-                for o in found_objects["Contents"]
-                if o["Key"].split(S3_NAME_VERSION_SEPARATOR)[0] == self._name
-            ]
-            if "Contents" in found_objects
-            else []
-        )
-
-    def _download(
-        self, remote_path: Any, local_path: Path, hide_progress: bool
-    ) -> None:
-        logger.info(f"Downloading {remote_path} from S3")
-
-        size = self._client.head_object(Bucket=self.bucket_name, Key=remote_path)[
-            "ContentLength"
-        ]
-
-        self._client.download_file(
-            Bucket=self.bucket_name,
-            Key=remote_path,
-            Filename=str(local_path),
-            Callback=None
-            if hide_progress
-            else DownloadProgressBar(name=str(remote_path), size=size, logger=logger),
-        )
-
-    def _upload(self, local_path: Path, hide_progress: bool) -> None:
-        key = f"{self._name}/{self.version}"
-        logger.info(f"Uploading {self._local_name} to S3 as {key}")
-
-        self._client.upload_file(
-            Filename=str(local_path),
-            Bucket=self.bucket_name,
-            Key=key,
-            Callback=None
-            if hide_progress
-            else UploadProgressBar(path=local_path, logger=logger),
-        )
-
-    def _delete_old_remote_versions(self) -> None:
-        if self._keep_last_n is not None:
-            for i in (
-                self._instances[: -self._keep_last_n]
-                if self._keep_last_n > 0
-                else self._instances
-            ):
-                logger.info(
-                    f"Removing old version from S3 (keep_last_n={self._keep_last_n}): {i.remote_path}"
-                )
-                self._client.delete_object(Bucket=self.bucket_name, Key=i.remote_path)

src/great_ai/large_file/models/__init__.py

@@ -1 +0,0 @@
-from .data_instance import DataInstance

src/great_ai/large_file/models/data_instance.py

@@ -1,9 +0,0 @@
-from typing import Any
-
-from pydantic import BaseModel
-
-
-class DataInstance(BaseModel):
-    name: str
-    version: int
-    remote_path: Any

src/great_ai/large_file/parse_arguments.py

@@ -1,56 +0,0 @@
-from argparse import ArgumentParser, Namespace
-from typing import Tuple
-
-
-def parse_arguments() -> Tuple[ArgumentParser, Namespace]:
-    parser = ArgumentParser(
-        description="Store and version large files in S3; open them like regular files. Caching included.",
-    )
-
-    parser.add_argument(
-        "-b",
-        "--backend",
-        type=str,
-        help="choose which backend to use, available options: `local`, `s3`, `mongodb`",
-        required=True,
-    )
-
-    parser.add_argument(
-        "-s",
-        "--secrets",
-        type=str,
-        help="path to an .ini configuration file with your S3 credentials",
-        required=False,
-    )
-
-    parser.add_argument(
-        "-c",
-        "--cache",
-        nargs="+",
-        type=str,
-        help="download file into local cache, example: file_name.txt:version",
-        required=False,
-    )
-
-    parser.add_argument(
-        "-p",
-        "--push",
-        nargs="+",
-        type=str,
-        help="push a local file into S3 and set it as the most recent version",
-        required=False,
-    )
-
-    parser.add_argument(
-        "-d",
-        "--delete",
-        nargs="+",
-        type=str,
-        help="delete every version of file from S3",
-        required=False,
-    )
-
-    parser.print_usage = parser.print_help  # type: ignore
-    args = parser.parse_args()
-
-    return parser, args

src/great_ai/parse_arguments.py

@@ -1,51 +0,0 @@
-from argparse import ArgumentParser, Namespace
-
-
-def parse_arguments() -> Namespace:
-    parser = ArgumentParser(
-        description="GreatAI-Server for deploying you AI applications with ease.",
-    )
-
-    parser.add_argument(
-        "file_name",
-        type=str,
-        help="the name of the file containing your to-be-served function such as `main.py`\n",
-    )
-
-    default_host = "0.0.0.0"
-    parser.add_argument(
-        "--host",
-        type=str,
-        help=f"it is passed to uvicorn which starts a server listening on this address (default: {default_host})",
-        default=default_host,
-        required=False,
-    )
-
-    default_port = 6060
-    parser.add_argument(
-        "--port",
-        type=int,
-        help=f"it is passed to uvicorn which starts a server listening on this port (default: {default_port})",
-        default=default_port,
-        required=False,
-    )
-
-    default_timeout_keep_alive = 600
-    parser.add_argument(
-        "--timeout_keep_alive",
-        type=int,
-        help=f"it is passed to uvicorn which uses it for timing out requests taking longer than this many seconds (default: {default_timeout_keep_alive})",
-        default=600,
-        required=False,
-    )
-
-    default_worker_count = 1
-    parser.add_argument(
-        "--worker_count",
-        type=int,
-        help=f"it is passed to uvicorn which starts this many server processes (default: {default_worker_count})",
-        default=default_worker_count,
-        required=False,
-    )
-
-    return parser.parse_args()

src/great_ai/persistence/tracing_database_driver.py

@@ -3,7 +3,8 @@ from datetime import datetime
 from pathlib import Path
 from typing import List, Optional, Sequence, Tuple, Union
 
-from ...utilities import ConfigFile
+from utilities import ConfigFile
+
 from ..views import Filter, SortBy, Trace
 
 

src/great_ai/remote/call_remote_great_ai.py

@@ -3,7 +3,8 @@ from typing import Any, Mapping, Optional, Type, TypeVar
 
 from pydantic import BaseModel
 
-from ...utilities import get_logger
+from utilities import get_logger
+
 from ..views import Trace
 from .call_remote_great_ai_async import call_remote_great_ai_async
 

src/great_ai/utilities/__init__.py

@@ -1,11 +0,0 @@
-from .chunk import chunk
-from .clean import clean
-from .config_file import ConfigFile, ParseError
-from .evaluate_ranking import evaluate_ranking
-from .get_sentences import get_sentences
-from .language import english_name_of_language, is_english, predict_language
-from .logger import get_logger
-from .match_names import match_names
-from .parallel_map import WorkerException, parallel_map, threaded_parallel_map
-from .unchunk import unchunk
-from .unique import unique

src/great_ai/utilities/chunk.py

@@ -1,17 +0,0 @@
-from typing import Iterable, List, TypeVar
-
-T = TypeVar("T")
-
-
-def chunk(values: Iterable[T], chunk_size: int) -> Iterable[T]:
-    assert chunk_size >= 1
-
-    result: List[T] = []
-    for v in values:
-        result.append(v)
-        if len(result) == chunk_size:
-            yield result
-            result = []
-
-    if len(result) > 0:
-        yield result

src/great_ai/utilities/clean.py

@@ -1,68 +0,0 @@
-import html
-import re
-import unicodedata
-
-import unidecode
-
-from .data import left_regular_punctuations, right_regular_punctuations
-from .external.pylatexenc.latex2text import LatexNodes2Text
-from .logger import get_logger
-
-logger = get_logger("clean")
-latex = LatexNodes2Text()
-
-
-joined_left_punctuations = "".join(left_regular_punctuations).replace("]", r"\]")
-joined_right_punctuations = "".join(right_regular_punctuations).replace("[", r"\[")
-
-
-def clean(
-    text: str,
-    ignore_xml: bool = False,
-    ignore_latex: bool = False,
-    remove_brackets: bool = False,
-    convert_to_ascii: bool = False,
-) -> str:
-    if not ignore_xml:
-        text = re.sub(r"<[^>]*>", " ", text)
-        text = html.unescape(text)
-
-    if not ignore_latex:
-        text = text.replace("%", "\\%")  # escape LaTeX comments before parsing as LaTeX
-
-        try:
-            text = latex.latex_to_text(text, tolerant_parsing=True, strict_braces=False)
-            text = text.replace("%s", " ")
-        except:
-            logger.exception("Latex parsing error")
-
-    if convert_to_ascii:
-        text = unicodedata.normalize("NFKD", text)
-
-        try:
-            text.encode("ASCII", errors="strict")
-        except UnicodeEncodeError:
-            text = "".join([c for c in text if not unicodedata.combining(c)])
-            text = unidecode.unidecode(text)
-
-    text = re.sub(
-        r"\b[a-zA-Z](?:[\t ]+[a-zA-Z]\b)+", lambda m: re.sub(r"[\t ]", "", m[0]), text
-    )  # A R T I C L E => ARTICLE
-
-    if remove_brackets:
-        text = re.sub(r"\[[^\]]*\]", " ", text)
-
-    # fix hypens: break- word => break-word
-    text = re.sub(r"(\S)-\s+", r"\1-", text)
-    text = re.sub(r"\s+-(\S)", r"-\1", text)
-
-    # collapse whitespace
-    text = re.sub(r"\s+", " ", text)
-
-    # fix punctuation
-    text = re.sub(rf" ([{joined_left_punctuations}])", r"\1", text)
-    text = re.sub(rf"([{joined_right_punctuations}]) ", r"\1", text)
-
-    text = text.strip()
-
-    return text

src/great_ai/utilities/config_file/__init__.py

@@ -1,2 +0,0 @@
-from .config_file import ConfigFile
-from .parse_error import ParseError

src/great_ai/utilities/config_file/config_file.py

@@ -1,91 +0,0 @@
-import os
-from pathlib import Path
-from typing import Dict, Iterable, Tuple, Union
-
-from ..logger import get_logger
-from .parse_error import ParseError
-from .pattern import pattern
-
-ENVIRONMENT_VARIABLE_KEY_PREFIX = "ENV"
-
-logger = get_logger("ConfigFile")
-
-
-class ConfigFile:
-    def __init__(self, path: Union[Path, str]) -> None:
-        if not isinstance(path, Path):
-            path = Path(path)
-
-        if not path.exists():
-            raise FileNotFoundError(path.absolute())
-
-        self._path = path
-        self._key_values: Dict[str, str] = {}
-
-        self._parse()
-
-    def _parse(self):
-        with open(self._path, encoding="utf-8") as f:
-            lines: str = f.read()
-
-        matches = pattern.findall(lines)
-        for key, *values in matches:
-            try:
-                value = next(v for v in values if v)
-            except StopIteration:
-                raise ParseError(
-                    f"Cannot parse config file ({self._path.absolute()}), error at key `{key}`"
-                )
-
-            already_exists = key in self._key_values
-            if already_exists and not value.startswith(
-                f"{ENVIRONMENT_VARIABLE_KEY_PREFIX}:"
-            ):
-                raise KeyError(
-                    f"Key `{key}` has been already defined and its value is `{self._key_values[key]}`"
-                )
-
-            if value.startswith(f"{ENVIRONMENT_VARIABLE_KEY_PREFIX}:"):
-                _, value = value.split(":")
-                if value not in os.environ:
-                    issue = f'The value of `{key}` contains the "{ENVIRONMENT_VARIABLE_KEY_PREFIX}` prefix but `{value}` is not defined as an environment variable'
-                    if already_exists:
-                        logger.warning(
-                            f"{issue}, using the default value defined above (`{self._key_values[key]}`)"
-                        )
-                        continue
-                    else:
-                        raise KeyError(
-                            f"{issue} and no default value has been provided"
-                        )
-                else:
-                    value = os.environ[value]
-
-            self._key_values[key] = value
-
-    def __getattr__(self, key: str) -> str:
-        if key in self._key_values:
-            return self._key_values[key]
-        raise KeyError(
-            f"Key `{key}` is not found in configuration file ({self._path.absolute()})"
-        )
-
-    __getitem__ = __getattr__
-
-    def __iter__(self) -> Iterable[Tuple[str, str]]:
-        return iter(self._key_values)
-
-    def __len__(self) -> int:
-        return len(self._key_values)
-
-    def keys(self):
-        return self._key_values.keys()
-
-    def values(self):
-        return self._key_values.values()
-
-    def items(self):
-        return self._key_values.items()
-
-    def __repr__(self):
-        return f"{type(self).__name__}(path={self._path}) {self._key_values}"

src/great_ai/utilities/config_file/parse_error.py

@@ -1,2 +0,0 @@
-class ParseError(Exception):
-    pass

src/great_ai/utilities/config_file/pattern.py

@@ -1,20 +0,0 @@
-import re
-
-pattern = re.compile(
-    r"""
-        (?:^|\n)        # new key-value pairs must start on a new line
-        \s*             # leading whitespace is allowed
-        (?!\#)          # the key cannot start with a `#` symbol
-        (\w+?)          # then comes the key
-        \s*=\s*         # the key and value are separated by an equal sign
-        (?:             # then comes the value
-              "([^"]*)"   # the value can be surrounded by quotes: "value"
-            | '([^']*)'   # the value can be surrounded by quotes: 'value'
-            | `([^`]*)`   # the value can be surrounded by quotes: `value`
-            | ([^#\n]*?)  # or it is bare, in that case, the trailing whitespace is ignored
-        )
-        \s*(?:\#.*)?    # comments can be added with the `#` symbol
-        (?=\n|$)        # a key-value pairs are separated by new lines
-    """,
-    flags=re.UNICODE | re.VERBOSE,
-)

src/great_ai/utilities/data/__init__.py

@@ -1,6 +0,0 @@
-from .american_spellings import american_spellings
-from .punctuations import (
-    left_regular_punctuations,
-    right_regular_punctuations,
-    sentence_ending_punctuations,
-)

src/great_ai/utilities/data/punctuations.py

@@ -1,22 +0,0 @@
-sentence_ending_punctuations = [".", "?", "!", ":", ";"]
-
-# punctuations that usually have no space between them and the character to their left
-left_regular_punctuations = [
-    *sentence_ending_punctuations,
-    ",",
-    "'",
-    "%",
-    ")",
-    "}",
-    "]",
-]
-
-# punctuations that usually have no space between them and the character to their right
-right_regular_punctuations = [
-    "(",
-    "{",
-    "[",
-    "$",
-    "€",
-    "#",
-]

src/great_ai/utilities/evaluate_ranking/__init__.py

@@ -1,2 +0,0 @@
-from .draw_f1_iso_lines import draw_f1_iso_lines
-from .evaluate_ranking import evaluate_ranking

src/great_ai/utilities/evaluate_ranking/draw_f1_iso_lines.py

@@ -1,28 +0,0 @@
-from typing import Optional
-
-import matplotlib.pyplot as plt
-import numpy as np
-from matplotlib.axes import Axes
-
-
-def draw_f1_iso_lines(
-    resolution: int = 1000,
-    min: float = 0.2,
-    max: float = 0.8,
-    steps: int = 4,
-    axes: Optional[Axes] = None,
-) -> None:
-    if axes is None:
-        axes = plt.axes()
-
-    for f_score in np.linspace(min, max, num=steps):
-        x = np.linspace(f_score / (2 - f_score), 1, num=resolution)
-        y = f_score * x / (2 * x - f_score)
-
-        axes.plot(x[y >= 0], y[y >= 0], color="gray", alpha=0.2)
-
-        axes.annotate(
-            f"f1={f_score:0.1f}",
-            backgroundcolor="w",
-            xy=(0.9, y[int(resolution * 0.9)] + 0.02),
-        )

src/great_ai/utilities/evaluate_ranking/evaluate_ranking.py

@@ -1,90 +0,0 @@
-from pathlib import Path
-from typing import Dict, List, Optional, Union
-
-import matplotlib
-import matplotlib.pyplot as plt
-import numpy as np
-from sklearn.metrics import average_precision_score, precision_recall_curve
-
-from ..unique import unique
-from .draw_f1_iso_lines import draw_f1_iso_lines
-
-
-def evaluate_ranking(
-    expected: List[Union[str, float]],
-    actual_scores: List[float],
-    target_recall: float,
-    title: Optional[str] = "",
-    disable_interpolation: bool = False,
-    axes: Optional[plt.Axes] = None,
-    output_svg: Optional[Path] = None,
-    reverse_order: bool = False,
-    plot: bool = True,
-) -> Dict[Union[str, float], float]:
-    assert 0 <= target_recall <= 1
-
-    if plot and axes is None:
-        fig = plt.figure(figsize=(20, 10))
-        fig.patch.set_facecolor("white")
-        ax = plt.axes()
-    else:
-        ax = axes
-
-    classes = sorted(unique(expected), reverse=reverse_order)
-    str_classes = [str(c) for c in classes]
-
-    with matplotlib.rc_context({"font.size": 20}):
-        if plot:
-            ax.set_xmargin(0)
-
-            draw_f1_iso_lines(axes=ax)
-
-        results: Dict[Union[str, float], float] = {}
-        for i in range(len(classes) - 1):
-            binarized_expected = [
-                (v < classes[i]) if reverse_order else (v > classes[i])
-                for v in expected
-            ]
-            precision, recall, _ = precision_recall_curve(
-                binarized_expected, actual_scores
-            )
-
-            if not disable_interpolation:
-                for j in range(1, len(precision)):
-                    precision[j] = max(precision[j - 1], precision[j])
-
-            closest_recall_index = np.argmin(np.abs(recall - target_recall))
-            precision_at_closest_recall = precision[closest_recall_index]
-            average_precision = average_precision_score(
-                binarized_expected, actual_scores
-            )
-            results[classes[i]] = precision_at_closest_recall
-
-            if plot:
-                ax.plot(
-                    recall,
-                    precision,
-                    label=f"{'|'.join(str_classes[:i + 1])} ↔ {'|'.join(str_classes[i+1:])} (P@{target_recall:.2f}={precision_at_closest_recall:.2f}, AP={average_precision:.2f})",
-                )
-
-        if plot:
-            ax.legend(loc="upper right")
-            ax.axvline(x=target_recall, linestyle="--", color="#55c6bb", linewidth=2.0)
-
-            if title is None:
-                title = "Ranking evaluation"
-
-            ax.set_title(f'{title} ({" < ".join(str_classes)})', pad=20)
-
-            ax.set_xlabel("Recall")
-            ax.set_ylabel("Precision")
-
-            ax.set_xticks([target_recall] + sorted(ax.get_xticks()))
-
-        if plot and output_svg is None:
-            if axes is None:
-                plt.show()
-        elif output_svg:
-            plt.savefig(output_svg, format="svg")
-
-    return results

src/great_ai/utilities/external/pylatexenc/README.md

@@ -1 +0,0 @@
-https://github.com/phfaist/pylatexenc

src/great_ai/utilities/external/pylatexenc/__init__.py

@@ -1,37 +0,0 @@
-#
-# The MIT License (MIT)
-#
-# Copyright (c) 2015 Philippe Faist
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-
-
-"""
-Utilities for LaTeX to/from Unicode Text conversion.
-
-Main Site:
-
-    https://github.com/phfaist/pylatexenc/
-
-"""
-
-from .version import version_str as _version_str
-
-__version__ = _version_str

src/great_ai/utilities/external/pylatexenc/_util.py

@@ -1,161 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# The MIT License (MIT)
-#
-# Copyright (c) 2019 Philippe Faist
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-
-
-# Internal module. Internal API may move, disappear or otherwise change at any
-# time and without notice.
-
-
-try:
-    # Python >= 3.3
-    from collections.abc import MutableMapping
-except ImportError:
-    from collections import MutableMapping
-
-import bisect
-import warnings
-
-# ------------------------------------------------------------------------------
-
-
-def pylatexenc_deprecated_ver(ver, msg, stacklevel=2):
-    warnings.warn(
-        "Deprecated (pylatexenc {}): {} ".format(ver, msg.strip()),
-        DeprecationWarning,
-        stacklevel=stacklevel + 1,
-    )
-
-
-def pylatexenc_deprecated_2(msg, stacklevel=2):
-    warnings.warn(
-        (
-            "Deprecated (pylatexenc 2.0): {} "
-            "[see https://pylatexenc.readthedocs.io/en/latest/new-in-pylatexenc-2/]"
-        ).format(msg.strip()),
-        DeprecationWarning,
-        stacklevel=stacklevel + 1,
-    )
-
-
-# ------------------------------------------------------------------------------
-
-
-class LazyDict(MutableMapping):
-    r"""
-    A lazy dictionary that loads its data when it is first queried.
-
-    This is used to store the legacy
-    :py:data:`pylatexenc.latexwalker.default_macro_dict` as well as
-    :py:data:`pylatexenc.latex2text.default_macro_dict` etc.  Such that these
-    "dictionaries" are still exposed at the module-level, but the data is loaded
-    only if they are actually queried.
-    """
-
-    def __init__(self, generate_dict_fn):
-        self._full_dict = None
-        self._generate_dict_fn = generate_dict_fn
-
-    def _ensure_instance(self):
-        if self._full_dict is not None:
-            return
-        self._full_dict = self._generate_dict_fn()
-
-    def __getitem__(self, key):
-        self._ensure_instance()
-        return self._full_dict.__getitem__(key)
-
-    def __setitem__(self, key, val):
-        self._ensure_instance()
-        return self._full_dict.__setitem__(key, val)
-
-    def __delitem__(self, key):
-        self._ensure_instance()
-        return self._full_dict.__delitem__(key)
-
-    def __iter__(self):
-        self._ensure_instance()
-        return iter(self._full_dict)
-
-    def __len__(self):
-        self._ensure_instance()
-        return len(self._full_dict)
-
-    def copy(self):
-        self._ensure_instance()
-        return self._full_dict.copy()
-
-    def clear(self):
-        self._ensure_instance()
-        return self._full_dict.clear()
-
-
-# ------------------------------------------------------------------------------
-
-
-class LineNumbersCalculator(object):
-    r"""
-    Utility to calculate line numbers.
-    """
-
-    def __init__(self, s):
-        super(LineNumbersCalculator, self).__init__()
-
-        def find_all_new_lines(x):
-            # first line starts at the beginning of the string
-            yield 0
-            k = 0
-            while k < len(x):
-                k = x.find("\n", k)
-                if k == -1:
-                    return
-                k += 1
-                # s[k] is the character after the newline, i.e., the 0-th column
-                # of the new line
-                yield k
-
-        self._pos_new_lines = list(find_all_new_lines(s))
-
-    def pos_to_lineno_colno(self, pos, as_dict=False):
-        r"""
-        Return the line and column number corresponding to the given `pos`.
-
-        Return a tuple `(lineno, colno)` giving line number and column number.
-        Line numbers start at 1 and column number start at zero, i.e., the
-        beginning of the document (`pos=0`) has line and column number `(1,0)`.
-        If `as_dict=True`, then a dictionary with keys 'lineno', 'colno' is
-        returned instead of a tuple.
-        """
-
-        # find line number in list
-
-        # line_no is the index of the last item in self._pos_new_lines that is <= pos.
-        line_no = bisect.bisect_right(self._pos_new_lines, pos) - 1
-        assert line_no >= 0 and line_no < len(self._pos_new_lines)
-
-        col_no = pos - self._pos_new_lines[line_no]
-        # 1+... so that line and column numbers start at 1
-        if as_dict:
-            return {"lineno": 1 + line_no, "colno": col_no}
-        return (1 + line_no, col_no)

src/great_ai/utilities/external/pylatexenc/latex2text/__main__.py

@@ -1,325 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# The MIT License (MIT)
-#
-# Copyright (c) 2018 Philippe Faist
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-
-import argparse
-import fileinput
-import logging
-import sys
-
-from .. import latexwalker
-from ..latex2text import LatexNodes2Text, _strict_latex_spaces_predef
-from ..version import version_str
-
-
-def main(argv=None):
-
-    if argv is None:
-        argv = sys.argv[1:]
-
-    parser = argparse.ArgumentParser(prog="latex2text", add_help=False)
-
-    codegroup = parser.add_argument_group("Input options")
-
-    codegroup.add_argument(
-        "--code",
-        "-c",
-        action="store",
-        default=None,
-        metavar="LATEX_CODE",
-        help="Convert the given LATEX_CODE to unicode text instead of reading "
-        "from FILE or standard input.  You cannot specify FILEs if you use this "
-        "option, and any standard input is ignored.",
-    )
-
-    codegroup.add_argument(
-        "files",
-        metavar="FILE",
-        nargs="*",
-        help="Input files to read LaTeX code from. If no FILE(s) is/are specified, "
-        "LaTeX code is read from standard input unless --code is specified",
-    )
-
-    group = parser.add_argument_group("LatexWalker options")
-
-    group.add_argument(
-        "--parser-keep-inline-math",
-        action="store_const",
-        const=True,
-        dest="parser_keep_inline_math",
-        default=None,
-        help=argparse.SUPPRESS,
-    )
-    group.add_argument(
-        "--no-parser-keep-inline-math",
-        action="store_const",
-        const=False,
-        dest="parser_keep_inline_math",
-        help=argparse.SUPPRESS,
-    )
-
-    group.add_argument(
-        "--tolerant-parsing",
-        action="store_const",
-        const=True,
-        dest="tolerant_parsing",
-        default=True,
-    )
-    group.add_argument(
-        "--no-tolerant-parsing",
-        action="store_const",
-        const=False,
-        dest="tolerant_parsing",
-        help="Tolerate syntax errors when parsing, and attempt to continue (default yes)",
-    )
-
-    # I'm not sure this flag is useful and if it should be exposed at all.
-    # Accept it, but make it hidden.
-    parser.add_argument(
-        "--strict-braces",
-        action="store_const",
-        const=True,
-        dest="strict_braces",
-        default=False,
-        help=argparse.SUPPRESS,
-    )
-    parser.add_argument(
-        "--no-strict-braces",
-        action="store_const",
-        const=False,
-        dest="strict_braces",
-        # help="Report errors for mismatching LaTeX braces (default no)"
-        help=argparse.SUPPRESS,
-    )
-
-    group = parser.add_argument_group("LatexNodes2Text options")
-
-    group.add_argument(
-        "--text-keep-inline-math",
-        action="store_const",
-        const=True,
-        dest="text_keep_inline_math",
-        default=None,
-        help=argparse.SUPPRESS,
-    )
-    group.add_argument(
-        "--no-text-keep-inline-math",
-        action="store_const",
-        const=False,
-        dest="text_keep_inline_math",
-        help=argparse.SUPPRESS,
-    )
-
-    group.add_argument(
-        "--math-mode",
-        action="store",
-        dest="math_mode",
-        choices=["text", "with-delimiters", "verbatim", "remove"],
-        default="text",
-        help="How to handle chunks of math mode LaTeX code. 'text' = convert "
-        "to text like the rest; 'with-delimiters' = same as 'text' but retain "
-        "the original math mode delimiters; 'verbatim' = keep verbatim LaTeX code; "
-        "'remove' = remove from input entirely",
-    )
-
-    group.add_argument(
-        "--fill-text",
-        dest="fill_text",
-        action="store",
-        nargs="?",
-        default=-1,
-        help="Attempt to wrap text to the given width, or 80 columns if option is "
-        "specified with no argument",
-    )
-
-    group.add_argument(
-        "--keep-comments",
-        action="store_const",
-        const=True,
-        dest="keep_comments",
-        default=False,
-    )
-    group.add_argument(
-        "--no-keep-comments",
-        action="store_const",
-        const=False,
-        dest="keep_comments",
-        help="Keep LaTeX comments in text output (default no)",
-    )
-
-    class ListWithHiddenItems(list):
-        def __init__(self, thelist, hiddenitems):
-            super(ListWithHiddenItems, self).__init__(thelist)
-            self.hiddenitems = hiddenitems
-
-        def __contains__(self, value):
-            return (
-                super(ListWithHiddenItems, self).__contains__(value)
-                or value in self.hiddenitems
-            )
-
-    strict_latex_spaces_choices = ListWithHiddenItems(
-        # the list
-        ["off", "on"]
-        + list(k for k in _strict_latex_spaces_predef.keys() if k != "default"),
-        # hidden items: Value is accepted, but not shown in list of choices
-        ["default"],
-    )
-    group.add_argument(
-        "--strict-latex-spaces",
-        choices=strict_latex_spaces_choices,
-        dest="strict_latex_spaces",
-        default="macros",
-        help="How to handle whitespace. See documentation for the class "
-        "LatexNodes2Text().",
-    )
-
-    group.add_argument(
-        "--keep-braced-groups",
-        action="store_const",
-        const=True,
-        dest="keep_braced_groups",
-        default=False,
-    )
-    group.add_argument(
-        "--no-keep-braced-groups",
-        action="store_const",
-        const=False,
-        dest="keep_braced_groups",
-        help="Keep LaTeX {braced groups} in text output (default no)",
-    )
-
-    group.add_argument(
-        "--keep-braced-groups-minlen",
-        type=int,
-        default=2,
-        dest="keep_braced_groups_minlen",
-        help="Only apply --keep-braced-groups to groups that contain at least "
-        "this many characters",
-    )
-
-    group = parser.add_argument_group("General options")
-
-    group.add_argument(
-        "-q",
-        "--quiet",
-        dest="logging_level",
-        action="store_const",
-        const=logging.ERROR,
-        default=logging.INFO,
-        help="Suppress warning messages",
-    )
-    group.add_argument(
-        "-v",
-        "--verbose",
-        dest="logging_level",
-        action="store_const",
-        const=logging.DEBUG,
-        help="Verbose output",
-    )
-    group.add_argument(
-        "--version",
-        action="version",
-        version="pylatexenc {}".format(version_str),
-        help="Show version information and exit",
-    )
-    group.add_argument(
-        "--help", action="help", help="Show this help information and exit"
-    )
-
-    args = parser.parse_args(argv)
-
-    logging.basicConfig()
-    logging.getLogger().setLevel(args.logging_level)
-    logger = logging.getLogger(__name__)
-
-    if (
-        args.parser_keep_inline_math is not None
-        or args.text_keep_inline_math is not None
-    ):
-        logger.warning(
-            "Options --parser-keep-inline-math and --text-keep-inline-math are "
-            "deprecated and no longer have any effect.  Please use "
-            "--math-mode=... instead."
-        )
-
-    latex = ""
-    if args.code:
-        if args.files:
-            logger.error(
-                "Cannot specify both FILEs and --code option. "
-                "Use --help option for more information."
-            )
-            sys.exit(1)
-        latex = args.code
-    else:
-        for line in fileinput.input(files=args.files):
-            latex += line
-
-    if args.fill_text != -1:
-        if args.fill_text is not None and len(args.fill_text):
-            fill_text = int(args.fill_text)
-        else:
-            fill_text = True
-    else:
-        fill_text = None
-
-    lw = latexwalker.LatexWalker(
-        latex, tolerant_parsing=args.tolerant_parsing, strict_braces=args.strict_braces
-    )
-
-    (nodelist, pos, len_) = lw.get_latex_nodes()
-
-    ln2t = LatexNodes2Text(
-        math_mode=args.math_mode,
-        keep_comments=args.keep_comments,
-        strict_latex_spaces=args.strict_latex_spaces,
-        keep_braced_groups=args.keep_braced_groups,
-        keep_braced_groups_minlen=args.keep_braced_groups_minlen,
-        fill_text=fill_text,
-    )
-
-    print(ln2t.nodelist_to_text(nodelist))
-
-
-def run_main():
-
-    try:
-
-        main()
-
-    except SystemExit:
-        raise
-    except:  # lgtm [py/catch-base-exception]
-        import pdb
-        import traceback
-
-        traceback.print_exc()
-        pdb.post_mortem()
-
-
-if __name__ == "__main__":
-
-    main()
-    # run_main() # debug

src/great_ai/utilities/external/pylatexenc/latexencode/__init__.py

@@ -1,331 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# The MIT License (MIT)
-#
-# Copyright (c) 2018 Philippe Faist
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-
-r"""
-The `latexencode` module provides a set of routines that allows you to
-convert a unicode string to LaTeX escape sequences.
-
-For basic usage you can use the :py:func:`unicode_to_latex()` function
-directly::
-
-  >>> from pylatexenc.latexencode import unicode_to_latex
-  >>> print(unicode_to_latex('À votre santé'))
-  \`A votre sant\'e
-  >>> print(unicode_to_latex('The length of samples #3 & #4 is 3μm'))
-  The length of samples \#3 \& \#4 is 3\ensuremath{\mu}m
-
-The conversion is handled by the class :py:class:`UnicodeToLatexEncoder`.  If
-you are converting multiple strings, you may create an instance with the flags
-you like and invoke its method
-:py:meth:`~UnicodeToLatexEncoder.unicode_to_latex()` as many times as necessary::
-
-  >>> from pylatexenc.latexencode import UnicodeToLatexEncoder
-  >>> u = UnicodeToLatexEncoder(unknown_char_policy='replace')
-  >>> print(u.unicode_to_latex('À votre santé'))
-  \`A votre sant\'e
-  >>> print(u.unicode_to_latex('The length of samples #3 & #4 is 3μm'))
-  The length of samples \#3 \& \#4 is 3\ensuremath{\mu}m
-  >>> print(u.unicode_to_latex('À votre santé: 乾杯'))
-  No known latex representation for character: U+4E7E - ‘乾’
-  No known latex representation for character: U+676F - ‘杯’
-  \`A votre sant\'e: {\bfseries ?}{\bfseries ?}
-
-Example using custom conversion rules::
-
-  >>> from pylatexenc.latexencode import UnicodeToLatexEncoder, \
-  ...     UnicodeToLatexConversionRule, RULE_REGEX
-  >>> u = UnicodeToLatexEncoder(
-  ...     conversion_rules=[
-  ...         UnicodeToLatexConversionRule(rule_type=RULE_REGEX, rule=[
-  ...             (re.compile(r'-->'), r'\\textrightarrow'),
-  ...             (re.compile(r'<--'), r'\\textleftarrow'),
-  ...         ]),
-  ...         'defaults'
-  ...     ]
-  ... )
-  >>> print(u.unicode_to_latex("Cheers --> À votre santé"))
-  Cheers {\textrightarrow} \`A votre sant\'e
-
-See :py:class:`UnicodeToLatexEncoder` and
-:py:class:`UnicodeToLatexConversionRule`.  Note for regex rules, the replacement
-text is expanded like the second argument of `re.sub()` and backslashes need to
-be escaped even inside raw strings.
-
-.. versionadded:: 2.0
-
-   The class :py:class:`UnicodeToLatexEncoder` along with its helper functions
-   and classes were introduced in `pylatexenc 2.0`.
-
-   The earlier function :py:func:`utf8tolatex()` that was available in
-   `pylatexenc 1.x` is still provided unchanged, so code written for `pylatexenc
-   1.x` should work without changes.  New code is however strongly encouraged to
-   employ the new API.
-"""
-
-from __future__ import absolute_import, print_function, unicode_literals
-
-import functools
-import itertools
-import logging
-import sys
-import unicodedata
-
-if sys.version_info.major > 2:
-    unicode = str  # need to support unicode() w/ no arguments
-    basestring = str
-    # use MappingProxyType for keeping
-    # inspect function argument names
-    from inspect import getfullargspec
-    from types import MappingProxyType as _MappingProxyType
-else:
-    _MappingProxyType = dict
-    # inspect function argument names -- simulate getfullargspec with getargspec (argh...)
-    from inspect import getargspec as getfullargspec
-
-logger = logging.getLogger(__name__)
-
-
-from .. import _util
-from ._partial_latex_encoder import PartialLatexToLatexEncoder
-from ._unicode_to_latex_encoder import (
-    RULE_CALLABLE,
-    RULE_DICT,
-    RULE_REGEX,
-    UnicodeToLatexConversionRule,
-    UnicodeToLatexEncoder,
-    get_builtin_conversion_rules,
-    get_builtin_uni2latex_dict,
-)
-
-# ------------------------------------------------
-
-
-# ------------------------------------------------
-
-
-# ------------------------------------------------
-
-
-_u2l_obj_cache = {}
-
-
-def unicode_to_latex(
-    s,
-    non_ascii_only=False,
-    replacement_latex_protection="braces",
-    unknown_char_policy="keep",
-    unknown_char_warning=True,
-):
-    r"""
-    Shorthand for constructing a :py:class:`UnicodeToLatexEncoder` instance and
-    calling its :py:meth:`~UnicodeToLatexEncoder.unicode_to_latex()` method.
-
-    The :py:class:`UnicodeToLatexEncoder` instances for given option settings
-    are cached, making repeated calls to :py:func:`unicode_to_latex()` possible
-    without creating a new instance upon each call.
-
-    The parameters `non_ascii_only`, `replacement_latex_protection`,
-    `unknown_char_policy`, and `unknown_char_warning` are directly passed on to
-    the :py:class:`UnicodeToLatexEncoder` constructor.  See the class doc for
-    :py:class:`UnicodeToLatexEncoder` for more information about what they do.
-
-    You may only use arguments to this function that are python hashable (like
-    `True`, `False`, or simple strings) to help us keep a cache of previously
-    constructed :py:class:`UnicodeToLatexEncoder` instances.  For instance, it
-    is not possible to provide a callable to `unknown_char_policy`.  It is also
-    not possible to specify custom conversion rules with this helper function.
-    If you need any of these features, simply create a
-    :py:class:`UnicodeToLatexEncoder` instance directly.
-    """
-
-    key = (
-        non_ascii_only,
-        replacement_latex_protection,
-        unknown_char_policy,
-        unknown_char_warning,
-    )
-
-    if key in _u2l_obj_cache:
-        u = _u2l_obj_cache[key]
-    else:
-        u = UnicodeToLatexEncoder(
-            non_ascii_only=non_ascii_only,
-            replacement_latex_protection=replacement_latex_protection,
-            unknown_char_policy=unknown_char_policy,
-            unknown_char_warning=unknown_char_warning,
-        )
-        _u2l_obj_cache[key] = u
-
-    return u.unicode_to_latex(s)
-
-
-# ------------------------------------------------------------------------------
-
-# Don't change pylatexenc 1.x function:
-
-
-def _get_deprecated_utf82latex():
-    #
-    # Don't issue a deprecation warning, because utf8tolatex() uses the
-    # `utf82latex` dict even if it isn't modified by the user.
-    #
-    #     _util.pylatexenc_deprecated_2(
-    #         "The module-level dictionary `pylatexenc.latexencode.utf82latex` is deprecated "
-    #         "and might be removed in a future version of `pylatexenc`.",
-    #     )
-
-    # return a copy of the dict so that the user can modify the module-level
-    # `utf82latex` dict without influencing the behavior of the new
-    # `unicode_to_latex()` routines. (E.g., if two python modules use
-    # pylatexenc.latexencode, we don't want one python module's use of
-    # `utf2tolatex()` to influence the behavior of another module's use of
-    # `unicode_to_latex()`.  If both modules use `utf8tolatex()`, we can't avoid
-    # this influence.)
-    from ._uni2latexmap import uni2latex as _uni2latex
-
-    return _uni2latex.copy()
-
-
-utf82latex = _util.LazyDict(generate_dict_fn=_get_deprecated_utf82latex)
-"""
-.. deprecated:: 2.0
-
-   Pylatexenc 1.x exposed the module-level dictionary `utf82latex` that could be
-   modified to alter the behavior of `utf8tolatex()`.
-
-   If you would like to obtain a copy of the built-in unicode to text
-   dictionary, see :py:func:`get_builtin_uni2latex_dict()`.  If you would like
-   to alter the behavior of :py:func:`utf8tolatex()`, you should use
-   :py:class:`UnicodeToLatexEncoder` which provides a rich interface for
-   specifying rules how to convert chars to LaTeX escapes.
-
-   For backwards compatibility, you can still modify the module-level dictionary
-   `utf82latex` (but you can't assign a new object to it) and this will directly
-   modify the global built-in dictionary of known latex escapes.  This is not
-   recommended however, and the `utf82latex` module-level dictionary might be
-   removed in the future.
-
-   .. warning::
-
-      Modifying the `utf82latex` module-level dictionary is not recommended.
-      Doing so will alter the behavior of the `utf8tolatex()` function also for
-      all other modules that also use `pylatexenc`!
-"""
-
-
-def utf8tolatex(
-    s,
-    non_ascii_only=False,
-    brackets=True,
-    substitute_bad_chars=False,
-    fail_bad_chars=False,
-):
-    """
-    .. note::
-
-       Since `pylatexenc 2.0`, it is recommended to use the the
-       :py:func:`unicode_to_latex()` function or the
-       :py:class:`UnicodeToLatexEncoder` class instead of the earlier function
-       `utf8tolatex()`.
-
-       The new routines provide much more flexibility and versatility.  For
-       instance, you can specify custom escape sequences for certain characters.
-       Some cheap benchmarks seem to indicate that the new routines are not
-       significantly slower than the `utf8tolatex()` function.  Also, the name
-       `utf8tolatex()` was poorly chosen, since the argument is in fact not
-       'utf-8'-encoded but rather a Python unicode string object.
-
-       The function `utf8tolatex()` is still provided unchanged from `pylatexenc
-       1.x`.  We do not plan to remove this function in the near future so it is
-       not (yet) considered as deprecated and we will continue to provide it in
-       near future versions of `pylatexenc`.  Bug reports, improvements, and new
-       features will however be directed to :py:func:`UnicodeToLatexEncoder()`.
-
-    Encode a UTF-8 string to a LaTeX snippet.
-
-    If `non_ascii_only` is set to `True`, then usual (ascii) characters such as ``#``,
-    ``{``, ``}`` etc. will not be escaped.  If set to `False` (the default), they are
-    escaped to their respective LaTeX escape sequences.
-
-    If `brackets` is set to `True` (the default), then LaTeX macros are enclosed in
-    brackets.  For example, ``sant\N{LATIN SMALL LETTER E WITH ACUTE}`` is replaced by
-    ``sant{\\'e}`` if `brackets=True` and by ``sant\\'e`` if `brackets=False`.
-
-    .. warning::
-        Using `brackets=False` might give you an invalid LaTeX string, so avoid
-        it! (for instance, ``ma\N{LATIN SMALL LETTER I WITH CIRCUMFLEX}tre`` will be
-        replaced incorrectly by ``ma\\^\\itre`` resulting in an unknown macro ``\\itre``).
-
-    If `substitute_bad_chars=True`, then any non-ascii character for which no LaTeX escape
-    sequence is known is replaced by a question mark in boldface. Otherwise (by default),
-    the character is left as it is.
-
-    If `fail_bad_chars=True`, then a `ValueError` is raised if we cannot find a
-    character substitution for any non-ascii character.
-
-    .. versionchanged:: 1.3
-
-        Added `fail_bad_chars` switch
-    """
-
-    s = unicode(s)  # make sure s is unicode
-    s = unicodedata.normalize("NFC", s)
-
-    if not s:
-        return ""
-
-    result = ""
-    for ch in s:
-        # logger.longdebug("Encoding char %r", ch)
-        if non_ascii_only and ord(ch) < 127:
-            result += ch
-        else:
-            # use the `utf82latex` dict -- not `_uni2latex` which should NOT be
-            # modified externally even for backwards-compatible code
-            lch = utf82latex.get(ord(ch), None)
-            if lch is not None:
-                # add brackets if needed, i.e. if we have a substituting macro.
-                # note: in condition, beware, that lch might be of zero length.
-                result += "{" + lch + "}" if brackets and lch[0:1] == "\\" else lch
-            elif (ord(ch) >= 32 and ord(ch) <= 127) or (ch in "\n\r\t"):
-                # ordinary printable ascii char, just add it
-                result += ch
-            else:
-                # non-ascii char
-                msg = "Character cannot be encoded into LaTeX: U+%04X - `%s'" % (
-                    ord(ch),
-                    ch,
-                )
-                if fail_bad_chars:
-                    raise ValueError(msg)
-
-                logger.warning(msg)
-                if substitute_bad_chars:
-                    result += r"{\bfseries ?}"
-                else:
-                    # keep unescaped char
-                    result += ch
-
-    return result

src/great_ai/utilities/external/pylatexenc/latexencode/__main__.py

@@ -1,148 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# The MIT License (MIT)
-#
-# Copyright (c) 2019 Philippe Faist
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-
-
-import argparse
-import fileinput
-import logging
-import sys
-
-from ..latexencode import unicode_to_latex
-from ..version import version_str
-
-
-def main(argv=None):
-
-    if argv is None:
-        argv = sys.argv[1:]
-
-    parser = argparse.ArgumentParser(prog="latexencode", add_help=False)
-    parser.add_argument(
-        "files",
-        metavar="FILE",
-        nargs="*",
-        help="Input files (if none specified, read from stdandard input)",
-    )
-
-    parser.add_argument(
-        "--non-ascii-only",
-        action="store_const",
-        const=True,
-        dest="non_ascii_only",
-        default=False,
-    )
-    parser.add_argument(
-        "--no-non-ascii-only",
-        action="store_const",
-        const=False,
-        dest="non_ascii_only",
-        help="The option --non-ascii-only specifies that only non-ascii characters "
-        "are to be encoded into LaTeX sequences, and not characters like '$' "
-        "even though they might have a special LaTeX meaning.",
-    )
-
-    parser.add_argument(
-        "--replacement-latex-protection",
-        choices=(
-            "braces",
-            "braces-all",
-            "braces-almost-all",
-            "braces-after-macro",
-            "none",
-        ),
-        dest="replacement_latex_protection",
-        default="braces",
-        help=r"How to protect replacement latex code from producing invalid latex code "
-        r"when concatenated in a longer string.  One of 'braces', 'braces-all', "
-        r"'braces-almost-all', 'braces-after-macro', 'none'.  Example: using "
-        r"choice 'braces' we avoid the invalid replacement 'a→b' -> 'a\tob' "
-        r"with instead 'a{\to}b'.",
-    )
-
-    parser.add_argument(
-        "--unknown-char-policy",
-        choices=("keep", "replace", "ignore", "fail"),
-        dest="unknown_char_policy",
-        default="keep",
-        help="How to deal with nonascii characters with no known latex code equivalent.",
-    )
-
-    parser.add_argument(
-        "-q",
-        "--quiet",
-        dest="logging_level",
-        action="store_const",
-        const=logging.ERROR,
-        default=logging.INFO,
-        help="Suppress warning messages",
-    )
-    parser.add_argument(
-        "--version",
-        action="version",
-        version="pylatexenc {}".format(version_str),
-        help="Show version information and exit",
-    )
-    parser.add_argument(
-        "--help", action="help", help="Show this help information and exit"
-    )
-
-    args = parser.parse_args(argv)
-
-    logging.basicConfig()
-    logging.getLogger().setLevel(args.logging_level)
-
-    latex = ""
-    for line in fileinput.input(files=args.files):
-        latex += line
-
-    result = unicode_to_latex(
-        latex,
-        non_ascii_only=args.non_ascii_only,
-        replacement_latex_protection=args.replacement_latex_protection,
-        unknown_char_policy=args.unknown_char_policy,
-    )
-
-    sys.stdout.write(result)
-
-
-def run_main():
-    try:
-
-        main()
-
-    except SystemExit:
-        raise
-    except:  # lgtm [py/catch-base-exception]
-        import pdb
-        import traceback
-
-        traceback.print_exc()
-        pdb.post_mortem()
-
-
-if __name__ == "__main__":
-
-    # run_main()  ## DEBUG
-    main()

src/great_ai/utilities/external/pylatexenc/latexencode/_partial_latex_encoder.py

@@ -1,120 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# The MIT License (MIT)
-#
-# Copyright (c) 2021 Philippe Faist
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-
-from __future__ import absolute_import, print_function, unicode_literals
-
-# import sys
-import logging
-
-logger = logging.getLogger(__name__)
-
-
-from ._unicode_to_latex_encoder import (
-    RULE_CALLABLE,
-    UnicodeToLatexConversionRule,
-    UnicodeToLatexEncoder,
-)
-
-# if sys.version_info.major == 2:
-#     bytes = str
-#     str = unicode
-
-
-class PartialLatexToLatexEncoder(UnicodeToLatexEncoder):
-    r"""
-    Encode a string while preserving some (fuzzily detected) LaTeX constructs
-    that the input string already has (e.g. accent macros or inline math modes).
-
-    Sometimes you need to fully LaTeX-encode a string that already has some
-    LaTeX constructs.  For instance, titles of bibliographic entries might
-    include some inline math or accents, but they might also include unicode
-    characters that need to be encoded.  Using a
-    :py:class:`UnicodeToLatexEncoder` on such strings would result in ugly
-    doubly-escaped strings such as ``\textbackslash{}'\{e\}``.  Instead,
-    constructs such as ``\'{e}`` should be preserved while other characters
-    and/or constructs (say '&' or '%') as well as unicode characters should be
-    encoded.
-
-    This class offers a simple partial solution: Characters are encoded as per
-    the given `conversion_rules` (or the default conversion rules of
-    :py:class:`UnicodeToLatexEncoder` objects), except that the characters in
-    `keep_latex_chars` are to be interpreted as LaTeX and are not to be further
-    encoded.
-
-    .. versionadded: 2.10
-    """
-
-    def __init__(
-        self,
-        # keyword arguments:
-        keep_latex_chars=r"\${}^_",
-        conversion_rules=None,
-        **kwargs
-    ):
-
-        base_conversion_rules = conversion_rules
-        if base_conversion_rules is None:
-            base_conversion_rules = ["defaults"]
-
-        super(PartialLatexToLatexEncoder, self).__init__(
-            # only a single rule, our own special method that tries to parse
-            # partial latex.
-            conversion_rules=[
-                UnicodeToLatexConversionRule(
-                    rule_type=RULE_CALLABLE,
-                    rule=self._do_partial_latex_encode_step,
-                    replacement_latex_protection="none",
-                )
-            ]
-            + base_conversion_rules,
-            **kwargs
-        )
-
-        self.keep_latex_chars = keep_latex_chars
-
-    def _do_partial_latex_encode_step(self, s, pos):
-        r"""
-        This method is used as a "callable rule" for the
-        :py:class:`UnicodeToLatexEncoder` object.
-
-        The strategy is to see if we have something that looks like a LaTeX char
-        we want to keep.  If so, keep it as is; if not, return `None` so that
-        further rules can be considered by the base unicode encoder.
-        """
-
-        from ..latexwalker import LatexWalker
-
-        if s[pos] in self.keep_latex_chars:
-            # Read a token and if it is a macro, keep the full macro!
-            lw = LatexWalker(s, tolerant_parsing=False)
-
-            tok = lw.get_token(pos, environments=False)
-
-            tok_as_latex = tok.pre_space + s[tok.pos : tok.pos + tok.len]
-
-            # keep the LaTeX token as-is
-            return (tok.pos + tok.len - pos, tok_as_latex)
-
-        return None