Move files
This commit is contained in:
parent
cf0ac4b161
commit
0bf865ce2a
233 changed files with 117 additions and 2394 deletions
1
great_ai/.gitignore
vendored
1
great_ai/.gitignore
vendored
|
|
@ -1 +0,0 @@
|
|||
build
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
# **S**coutinScience **U**tilitie**S** for text processing [](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml)
|
||||
|
||||
> amogus
|
||||
|
||||
## Exports
|
||||
|
||||
- [clean](src/sus/clean.py)
|
||||
- [unique](src/sus/unique.py)
|
||||
- [parallel_map](src/sus/parallel_map.py)
|
||||
- [match_names](src/sus/match_names/match_names.py)
|
||||
- [evaluate_ranking](src/sus/evaluate_ranking/evaluate_ranking.py)
|
||||
- [get_sentences](src/sus/get_sentences.py)
|
||||
|
||||
### Requires loading spacy model
|
||||
|
||||
> This is automatic but will require some time.
|
||||
|
||||
> Add this to the Dockerfile for caching the spaCy model:
|
||||
>
|
||||
> ```docker
|
||||
> RUN pip install --no-cache-dir en-core-web-sm@https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl
|
||||
> ```
|
||||
|
||||
- [publication TEI](src/sus/publication_tei/publication_tei.py)
|
||||
- [lemmatize_text](src/sus/lemmatize_text.py)
|
||||
- [lemmatize_token](src/sus/lemmatize_token.py)
|
||||
- [spacy model (nlp)](src/sus/nlp.py)
|
||||
- [filter_sentences](src/sus/matcher/filter_sentences.py)
|
||||
|
||||
## Development
|
||||
|
||||
- Optional booleans must have a default value of `False`.
|
||||
- No imports in top-level `__init__.py`, in order to not load anything unnecessary automatically
|
||||
- Should only be updated through a PR
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
aws_region_name = your_region_like_eu-west-2
|
||||
aws_access_key_id = YOUR_ACCESS_KEY_ID
|
||||
aws_secret_access_key = YOUR_VERY_SECRET_ACCESS_KEY
|
||||
large_files_bucket_name = create_a_bucket_and_put_its_name_here
|
||||
aws_endpoint_url = this is optional, for backblaze, use this: https://s3.us-west-002.backblazeb2.com
|
||||
|
|
@ -1,118 +0,0 @@
|
|||
# [open(S3)](https://pypi.org/project/open-large/)
|
||||
|
||||
Storing, versioning, and downloading files from S3 made as easy as using `open()` in Python. Caching included.
|
||||
|
||||
## Motivation
|
||||
|
||||
Oftentimes, especially when working with data-heavy applications, large files can proliferate in a repository. Version controlling them is an obvious next step, however, GitHub's git LFS implementation [doesn't support the deletion](https://docs.github.com/en/repositories/working-with-files/managing-large-files/removing-files-from-git-large-file-storage#git-lfs-objects-in-your-repository) of large files, making it easy for them to eat-up the LFS quota and explode the size of your repos.
|
||||
|
||||
## Solution
|
||||
|
||||
```
|
||||
pip install open-large
|
||||
```
|
||||
|
||||
### Simple example
|
||||
|
||||
```python
|
||||
from large_file import LargeFileS3
|
||||
|
||||
LargeFileS3.configure_credentials({
|
||||
"aws_region_name": "your_region_like_eu-west-2",
|
||||
"aws_access_key_id": "YOUR_ACCESS_KEY_ID",
|
||||
"aws_secret_access_key": "YOUR_VERY_SECRET_ACCESS_KEY",
|
||||
"large_files_bucket_name": "create_a_bucket_and_put_its_name_here",
|
||||
})
|
||||
|
||||
# Creates a new version and deletes the older version leaving the 3 most recently used intact
|
||||
with LargeFileS3("test.txt", "w", keep_last_n=3) as f:
|
||||
for i in range(100000):
|
||||
f.write('test\n')
|
||||
|
||||
# By default the latest version is returned
|
||||
# but an optional `version` keyword argument can be provided as well
|
||||
with LargeFileS3("test.txt", "r") as f:
|
||||
print(f.readlines()[0])
|
||||
```
|
||||
|
||||
> Automatically creates a file, writes to it, uploads it to S3, and then queries the most recent version of it.
|
||||
> In this case, the latest version is already in the local cache, no download is required.
|
||||
|
||||
### More details
|
||||
|
||||
`LargeFile` behaves like an opened file (in the background it is a temp file after all). Binary reading and writing is supported along with the [different keywords](https://docs.python.org/3/library/functions.html#open) `open()` accepts.
|
||||
|
||||
The local cache can be configured with these properties:
|
||||
|
||||
```python
|
||||
LargeFile.cache_path = Path('.cache')
|
||||
LargeFile.max_cache_size = "30 GB"
|
||||
```
|
||||
|
||||
#### I only need a path
|
||||
|
||||
In case you only need a path to the "remote" file, this pattern can be applied:
|
||||
|
||||
```python
|
||||
path_to_model = LargeFile("folder-of-my-bert-model", version=31).get()
|
||||
```
|
||||
|
||||
> This will first download the file/folder into your local cache folder. Then, it returns a `Path` object to the local version. Which can be turned into a string with `str(path_to_model)`.
|
||||
|
||||
The same approach works for uploads:
|
||||
|
||||
```python
|
||||
LargeFile("folder-of-my-bert-model").push('path_to_local/folder_or_file')
|
||||
```
|
||||
|
||||
> This way, both regular files and folders can be handled. The uploaded file is called **folder-of-my-bert-model**, the local name is ignored.
|
||||
|
||||
Lastly, all version of the remote object can be deleted by calling `LargeFile("my-file").delete()`. It will still reside in your local cache afterwards, its deletion will happen next time your local cache has to be pruned.
|
||||
|
||||
### Command-line example
|
||||
|
||||
The package can be used as a module from the command-line to give you more flexibility.
|
||||
|
||||
#### Setup
|
||||
|
||||
Create an .ini file (or use _~/.aws/credentials_). It may look like this:
|
||||
|
||||
```ini
|
||||
aws_region_name = your_region_like_eu-west-2
|
||||
aws_access_key_id = YOUR_ACCESS_KEY_ID
|
||||
aws_secret_access_key = YOUR_VERY_SECRET_ACCESS_KEY
|
||||
large_files_bucket_name = my_large_files
|
||||
endpoint_url = this is optional, for backblaze, use this: https://s3.us-west-002.backblazeb2.com
|
||||
```
|
||||
|
||||
> Just like in [example secrets](example_secrets.ini).
|
||||
|
||||
#### Print the expected options
|
||||
|
||||
```sh
|
||||
python3 -m large_file --help
|
||||
```
|
||||
|
||||
#### Upload some files
|
||||
|
||||
```sh
|
||||
python3 -m large_file --backend s3 --secrets secrets.ini --push my_first_file.json folder/my_second_file my_folder
|
||||
```
|
||||
|
||||
> Only the filename is used as the S3 name, the rest of the path is ignored.
|
||||
|
||||
#### Download some files to the local cache
|
||||
|
||||
This can be useful when building a Docker image for example. This way, the files can already reside inside the container and need not be downloaded later.
|
||||
|
||||
```sh
|
||||
python3 -m large_file --backend s3 -secrets ~/.aws/credentials --cache my_first_file.json:3 my_second_file my_folder:0
|
||||
```
|
||||
|
||||
> Versions may be specified by using `:`-s.
|
||||
|
||||
#### Delete remote files
|
||||
|
||||
```sh
|
||||
python3 -m large_file --backend s3 --secrets ~/.aws/credentials --delete my_first_file.json
|
||||
```
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
[build-system]
|
||||
requires = [
|
||||
"setuptools>=42",
|
||||
"setuptools-git",
|
||||
"wheel"
|
||||
]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
click < 8.1.0
|
||||
unidecode >= 1.3.0
|
||||
multiprocess >= 0.70.0.0
|
||||
tqdm >= 4.0.0
|
||||
psutil >= 5.9.0
|
||||
beautifulsoup4 >= 4.10.0
|
||||
lxml >= 4.6.0
|
||||
spacy >= 3.3.0
|
||||
pydantic >= 1.8.0
|
||||
scikit-learn == 1.1.1
|
||||
matplotlib >= 3.5.0
|
||||
numpy >= 1.22.0
|
||||
langcodes[data] >= 3.3.0
|
||||
segtok >= 1.5.11
|
||||
langdetect >= 1.0.9
|
||||
tinydb >= 4.7.0
|
||||
pandas >= 1.4.0
|
||||
pyaml >= 21.0.0
|
||||
boto3 >= 1.23.0
|
||||
fastapi >= 0.70.0
|
||||
plotly >= 5.8.0
|
||||
dash >= 2.4.0
|
||||
uvicorn[standard] >= 0.17.0
|
||||
|
|
@ -1,52 +0,0 @@
|
|||
[metadata]
|
||||
name = great-ai
|
||||
version = 0.0.1
|
||||
author = András Schmelczer
|
||||
author_email = andras@scoutinscience.com
|
||||
description =
|
||||
long_description = file: README.md
|
||||
long_description_content_type = text/markdown
|
||||
url = https://github.com/ScoutinScience/great-ai
|
||||
project_urls =
|
||||
Bug Tracker = https://github.com/ScoutinScience/great-ai/issues
|
||||
classifiers =
|
||||
Programming Language :: Python :: 3
|
||||
Operating System :: OS Independent
|
||||
|
||||
[options]
|
||||
package_dir =
|
||||
= src
|
||||
packages = find:
|
||||
include_package_data = True
|
||||
python_requires = >=3.8
|
||||
install_requires =
|
||||
unidecode >= 1.3.0
|
||||
multiprocess >= 0.70.0.0
|
||||
tqdm >= 4.0.0
|
||||
psutil >= 5.9.0
|
||||
beautifulsoup4 >= 4.10.0
|
||||
lxml >= 4.6.0
|
||||
spacy >= 3.3.0
|
||||
pydantic >= 1.8.0
|
||||
scikit-learn == 1.1.1
|
||||
matplotlib >= 3.5.0
|
||||
numpy >= 1.22.0
|
||||
langcodes[data] >= 3.3.0
|
||||
segtok >= 1.5.11
|
||||
langdetect >= 1.0.9
|
||||
tinydb >= 4.7.0
|
||||
pandas >= 1.4.0
|
||||
pyaml >= 21.0.0
|
||||
boto3 >= 1.23.0
|
||||
fastapi >= 0.70.0
|
||||
plotly >= 5.8.0
|
||||
dash >= 2.4.0
|
||||
uvicorn[standard] >= 0.17.0
|
||||
watchdog >= 2.1.0
|
||||
pymongo >= 4.0.0
|
||||
|
||||
[options.package_data]
|
||||
* = *.json, *.yaml, *.yml, *.css
|
||||
|
||||
[options.packages.find]
|
||||
where = src
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
from .great_ai import *
|
||||
from .large_file import *
|
||||
from .utilities import *
|
||||
|
|
@ -1,194 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from importlib import import_module, reload
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import uvicorn
|
||||
from uvicorn.config import LOGGING_CONFIG, Config
|
||||
from uvicorn.subprocess import get_subprocess
|
||||
from uvicorn.supervisors.basereload import BaseReload
|
||||
from watchdog.events import FileSystemEvent, PatternMatchingEventHandler
|
||||
from watchdog.observers import Observer
|
||||
|
||||
from .great_ai.constants import SERVER_NAME
|
||||
from .great_ai.context import _is_in_production_mode
|
||||
from .great_ai.deploy import GreatAI
|
||||
from .great_ai.exceptions import ArgumentValidationError, MissingArgumentError
|
||||
from .parse_arguments import parse_arguments
|
||||
from .utilities import get_logger
|
||||
|
||||
logger = get_logger(SERVER_NAME)
|
||||
|
||||
|
||||
GREAT_AI_LOGGING_CONFIG = {
|
||||
**LOGGING_CONFIG,
|
||||
"formatters": {
|
||||
"default": {
|
||||
"()": "great_ai.logger.CustomFormatter",
|
||||
"fmt": "%(asctime)s | %(levelname)8s | %(message)s",
|
||||
},
|
||||
"access": {
|
||||
"()": "great_ai.logger.CustomFormatter",
|
||||
"fmt": "%(asctime)s | %(levelname)8s | %(message)s", # noqa: E501
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_arguments()
|
||||
should_auto_reload = not _is_in_production_mode(logger=None)
|
||||
|
||||
if args.workers > 1 and should_auto_reload:
|
||||
raise ArgumentValidationError(
|
||||
"Cannot use auto-reload with multiple workers: set the `--workers=1` CLI argument,"
|
||||
+ "or set the ENVIRONMENT environment variable to `production`."
|
||||
)
|
||||
|
||||
common_config = dict(
|
||||
host=args.host,
|
||||
port=args.port,
|
||||
timeout_keep_alive=args.timeout_keep_alive,
|
||||
workers=args.workers,
|
||||
server_header=False,
|
||||
reload=False,
|
||||
log_config=GREAT_AI_LOGGING_CONFIG,
|
||||
)
|
||||
|
||||
if not should_auto_reload:
|
||||
file_name = get_script_name(args.file_name)
|
||||
app = find_app(file_name)
|
||||
|
||||
logger.info(f"Starting uvicorn server with app={app}")
|
||||
|
||||
uvicorn.run(app, **common_config) # this will never return
|
||||
|
||||
class EventHandler(PatternMatchingEventHandler):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(patterns=["*.py", "*.ipynb"], ignore_patterns=["__*.py"])
|
||||
self.server: Optional[GreatAIReload] = None
|
||||
self.restart()
|
||||
|
||||
def on_closed(self, event: FileSystemEvent) -> None:
|
||||
logger.warning(f"File {event.src_path} has triggered a restart")
|
||||
self.restart()
|
||||
|
||||
def restart(self) -> None:
|
||||
file_name = get_script_name(args.file_name)
|
||||
app = find_app(file_name)
|
||||
if app is None:
|
||||
logger.warning("Auto-reloading skipped")
|
||||
return
|
||||
|
||||
self.stop_server()
|
||||
|
||||
config = Config(app, **common_config)
|
||||
socket = config.bind_socket()
|
||||
self.server = GreatAIReload(
|
||||
config, target=uvicorn.Server(config=config).run, sockets=[socket]
|
||||
)
|
||||
self.server.startup()
|
||||
|
||||
def stop_server(self) -> None:
|
||||
if self.server:
|
||||
self.server.shutdown()
|
||||
|
||||
restart_handler = EventHandler()
|
||||
observer = Observer()
|
||||
observer.schedule(restart_handler, path=".", recursive=True)
|
||||
observer.start()
|
||||
|
||||
try:
|
||||
while True:
|
||||
time.sleep(50)
|
||||
finally:
|
||||
observer.stop()
|
||||
restart_handler.stop_server()
|
||||
if args.file_name.endswith(".ipynb"):
|
||||
Path(get_script_name_of_notebook(args.file_name)).unlink(missing_ok=True)
|
||||
observer.join()
|
||||
|
||||
|
||||
def get_script_name(file_name_argument: str) -> str:
|
||||
if file_name_argument.endswith(".ipynb"):
|
||||
logger.info("Converting notebook to Python script")
|
||||
try:
|
||||
from nbconvert import PythonExporter
|
||||
|
||||
exporter = PythonExporter()
|
||||
content, _ = exporter.from_filename(file_name_argument)
|
||||
file_name_argument = get_script_name_of_notebook(file_name_argument)
|
||||
with open(file_name_argument, "w", encoding="utf-8") as f:
|
||||
f.write(content)
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"Install `nbconvert` to be able to use Jupyter notebook files or use a regular Python file instead"
|
||||
)
|
||||
|
||||
return re.sub(r"\.(py|ipynb)$", "", file_name_argument)
|
||||
|
||||
|
||||
def get_script_name_of_notebook(notebook_name: str) -> str:
|
||||
base_name = re.sub(r"\.ipynb$", "", notebook_name)
|
||||
return f"__{base_name}__.py"
|
||||
|
||||
|
||||
module = None
|
||||
|
||||
|
||||
def find_app(file_name: str) -> Optional[str]:
|
||||
global module
|
||||
|
||||
logging.disable(logging.CRITICAL)
|
||||
try:
|
||||
if module is None:
|
||||
module = import_module(file_name)
|
||||
else:
|
||||
module = reload(module)
|
||||
except Exception:
|
||||
logging.disable(logging.NOTSET)
|
||||
logger.exception("Could not load file because of an exception: fix your code")
|
||||
return None
|
||||
finally:
|
||||
logging.disable(logging.NOTSET)
|
||||
|
||||
for name, value in module.__dict__.items():
|
||||
if isinstance(value, GreatAI):
|
||||
app_name = name
|
||||
|
||||
if app_name:
|
||||
logger.info(f"Found `{app_name}` to be the GreatAI app ")
|
||||
else:
|
||||
raise MissingArgumentError(
|
||||
"GreatAI app could not be found, make sure to use `@GreatAI.deploy` on your prediction function"
|
||||
)
|
||||
|
||||
return f"{file_name}:{app_name}.app"
|
||||
|
||||
|
||||
class GreatAIReload(BaseReload):
|
||||
def startup(self) -> None:
|
||||
self.process = get_subprocess(
|
||||
config=self.config, target=self.target, sockets=self.sockets
|
||||
)
|
||||
self.process.start()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self.process.terminate()
|
||||
self.process.join()
|
||||
|
||||
for sock in self.sockets:
|
||||
sock.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except KeyboardInterrupt:
|
||||
exit()
|
||||
except Exception as e:
|
||||
logger.error(e)
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
from .context import configure
|
||||
from .deploy import GreatAI
|
||||
from .models import save_model, use_model
|
||||
from .output_models import (
|
||||
ClassificationOutput,
|
||||
MultiLabelClassificationOutput,
|
||||
RegressionOutput,
|
||||
)
|
||||
from .parameters import log_metric, parameter
|
||||
from .persistence import MongodbDriver, ParallelTinyDbDriver, TracingDatabaseDriver
|
||||
from .tracing import add_ground_truth, delete_ground_truth, query_ground_truth
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
from ..large_file import LargeFileMongo, LargeFileS3
|
||||
from .persistence.mongodb_driver import MongodbDriver
|
||||
|
||||
ENV_VAR_KEY = "ENVIRONMENT"
|
||||
PRODUCTION_KEY = "production"
|
||||
DASHBOARD_PATH = "/dashboard"
|
||||
|
||||
MONGO_CONFIG_PATHS = ["mongodb.ini", "mongo.ini", "mongo_db.ini", "mongo-db.ini"]
|
||||
DEFAULT_TRACING_DATABASE_CONFIG_PATHS = {
|
||||
MongodbDriver: MONGO_CONFIG_PATHS,
|
||||
}
|
||||
|
||||
DEFAULT_LARGE_FILE_CONFIG_PATHS = {
|
||||
LargeFileS3: ["s3.ini", "b2.ini"],
|
||||
LargeFileMongo: MONGO_CONFIG_PATHS,
|
||||
}
|
||||
|
||||
GITHUB_LINK = "https://github.com/ScoutinScience/great-ai"
|
||||
|
||||
TRAIN_SPLIT_TAG_NAME = "train"
|
||||
TEST_SPLIT_TAG_NAME = "test"
|
||||
VALIDATION_SPLIT_TAG_NAME = "validation"
|
||||
GROUND_TRUTH_TAG_NAME = "ground_truth"
|
||||
PRODUCTION_TAG_NAME = "production"
|
||||
DEVELOPMENT_TAG_NAME = "development"
|
||||
ONLINE_TAG_NAME = "online"
|
||||
|
||||
SERVER_NAME = "GreatAI-Server"
|
||||
|
||||
SE4ML_WEBSITE = "https://se-ml.github.io/practices/"
|
||||
|
|
@ -1,191 +0,0 @@
|
|||
import os
|
||||
import random
|
||||
from logging import DEBUG, Logger
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional, Type, cast
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from great_ai.large_file import LargeFile, LargeFileLocal
|
||||
from great_ai.utilities import get_logger
|
||||
|
||||
from .constants import (
|
||||
DEFAULT_LARGE_FILE_CONFIG_PATHS,
|
||||
DEFAULT_TRACING_DATABASE_CONFIG_PATHS,
|
||||
ENV_VAR_KEY,
|
||||
PRODUCTION_KEY,
|
||||
SE4ML_WEBSITE,
|
||||
)
|
||||
from .persistence import ParallelTinyDbDriver, TracingDatabaseDriver
|
||||
|
||||
|
||||
class Context(BaseModel):
|
||||
tracing_database: TracingDatabaseDriver
|
||||
large_file_implementation: Type[LargeFile]
|
||||
is_production: bool
|
||||
logger: Logger
|
||||
should_log_exception_stack: bool
|
||||
prediction_cache_size: int
|
||||
dashboard_table_size: int
|
||||
|
||||
class Config:
|
||||
arbitrary_types_allowed = True
|
||||
|
||||
def to_flat_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"tracing_database": type(self.tracing_database).__name__,
|
||||
"large_file_implementation": self.large_file_implementation.__name__,
|
||||
"is_production": self.is_production,
|
||||
"should_log_exception_stack": self.should_log_exception_stack,
|
||||
"prediction_cache_size": self.prediction_cache_size,
|
||||
"dashboard_table_size": self.dashboard_table_size,
|
||||
}
|
||||
|
||||
|
||||
_context: Optional[Context] = None
|
||||
|
||||
|
||||
def get_context() -> Context:
|
||||
if _context is None:
|
||||
configure()
|
||||
|
||||
return cast(Context, _context)
|
||||
|
||||
|
||||
def configure(
|
||||
*,
|
||||
log_level: int = DEBUG,
|
||||
seed: int = 42,
|
||||
tracing_database: Optional[Type[TracingDatabaseDriver]] = None,
|
||||
large_file_implementation: Optional[Type[LargeFile]] = None,
|
||||
should_log_exception_stack: Optional[bool] = None,
|
||||
prediction_cache_size: int = 512,
|
||||
disable_se4ml_banner: bool = False,
|
||||
dashboard_table_size: int = 50,
|
||||
) -> None:
|
||||
global _context
|
||||
logger = get_logger("great_ai", level=log_level)
|
||||
|
||||
if _context is not None:
|
||||
logger.warn(
|
||||
"Configuration has been already initialised, overwriting.\n"
|
||||
+ "Make sure to call `configure()` before importing your application code."
|
||||
)
|
||||
|
||||
is_production = _is_in_production_mode(logger=logger)
|
||||
|
||||
_set_seed(seed)
|
||||
|
||||
tracing_database = _initialize_tracing_database(tracing_database, logger=logger)()
|
||||
|
||||
if not tracing_database.is_production_ready:
|
||||
if is_production:
|
||||
logger.error(
|
||||
f"The selected tracing database ({type(tracing_database).__name__}) is not recommended for production"
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
f"The selected tracing database ({type(tracing_database).__name__}) is not recommended for production"
|
||||
)
|
||||
|
||||
_context = Context(
|
||||
tracing_database=tracing_database,
|
||||
large_file_implementation=_initialize_large_file(
|
||||
large_file_implementation, logger=logger
|
||||
),
|
||||
is_production=is_production,
|
||||
logger=logger,
|
||||
should_log_exception_stack=not is_production
|
||||
if should_log_exception_stack is None
|
||||
else should_log_exception_stack,
|
||||
prediction_cache_size=prediction_cache_size,
|
||||
dashboard_table_size=dashboard_table_size,
|
||||
)
|
||||
|
||||
logger.info("Settings: configured ✅")
|
||||
for k, v in get_context().to_flat_dict().items():
|
||||
logger.info(f"🔩 {k}: {v}")
|
||||
|
||||
if not is_production and not disable_se4ml_banner:
|
||||
logger.warning(
|
||||
"You still need to check whether you follow all best practices before trusting your deployment."
|
||||
)
|
||||
logger.warning(f"> Find out more at {SE4ML_WEBSITE}")
|
||||
|
||||
|
||||
def _is_in_production_mode(logger: Optional[Logger]) -> bool:
|
||||
environment = os.environ.get(ENV_VAR_KEY)
|
||||
|
||||
if environment is None:
|
||||
if logger:
|
||||
logger.warning(
|
||||
f"Environment variable {ENV_VAR_KEY} is not set, defaulting to development mode ‼️"
|
||||
)
|
||||
is_production = False
|
||||
else:
|
||||
is_production = environment.lower() == PRODUCTION_KEY
|
||||
if logger:
|
||||
if not is_production:
|
||||
logger.info(
|
||||
f"Value of {ENV_VAR_KEY} is `{environment}` which is not equal to `{PRODUCTION_KEY}`"
|
||||
+ "defaulting to development mode ‼️"
|
||||
)
|
||||
else:
|
||||
logger.info("Running in production mode ✅")
|
||||
|
||||
return is_production
|
||||
|
||||
|
||||
def _initialize_tracing_database(
|
||||
selected: Optional[Type[TracingDatabaseDriver]], logger: Logger
|
||||
) -> Type[TracingDatabaseDriver]:
|
||||
for tracing_driver, paths in DEFAULT_TRACING_DATABASE_CONFIG_PATHS.items():
|
||||
if selected is None or selected == tracing_driver:
|
||||
if tracing_driver.initialized:
|
||||
logger.warning(
|
||||
f"{tracing_driver.__name__} has been already configured: skipping initialisation"
|
||||
)
|
||||
return tracing_driver
|
||||
for p in paths:
|
||||
if Path(p).exists():
|
||||
logger.info(
|
||||
f"Found credentials file ({Path(p).absolute()}), initialising {tracing_driver.__name__}"
|
||||
)
|
||||
tracing_driver.configure_credentials_from_file(p)
|
||||
return tracing_driver
|
||||
logger.warning(
|
||||
"Cannot find credentials files, defaulting to using ParallelTinyDbDriver"
|
||||
)
|
||||
return ParallelTinyDbDriver
|
||||
|
||||
|
||||
def _initialize_large_file(
|
||||
selected: Optional[Type[LargeFile]], logger: Logger
|
||||
) -> Type[LargeFile]:
|
||||
for large_file, paths in DEFAULT_LARGE_FILE_CONFIG_PATHS.items():
|
||||
if selected is None or selected == large_file:
|
||||
if large_file.initialized:
|
||||
logger.warning(
|
||||
f"{large_file.__name__} has been already configured: skipping initialisation"
|
||||
)
|
||||
return large_file
|
||||
for p in paths:
|
||||
if Path(p).exists():
|
||||
logger.info(
|
||||
f"Found credentials file ({Path(p).absolute()}), initialising {large_file.__name__}"
|
||||
)
|
||||
large_file.configure_credentials_from_file(p)
|
||||
return large_file
|
||||
logger.warning("Cannot find credentials files, defaulting to using LargeFileLocal")
|
||||
return LargeFileLocal
|
||||
|
||||
|
||||
def _set_seed(seed: int) -> None:
|
||||
random.seed(seed)
|
||||
|
||||
try:
|
||||
import numpy
|
||||
|
||||
numpy.random.seed(seed + 1)
|
||||
except ImportError:
|
||||
pass
|
||||
|
|
@ -1 +0,0 @@
|
|||
from .great_ai import GreatAI
|
||||
|
|
@ -1,211 +0,0 @@
|
|||
import inspect
|
||||
from functools import lru_cache, partial, wraps
|
||||
from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
Generic,
|
||||
Iterable,
|
||||
List,
|
||||
Optional,
|
||||
Type,
|
||||
TypeVar,
|
||||
Union,
|
||||
cast,
|
||||
)
|
||||
|
||||
from fastapi import APIRouter, FastAPI, status
|
||||
from pydantic import BaseModel, create_model
|
||||
|
||||
from great_ai.great_ai.deploy.routes.bootstrap_dashboard import bootstrap_dashboard
|
||||
from great_ai.great_ai.views.cache_statistics import CacheStatistics
|
||||
from great_ai.utilities import parallel_map
|
||||
|
||||
from ..constants import DASHBOARD_PATH
|
||||
from ..context import get_context
|
||||
from ..helper import (
|
||||
freeze_arguments,
|
||||
get_function_metadata_store,
|
||||
snake_case_to_text,
|
||||
use_http_exceptions,
|
||||
)
|
||||
from ..parameters import automatically_decorate_parameters
|
||||
from ..tracing.tracing_context import TracingContext
|
||||
from ..views import ApiMetadata, HealthCheckResponse, Trace
|
||||
from .routes import (
|
||||
bootstrap_docs_endpoints,
|
||||
bootstrap_feedback_endpoints,
|
||||
bootstrap_trace_endpoints,
|
||||
)
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
class GreatAI(Generic[T]):
|
||||
def __init__(self, func: Callable[..., Any], version: str):
|
||||
func = automatically_decorate_parameters(func)
|
||||
get_function_metadata_store(func).is_finalised = True
|
||||
|
||||
self._func = func
|
||||
|
||||
def func_in_tracing_context(*args: Any, **kwargs: Any) -> Trace[T]:
|
||||
with TracingContext[T](func.__name__) as t:
|
||||
result = func(*args, **kwargs)
|
||||
output = t.finalise(output=result)
|
||||
return output
|
||||
|
||||
self._cached_func = lru_cache(get_context().prediction_cache_size)(
|
||||
func_in_tracing_context
|
||||
) # cannot put decorator on method, because it require the context to be setup
|
||||
|
||||
wraps(func)(self)
|
||||
|
||||
self._version = version
|
||||
|
||||
self.app = FastAPI(
|
||||
title=self.name,
|
||||
version=self.version,
|
||||
description=self.documentation
|
||||
+ f"\n\nFind out more in the [dashboard]({DASHBOARD_PATH}).",
|
||||
docs_url=None,
|
||||
redoc_url=None,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def deploy(
|
||||
func: Optional[Callable[..., T]] = None,
|
||||
*,
|
||||
version: str = "0.0.1",
|
||||
disable_rest_api: bool = False,
|
||||
disable_docs: bool = False,
|
||||
disable_dashboard: bool = False,
|
||||
) -> Union[Callable[[Callable[..., T]], "GreatAI[T]"], "GreatAI[T]"]:
|
||||
if func is None:
|
||||
return cast(
|
||||
Callable[[Callable[..., T]], GreatAI[T]],
|
||||
partial(
|
||||
GreatAI.deploy,
|
||||
disable_http=disable_rest_api,
|
||||
disable_docs=disable_docs,
|
||||
disable_dashboard=disable_dashboard,
|
||||
),
|
||||
)
|
||||
|
||||
instance = GreatAI[T](func, version=version)
|
||||
|
||||
if not disable_rest_api:
|
||||
instance._bootstrap_rest_api(
|
||||
disable_docs=disable_docs, disable_dashboard=disable_dashboard
|
||||
)
|
||||
|
||||
return instance
|
||||
|
||||
@freeze_arguments
|
||||
def __call__(self, *args: Any, **kwargs: Any) -> Trace[T]:
|
||||
return self._cached_func(*args, **kwargs)
|
||||
|
||||
def process_batch(
|
||||
self,
|
||||
batch: Iterable[Any],
|
||||
concurrency: Optional[int] = None,
|
||||
) -> List[Trace[T]]:
|
||||
return parallel_map(
|
||||
freeze_arguments(self._cached_func), batch, concurrency=concurrency
|
||||
)
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return snake_case_to_text(self._func.__name__)
|
||||
|
||||
@property
|
||||
def version(self) -> str:
|
||||
return (
|
||||
f"{self._version}+{get_function_metadata_store(self._func).model_versions}"
|
||||
)
|
||||
|
||||
@property
|
||||
def documentation(self) -> str:
|
||||
return (
|
||||
f"GreatAI wrapper for interacting with the `{self._func.__name__}` function.\n\n"
|
||||
+ (
|
||||
"\n".join(
|
||||
line.strip()
|
||||
for line in (self._func.__doc__ or "").split("\n")
|
||||
if line.strip()
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
def _bootstrap_rest_api(self, disable_docs: bool, disable_dashboard: bool) -> None:
|
||||
self._bootstrap_prediction_endpoint()
|
||||
|
||||
if not disable_docs:
|
||||
bootstrap_docs_endpoints(self.app)
|
||||
|
||||
if not disable_dashboard:
|
||||
bootstrap_dashboard(
|
||||
self.app,
|
||||
function_name=self._func.__name__,
|
||||
documentation=self.documentation,
|
||||
)
|
||||
bootstrap_trace_endpoints(self.app)
|
||||
|
||||
bootstrap_feedback_endpoints(self.app)
|
||||
self._bootstrap_meta_endpoints()
|
||||
|
||||
def _bootstrap_prediction_endpoint(self) -> None:
|
||||
router = APIRouter(
|
||||
prefix="/predict",
|
||||
tags=["predictions"],
|
||||
)
|
||||
|
||||
schema = self._get_schema()
|
||||
|
||||
@router.post("/", status_code=status.HTTP_200_OK, response_model=Trace[T])
|
||||
@use_http_exceptions
|
||||
def predict(input_value: schema) -> Trace[T]: # type: ignore
|
||||
return self(**cast(BaseModel, input_value).dict())
|
||||
|
||||
self.app.include_router(router)
|
||||
|
||||
def _get_schema(self) -> Type[BaseModel]:
|
||||
signature = inspect.signature(self._func)
|
||||
parameters = {
|
||||
p.name: (
|
||||
p.annotation if p.annotation != inspect._empty else Any,
|
||||
p.default if p.default != inspect._empty else ...,
|
||||
)
|
||||
for p in signature.parameters.values()
|
||||
if p.name in get_function_metadata_store(self._func).input_parameter_names
|
||||
}
|
||||
|
||||
schema: Type[BaseModel] = create_model("InputModel", **parameters) # type: ignore
|
||||
return schema
|
||||
|
||||
def _bootstrap_meta_endpoints(self) -> None:
|
||||
router = APIRouter(
|
||||
tags=["meta"],
|
||||
)
|
||||
|
||||
@router.get("/health", status_code=status.HTTP_200_OK)
|
||||
def check_health() -> HealthCheckResponse:
|
||||
hits, misses, maxsize, cache_size = self._cached_func.cache_info()
|
||||
cache_statistics = CacheStatistics(
|
||||
hits=hits, misses=misses, size=cache_size, max_size=maxsize
|
||||
)
|
||||
|
||||
return HealthCheckResponse(
|
||||
is_healthy=True, cache_statistics=cache_statistics
|
||||
)
|
||||
|
||||
@router.get(
|
||||
"/version", response_model=ApiMetadata, status_code=status.HTTP_200_OK
|
||||
)
|
||||
def get_version() -> ApiMetadata:
|
||||
return ApiMetadata(
|
||||
name=self.name,
|
||||
version=self.version,
|
||||
documentation=self.documentation,
|
||||
configuration=get_context().to_flat_dict(),
|
||||
)
|
||||
|
||||
self.app.include_router(router)
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
from .bootstrap_dashboard import bootstrap_dashboard
|
||||
from .bootstrap_docs_endpoints import bootstrap_docs_endpoints
|
||||
from .bootstrap_feedback_endpoints import bootstrap_feedback_endpoints
|
||||
from .bootstrap_trace_endpoints import bootstrap_trace_endpoints
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.wsgi import WSGIMiddleware
|
||||
from fastapi.responses import RedirectResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
|
||||
from ...constants import DASHBOARD_PATH
|
||||
from .dashboard import create_dash_app
|
||||
|
||||
PATH = Path(__file__).parent.resolve()
|
||||
|
||||
|
||||
def bootstrap_dashboard(app: FastAPI, function_name: str, documentation: str) -> None:
|
||||
dash_app = create_dash_app(function_name, documentation)
|
||||
|
||||
app.mount(DASHBOARD_PATH, WSGIMiddleware(dash_app))
|
||||
|
||||
@app.get("/", include_in_schema=False)
|
||||
def redirect_to_entrypoint() -> RedirectResponse:
|
||||
return RedirectResponse(DASHBOARD_PATH)
|
||||
|
||||
app.mount(
|
||||
"/assets",
|
||||
StaticFiles(directory=PATH / "dashboard/assets"),
|
||||
name="static",
|
||||
)
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
from fastapi import FastAPI
|
||||
from fastapi.openapi.docs import get_swagger_ui_html
|
||||
from fastapi.responses import RedirectResponse
|
||||
from starlette.responses import HTMLResponse
|
||||
|
||||
|
||||
def bootstrap_docs_endpoints(app: FastAPI) -> None:
|
||||
@app.get("/docs", include_in_schema=False)
|
||||
def custom_swagger_ui_html() -> HTMLResponse:
|
||||
return get_swagger_ui_html(openapi_url="openapi.json", title=app.title)
|
||||
|
||||
@app.get("/docs/index.html", include_in_schema=False)
|
||||
def redirect_to_docs() -> RedirectResponse:
|
||||
return RedirectResponse("/docs")
|
||||
|
|
@ -1,44 +0,0 @@
|
|||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, FastAPI, HTTPException, Response, status
|
||||
|
||||
from ...context import get_context
|
||||
from ...views import EvaluationFeedbackRequest
|
||||
|
||||
|
||||
def bootstrap_feedback_endpoints(app: FastAPI) -> None:
|
||||
router = APIRouter(
|
||||
prefix="/traces/{trace_id}/feedback",
|
||||
tags=["feedback"],
|
||||
)
|
||||
|
||||
@router.put("/", status_code=status.HTTP_202_ACCEPTED)
|
||||
def set_feedback(trace_id: str, input: EvaluationFeedbackRequest) -> Response:
|
||||
trace = get_context().tracing_database.get(trace_id)
|
||||
if trace is None:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
|
||||
|
||||
trace.feedback = input.feedback
|
||||
|
||||
get_context().tracing_database.update(trace_id, trace)
|
||||
return Response(status_code=status.HTTP_202_ACCEPTED)
|
||||
|
||||
@router.get("/", status_code=status.HTTP_200_OK)
|
||||
def get_feedback(trace_id: str) -> Any:
|
||||
trace = get_context().tracing_database.get(trace_id)
|
||||
if trace is None:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
|
||||
return trace.feedback
|
||||
|
||||
@router.delete("/", status_code=status.HTTP_204_NO_CONTENT)
|
||||
def delete_feedback(trace_id: str) -> Any:
|
||||
trace = get_context().tracing_database.get(trace_id)
|
||||
if trace is None:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
|
||||
|
||||
trace.feedback = None
|
||||
|
||||
get_context().tracing_database.update(trace_id, trace)
|
||||
return Response(status_code=status.HTTP_204_NO_CONTENT)
|
||||
|
||||
app.include_router(router)
|
||||
|
|
@ -1,44 +0,0 @@
|
|||
from typing import List
|
||||
|
||||
from fastapi import APIRouter, FastAPI, HTTPException, Response, status
|
||||
|
||||
from ...context import get_context
|
||||
from ...views import Query, Trace
|
||||
|
||||
|
||||
def bootstrap_trace_endpoints(app: FastAPI) -> None:
|
||||
router = APIRouter(
|
||||
prefix="/traces",
|
||||
tags=["traces"],
|
||||
)
|
||||
|
||||
@router.post("/", status_code=status.HTTP_200_OK, response_model=List[Trace])
|
||||
def query_traces(
|
||||
query: Query,
|
||||
skip: int = 0,
|
||||
take: int = 100,
|
||||
) -> List[Trace]:
|
||||
return get_context().tracing_database.query(
|
||||
conjunctive_filters=query.filter,
|
||||
conjunctive_tags=query.conjunctive_tags,
|
||||
since=query.since,
|
||||
until=query.until,
|
||||
has_feedback=query.has_feedback,
|
||||
sort_by=query.sort,
|
||||
skip=skip,
|
||||
take=take,
|
||||
)[0]
|
||||
|
||||
@router.get("/{trace_id}", status_code=status.HTTP_200_OK, response_model=Trace)
|
||||
def get_trace(trace_id: str) -> Trace:
|
||||
result = get_context().tracing_database.get(trace_id)
|
||||
if result is None:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
|
||||
return result
|
||||
|
||||
@router.delete("/{trace_id}", status_code=status.HTTP_204_NO_CONTENT)
|
||||
def delete_trace(trace_id: str) -> Response:
|
||||
get_context().tracing_database.delete(trace_id)
|
||||
return Response(status_code=status.HTTP_204_NO_CONTENT)
|
||||
|
||||
app.include_router(router)
|
||||
|
|
@ -1 +0,0 @@
|
|||
from .create_dash_app import create_dash_app
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 4.2 KiB |
|
|
@ -1,227 +0,0 @@
|
|||
:root {
|
||||
--important-color: #a30808;
|
||||
--background-color: #edf5f6;
|
||||
--small-padding: 10px;
|
||||
--medium-padding: 20px;
|
||||
--large-padding: 40px;
|
||||
--border-radius: 10px;
|
||||
--shadow: 0 4px 6px -1px rgb(0 0 0 / 10%), 0 2px 4px -1px rgb(0 0 0 / 6%);
|
||||
--disclaimer-width: 180px;
|
||||
--disclaimer-height: 35px;
|
||||
}
|
||||
|
||||
@media (max-width: 900px) {
|
||||
body {
|
||||
zoom: 0.8;
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 550px) {
|
||||
:root {
|
||||
--small-padding: 5px;
|
||||
--medium-padding: 10px;
|
||||
--large-padding: 20px;
|
||||
--border-radius: 8px;
|
||||
}
|
||||
|
||||
.environment {
|
||||
margin-top: calc(-1 * var(--large-padding));
|
||||
margin-bottom: var(--large-padding);
|
||||
}
|
||||
}
|
||||
|
||||
@media (min-width: 551px) {
|
||||
.environment {
|
||||
position: absolute;
|
||||
width: var(--disclaimer-width);
|
||||
height: var(--disclaimer-height);
|
||||
transform: rotate(-45deg);
|
||||
top: calc(
|
||||
var(--disclaimer-width) / 1.4142 - var(--disclaimer-height) / 1.4142
|
||||
);
|
||||
left: calc(-1 * var(--disclaimer-height) / 1.4142);
|
||||
transform-origin: top left;
|
||||
z-index: 100;
|
||||
}
|
||||
}
|
||||
|
||||
* {
|
||||
margin: 0;
|
||||
box-sizing: border-box;
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
body {
|
||||
background-color: var(--background-color);
|
||||
font-family: Arial, Helvetica, sans-serif;
|
||||
}
|
||||
|
||||
h1,
|
||||
h2,
|
||||
h3,
|
||||
h4,
|
||||
h5,
|
||||
h6 {
|
||||
margin: var(--medium-padding) 0 var(--small-padding) 0;
|
||||
}
|
||||
|
||||
h6 {
|
||||
margin-top: 0;
|
||||
font-size: 3rem;
|
||||
}
|
||||
|
||||
html,
|
||||
body,
|
||||
#react-entry-point,
|
||||
main {
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
main {
|
||||
padding-top: var(--large-padding);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.environment {
|
||||
background-color: var(--important-color);
|
||||
color: white;
|
||||
text-align: center;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
main > header,
|
||||
.configuration-container,
|
||||
.traces-table-container,
|
||||
.parallel-coordinates,
|
||||
main > footer {
|
||||
padding: var(--large-padding);
|
||||
flex-shrink: 0;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
main > header,
|
||||
.configuration-container,
|
||||
.traces-table-container,
|
||||
.parallel-coordinates {
|
||||
margin: 0 var(--large-padding) var(--large-padding) var(--large-padding);
|
||||
border-radius: var(--border-radius);
|
||||
box-shadow: var(--shadow);
|
||||
background-color: white;
|
||||
}
|
||||
|
||||
main > header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
flex-wrap: wrap;
|
||||
justify-content: space-between;
|
||||
}
|
||||
|
||||
main > header > div:nth-child(1) {
|
||||
min-width: 350px;
|
||||
max-width: 450px;
|
||||
margin-bottom: var(--large-padding);
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
main > header > div > h1 {
|
||||
margin-top: 0;
|
||||
}
|
||||
|
||||
main > header > *:nth-child(2) {
|
||||
min-width: 250px;
|
||||
max-width: 550px;
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
main > header .placeholder {
|
||||
opacity: 0.35;
|
||||
font-size: 1.5rem;
|
||||
text-align: center;
|
||||
display: block;
|
||||
min-width: 250px;
|
||||
width: 60%;
|
||||
margin: auto;
|
||||
}
|
||||
|
||||
.configuration-container {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.configuration-item {
|
||||
border-left: 2px solid var(--important-color);
|
||||
padding-left: var(--small-padding);
|
||||
margin: var(--medium-padding);
|
||||
}
|
||||
|
||||
.configuration-item h4 {
|
||||
font-weight: bold;
|
||||
margin: 0 0 var(--small-padding) 0;
|
||||
}
|
||||
|
||||
.traces-table-container {
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.traces-table-container header {
|
||||
padding: var(--large-padding);
|
||||
}
|
||||
|
||||
.traces-table-container header h2 {
|
||||
margin-top: 0;
|
||||
}
|
||||
|
||||
.dash-filter--case {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.traces-table-container td > div {
|
||||
white-space: pre !important;
|
||||
max-height: 150px !important;
|
||||
overflow: auto !important;
|
||||
display: inline-block !important;
|
||||
text-align: left !important;
|
||||
}
|
||||
|
||||
.traces-table-container th > div {
|
||||
text-align: left !important;
|
||||
}
|
||||
|
||||
.space-filler {
|
||||
flex-grow: 1;
|
||||
}
|
||||
|
||||
main > footer {
|
||||
opacity: 0.35;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
main > footer {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
padding: var(--large-padding);
|
||||
background-color: #ddd;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.parallel-coordinates {
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
a img {
|
||||
display: block;
|
||||
margin-left: var(--large-padding);
|
||||
width: 64px;
|
||||
height: 64px;
|
||||
cursor: pointer;
|
||||
transition: transform 300ms;
|
||||
}
|
||||
|
||||
a img:hover {
|
||||
transform: scale(1.1);
|
||||
}
|
||||
|
|
@ -1,254 +0,0 @@
|
|||
from math import ceil
|
||||
from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
|
||||
|
||||
import pandas as pd
|
||||
import plotly.express as px
|
||||
import plotly.graph_objects as go
|
||||
from dash import Dash, dcc, html
|
||||
from dash.dependencies import Input, Output
|
||||
from flask import Flask
|
||||
|
||||
from great_ai.utilities import unique
|
||||
|
||||
from ....constants import DASHBOARD_PATH, ONLINE_TAG_NAME
|
||||
from ....context import get_context
|
||||
from ....helper import snake_case_to_text, text_to_hex_color
|
||||
from ....views import SortBy, Trace
|
||||
from .get_description import get_description
|
||||
from .get_filter_from_datatable import get_filter_from_datatable
|
||||
from .get_footer import get_footer
|
||||
from .get_traces_table import get_traces_table
|
||||
|
||||
|
||||
def create_dash_app(function_name: str, function_docs: str) -> Flask:
|
||||
accent_color = text_to_hex_color(function_name)
|
||||
|
||||
app = Dash(
|
||||
function_name,
|
||||
requests_pathname_prefix=DASHBOARD_PATH + "/",
|
||||
server=Flask(__name__),
|
||||
title=snake_case_to_text(function_name),
|
||||
update_title=None,
|
||||
external_stylesheets=[
|
||||
"/assets/index.css",
|
||||
],
|
||||
)
|
||||
|
||||
app.layout = html.Main(
|
||||
[
|
||||
html.Div(
|
||||
html.P("PRODUCTION" if get_context().is_production else "DEVELOPMENT"),
|
||||
className="environment",
|
||||
),
|
||||
html.Header(
|
||||
[
|
||||
get_description(
|
||||
function_name=function_name,
|
||||
function_docs=function_docs,
|
||||
accent_color=accent_color,
|
||||
),
|
||||
execution_time_histogram_container := html.Div(),
|
||||
],
|
||||
),
|
||||
configuration_container := html.Div(
|
||||
className="configuration-container",
|
||||
),
|
||||
traces_table_container := html.Div(
|
||||
[
|
||||
html.Header(
|
||||
[
|
||||
html.H2("Latest traces"),
|
||||
html.P(
|
||||
"Recent traces and aggregated metrics are presented below. Try filtering the table."
|
||||
),
|
||||
html.A(
|
||||
"Filtering syntax.",
|
||||
href="https://dash.plotly.com/datatable/filtering",
|
||||
target="_blank",
|
||||
),
|
||||
]
|
||||
),
|
||||
table := get_traces_table(),
|
||||
],
|
||||
className="traces-table-container",
|
||||
),
|
||||
parallel_coordinates := dcc.Graph(
|
||||
className="parallel-coordinates", config={"displaylogo": False}
|
||||
),
|
||||
html.Div(className="space-filler"),
|
||||
get_footer(),
|
||||
interval := dcc.Interval(
|
||||
interval=4 * 1000, # in milliseconds
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
@app.callback(
|
||||
Output(configuration_container, "children"),
|
||||
Input(interval, "n_intervals"),
|
||||
)
|
||||
def update_configuration(
|
||||
n_intervals: int,
|
||||
) -> List[html.Div]:
|
||||
config = get_context().to_flat_dict()
|
||||
return [
|
||||
html.Div(
|
||||
[
|
||||
html.H4(snake_case_to_text(key)),
|
||||
html.P(str(value)),
|
||||
],
|
||||
className="configuration-item",
|
||||
)
|
||||
for key, value in config.items()
|
||||
]
|
||||
|
||||
@app.callback(
|
||||
Output(table, "data"),
|
||||
Output(table, "page_count"),
|
||||
Output(table, "columns"),
|
||||
Output(traces_table_container, "style"),
|
||||
Output(execution_time_histogram_container, "children"),
|
||||
Output(parallel_coordinates, "figure"),
|
||||
Output(parallel_coordinates, "style"),
|
||||
Input(table, "page_current"),
|
||||
Input(table, "page_size"),
|
||||
Input(table, "sort_by"),
|
||||
Input(table, "filter_query"),
|
||||
Input(interval, "n_intervals"),
|
||||
)
|
||||
def update_page(
|
||||
page_current: int,
|
||||
page_size: int,
|
||||
sort_by: List[Dict[str, Union[str, int]]],
|
||||
filter_query: str,
|
||||
n_intervals: int,
|
||||
) -> Tuple[
|
||||
List[Dict[str, Any]],
|
||||
int,
|
||||
List[Dict[str, Sequence[str]]],
|
||||
Dict[str, Any],
|
||||
Any,
|
||||
go.Figure,
|
||||
Dict[str, Any],
|
||||
]:
|
||||
conjunctive_filters = (
|
||||
[get_filter_from_datatable(f) for f in filter_query.split(" && ")]
|
||||
if filter_query
|
||||
else []
|
||||
)
|
||||
non_null_conjunctive_filters = [f for f in conjunctive_filters if f is not None]
|
||||
|
||||
elements, count = get_context().tracing_database.query(
|
||||
skip=page_current * page_size,
|
||||
take=page_size,
|
||||
conjunctive_filters=non_null_conjunctive_filters,
|
||||
conjunctive_tags=[ONLINE_TAG_NAME],
|
||||
sort_by=[SortBy.parse_obj(s) for s in sort_by],
|
||||
)
|
||||
|
||||
columns, style = update_layout(elements[0] if elements else None)
|
||||
execution_time_histogram, parallel_coords_fig, style = update_charts(
|
||||
elements=elements, function_name=function_name, accent_color=accent_color
|
||||
)
|
||||
|
||||
return (
|
||||
[e.to_flat_dict(include_original=False) for e in elements],
|
||||
max(1, ceil(count / page_size)),
|
||||
columns,
|
||||
style,
|
||||
execution_time_histogram,
|
||||
parallel_coords_fig,
|
||||
style,
|
||||
)
|
||||
|
||||
return app.server
|
||||
|
||||
|
||||
def update_layout(
|
||||
first_element: Optional[Trace],
|
||||
) -> Tuple[List[Dict[str, Sequence[str]]], Dict[str, Any]]:
|
||||
|
||||
if first_element:
|
||||
keys = list(first_element.to_flat_dict(include_original=False).keys())
|
||||
header_height = max(len(i.split(":")) for i in keys)
|
||||
columns = [
|
||||
{
|
||||
"name": [""] * (header_height - len(k.split(":")))
|
||||
+ k.replace("_flat", "").split(":"),
|
||||
"id": k,
|
||||
}
|
||||
for k in keys
|
||||
]
|
||||
else:
|
||||
columns = []
|
||||
|
||||
return (
|
||||
columns,
|
||||
{"display": "none" if first_element is None else "block"},
|
||||
)
|
||||
|
||||
|
||||
def update_charts(
|
||||
elements: List[Trace], function_name: str, accent_color: str
|
||||
) -> Tuple[Any, go.Figure, Dict[str, Any]]:
|
||||
if not elements:
|
||||
return (
|
||||
html.Span(
|
||||
f"No traces yet: call your function ({function_name}) to create one.",
|
||||
className="placeholder",
|
||||
),
|
||||
go.Figure(),
|
||||
{"display": "none"},
|
||||
)
|
||||
|
||||
flat_elements = [e.to_flat_dict(include_original=False) for e in elements]
|
||||
|
||||
execution_time_histogram = dcc.Graph(config={"displaylogo": False})
|
||||
df = pd.DataFrame(flat_elements)
|
||||
fig = px.histogram(
|
||||
df,
|
||||
x="original_execution_time_ms",
|
||||
labels={"original_execution_time_ms": "Execution time (ms)"},
|
||||
nbins=20,
|
||||
height=400,
|
||||
log_y=True,
|
||||
color_discrete_sequence=[accent_color],
|
||||
)
|
||||
fig.update_layout(
|
||||
margin=dict(l=0, r=0, b=0, t=0, pad=0),
|
||||
)
|
||||
execution_time_histogram.figure = fig
|
||||
|
||||
parallel_coords_fig = go.Figure(
|
||||
go.Parcoords(
|
||||
dimensions=[
|
||||
get_dimension_descriptor(df, c)
|
||||
for c in df.columns
|
||||
if c not in {"trace_id", "created", "output", "exception", "feedback"}
|
||||
and "_flat" not in c
|
||||
],
|
||||
line_color=accent_color,
|
||||
)
|
||||
)
|
||||
return execution_time_histogram, parallel_coords_fig, {}
|
||||
|
||||
|
||||
def get_dimension_descriptor(df: pd.DataFrame, column: str) -> Dict[str, Any]:
|
||||
dimension: Dict[str, Any] = {
|
||||
"label": snake_case_to_text(column),
|
||||
}
|
||||
|
||||
values = df[column]
|
||||
|
||||
try:
|
||||
dimension["values"] = [float(v) for v in values]
|
||||
except (TypeError, ValueError):
|
||||
MAX_LENGTH = 40
|
||||
unique_values = unique(values)
|
||||
value_mapping = {str(v)[-MAX_LENGTH:]: i for i, v in enumerate(unique_values)}
|
||||
|
||||
dimension["values"] = [value_mapping[str(v)[-MAX_LENGTH:]] for v in values]
|
||||
dimension["tickvals"] = list(value_mapping.values())
|
||||
dimension["ticktext"] = [k[-MAX_LENGTH:] for k in value_mapping.keys()]
|
||||
|
||||
return dimension
|
||||
|
|
@ -1,32 +0,0 @@
|
|||
from dash import dcc, html
|
||||
|
||||
from ....helper import snake_case_to_text, strip_lines
|
||||
|
||||
|
||||
def get_description(
|
||||
function_name: str, function_docs: str, accent_color: str
|
||||
) -> html.Div:
|
||||
return html.Div(
|
||||
[
|
||||
html.H1(
|
||||
f"{snake_case_to_text(function_name)} - dashboard",
|
||||
style={"color": accent_color},
|
||||
),
|
||||
dcc.Markdown(
|
||||
strip_lines(
|
||||
f"""
|
||||
> View the live data of your deployment here.
|
||||
|
||||
## Using the API
|
||||
|
||||
You can find the available endpoints at [/docs](/docs).
|
||||
|
||||
## Details
|
||||
|
||||
{function_docs}
|
||||
"""
|
||||
),
|
||||
className="description",
|
||||
),
|
||||
]
|
||||
)
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
from typing import Optional, Union
|
||||
|
||||
from ....views import Filter, operators
|
||||
|
||||
|
||||
def get_filter_from_datatable(description: str) -> Optional[Filter]:
|
||||
for operator in operators:
|
||||
if operator in description:
|
||||
name_part, value_part = description.split(operator, 1)
|
||||
value_part = value_part.strip()
|
||||
name_part = name_part[name_part.find("{") + 1 : name_part.rfind("}")]
|
||||
|
||||
v0 = value_part[0]
|
||||
if v0 == value_part[-1] and v0 in ("'", '"', "`"):
|
||||
value: Union[str, float] = value_part[1:-1].replace("\\" + v0, v0)
|
||||
else:
|
||||
try:
|
||||
value = float(value_part)
|
||||
except ValueError:
|
||||
value = value_part
|
||||
return Filter(property=name_part, operator=operator, value=value)
|
||||
|
||||
return None
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
from dash import html
|
||||
|
||||
from ....constants import GITHUB_LINK
|
||||
|
||||
|
||||
def get_footer() -> html.Footer:
|
||||
return html.Footer(
|
||||
[
|
||||
html.Div(
|
||||
[
|
||||
html.H6("GreatAI"),
|
||||
html.P(
|
||||
"A human-friendly framework for robust end-to-end AI deployments."
|
||||
),
|
||||
]
|
||||
),
|
||||
html.A(
|
||||
html.Img(src="/assets/github.png"),
|
||||
href=GITHUB_LINK,
|
||||
target="_blank",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
from dash import dash_table
|
||||
|
||||
from great_ai.great_ai.context import get_context
|
||||
|
||||
|
||||
def get_traces_table() -> dash_table.DataTable:
|
||||
return dash_table.DataTable(
|
||||
page_current=0,
|
||||
page_size=get_context().dashboard_table_size,
|
||||
page_action="custom",
|
||||
filter_action="custom",
|
||||
sort_action="custom",
|
||||
sort_mode="multi",
|
||||
sort_by=[
|
||||
{"column_id": "created", "direction": "desc"},
|
||||
],
|
||||
style_data={
|
||||
"white-space": "normal",
|
||||
"height": "auto",
|
||||
"max-height": "300px",
|
||||
"overflow": "hidden",
|
||||
"text-overflow": "ellipsis",
|
||||
},
|
||||
style_cell={"padding": "5px"},
|
||||
style_header={
|
||||
"background-color": "white",
|
||||
"font-weight": "bold",
|
||||
},
|
||||
merge_duplicate_headers=True,
|
||||
style_cell_conditional=[
|
||||
{"if": {"column_id": "output"}, "width": 1500},
|
||||
],
|
||||
)
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
from .argument_validation_error import ArgumentValidationError
|
||||
from .missing_argument_error import MissingArgumentError
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
class ArgumentValidationError(Exception):
|
||||
pass
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
class MissingArgumentError(Exception):
|
||||
pass
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
from .freeze_arguments import freeze_arguments
|
||||
from .get_arguments import get_arguments
|
||||
from .get_function_metadata_store import get_function_metadata_store
|
||||
from .hashable_base_model import HashableBaseModel
|
||||
from .snake_case_to_text import snake_case_to_text
|
||||
from .strip_lines import strip_lines
|
||||
from .text_to_hex_color import text_to_hex_color
|
||||
from .use_http_exceptions import use_http_exceptions
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
from typing import Any, Callable
|
||||
|
||||
from ..context import get_context
|
||||
from .get_function_metadata_store import get_function_metadata_store
|
||||
|
||||
|
||||
def assert_function_is_not_finalised(func: Callable[..., Any]) -> None:
|
||||
error_message = (
|
||||
"The outer-most (first) decorator has to be `@GreatAI.deploy`. "
|
||||
+ f"In the case of `{func.__name__}`, it is not: fix this by moving `@GreatAI.deploy` to the top."
|
||||
)
|
||||
|
||||
if get_function_metadata_store(func).is_finalised:
|
||||
get_context().logger.error(error_message)
|
||||
exit(-1)
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
from functools import wraps
|
||||
from typing import Any, Callable, Dict, List
|
||||
|
||||
|
||||
class FrozenDict(dict):
|
||||
def __hash__(self) -> int: # type: ignore
|
||||
return hash(frozenset(self.items()))
|
||||
|
||||
|
||||
def freeze_arguments(func: Callable[..., Any]) -> Callable[..., Any]:
|
||||
"""Transform mutable dictionary
|
||||
Into immutable
|
||||
Useful to be compatible with cache
|
||||
source: https://stackoverflow.com/questions/6358481/using-functools-lru-cache-with-dictionary-arguments
|
||||
"""
|
||||
|
||||
@wraps(func)
|
||||
def wrapper(*args: List[Any], **kwargs: Dict[str, Any]) -> Any:
|
||||
args = tuple(FrozenDict(arg) if isinstance(arg, dict) else arg for arg in args)
|
||||
kwargs = {
|
||||
k: FrozenDict(v) if isinstance(v, dict) else v for k, v in kwargs.items()
|
||||
}
|
||||
return func(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
import inspect
|
||||
from typing import Any, Callable, Dict, Mapping, Sequence
|
||||
|
||||
|
||||
def get_arguments(
|
||||
func: Callable[..., Any], args: Sequence[Any], kwargs: Mapping[str, Any]
|
||||
) -> Dict[str, Any]:
|
||||
"""Return mapping from parameter names to actual argument values"""
|
||||
|
||||
signature = inspect.signature(func)
|
||||
|
||||
defaults = {
|
||||
p.name: p.default
|
||||
for p in signature.parameters.values()
|
||||
if p.default != inspect._empty
|
||||
}
|
||||
|
||||
filter_keys = [
|
||||
param.name
|
||||
for param in signature.parameters.values()
|
||||
if param.kind == param.POSITIONAL_OR_KEYWORD
|
||||
]
|
||||
|
||||
return {**defaults, **dict(zip(filter_keys, args)), **kwargs}
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
from typing import Any, Callable, cast
|
||||
|
||||
from ..views.function_metadata import FunctionMetadata
|
||||
|
||||
|
||||
def get_function_metadata_store(func: Callable[..., Any]) -> FunctionMetadata:
|
||||
any_func = cast(Any, func)
|
||||
|
||||
if not hasattr(any_func, "_great_ai_metadata"):
|
||||
any_func._great_ai_metadata = FunctionMetadata()
|
||||
|
||||
return any_func._great_ai_metadata
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class HashableBaseModel(BaseModel):
|
||||
def __hash__(self) -> int:
|
||||
return hash((type(self),) + tuple(self.__dict__.values()))
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
def snake_case_to_text(snake_case: str) -> str:
|
||||
return snake_case.capitalize().replace("_", " ")
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
def strip_lines(text: str) -> str:
|
||||
return "\n".join(line.strip() for line in text.split("\n"))
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
import colorsys
|
||||
from hashlib import md5
|
||||
|
||||
|
||||
def text_to_hex_color(text: str) -> str:
|
||||
ascii_bytes = text.encode("ascii")
|
||||
digest = md5(
|
||||
ascii_bytes
|
||||
).hexdigest() # the built-in hash function is salted differently in each process
|
||||
integer = int(digest, 16)
|
||||
hue = integer % 6311 / 6311.0
|
||||
rgb = colorsys.hsv_to_rgb(hue, 0.75, 0.6)
|
||||
return "#" + "".join("%02X" % round(i * 255) for i in rgb)
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
from functools import wraps
|
||||
from typing import Any, Callable, Dict, List, TypeVar, cast
|
||||
|
||||
from fastapi import HTTPException, status
|
||||
|
||||
F = TypeVar("F", bound=Callable[..., Any])
|
||||
|
||||
|
||||
def use_http_exceptions(func: F) -> F:
|
||||
@wraps(func)
|
||||
def wrapper(*args: List[Any], **kwargs: Dict[str, Any]) -> Any:
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"The following exception has occurred: {type(e).__name__}: {e}",
|
||||
)
|
||||
|
||||
return cast(F, wrapper)
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
from .save_model import save_model
|
||||
from .use_model import use_model
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
from typing import Any, Optional, Tuple
|
||||
|
||||
from joblib import load
|
||||
|
||||
from ..context import get_context
|
||||
|
||||
|
||||
def load_model(
|
||||
key: str, version: Optional[int] = None, return_path: bool = False
|
||||
) -> Tuple[Any, int]:
|
||||
file = get_context().large_file_implementation(name=key, mode="rb", version=version)
|
||||
|
||||
if return_path:
|
||||
return file.get(), file.version
|
||||
|
||||
with file as f:
|
||||
return load(f), file.version
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
from pathlib import Path
|
||||
from typing import Optional, Union
|
||||
|
||||
from joblib import dump
|
||||
|
||||
from ..context import get_context
|
||||
|
||||
|
||||
def save_model(
|
||||
model: Union[Path, str, object], key: str, *, keep_last_n: Optional[int] = None
|
||||
) -> str:
|
||||
file = get_context().large_file_implementation(
|
||||
name=key, mode="wb", keep_last_n=keep_last_n
|
||||
)
|
||||
|
||||
if isinstance(model, Path) or isinstance(model, str):
|
||||
file.push(model)
|
||||
else:
|
||||
with file as f:
|
||||
dump(model, f)
|
||||
|
||||
get_context().logger.info(f"Model {key} uploaded with version {file.version}")
|
||||
|
||||
return f"{key}:{file.version}"
|
||||
|
|
@ -1,48 +0,0 @@
|
|||
from functools import wraps
|
||||
from typing import Any, Callable, Dict, List, Literal, TypeVar, Union, cast
|
||||
|
||||
from ..helper import get_function_metadata_store
|
||||
from ..helper.assert_function_is_not_finalised import assert_function_is_not_finalised
|
||||
from ..tracing.tracing_context import TracingContext
|
||||
from ..views import Model
|
||||
from .load_model import load_model
|
||||
|
||||
F = TypeVar("F", bound=Callable[..., Any])
|
||||
|
||||
|
||||
def use_model(
|
||||
key: str,
|
||||
*,
|
||||
version: Union[int, Literal["latest"]],
|
||||
return_path: bool = False,
|
||||
model_kwarg_name: str = "model",
|
||||
) -> Callable[[F], F]:
|
||||
assert (
|
||||
isinstance(version, int) or version == "latest"
|
||||
), "Only integers or the string literal `latest` is allowed as version"
|
||||
|
||||
model, actual_version = load_model(
|
||||
key=key,
|
||||
version=None if version == "latest" else version,
|
||||
return_path=return_path,
|
||||
)
|
||||
|
||||
def decorator(func: F) -> F:
|
||||
assert_function_is_not_finalised(func)
|
||||
|
||||
store = get_function_metadata_store(func)
|
||||
store.model_parameter_names.append(model_kwarg_name)
|
||||
if store.model_versions:
|
||||
store.model_versions += "."
|
||||
store.model_versions += f"{key}-v{actual_version}"
|
||||
|
||||
@wraps(func)
|
||||
def wrapper(*args: List[Any], **kwargs: Dict[str, Any]) -> Any:
|
||||
tracing_context = TracingContext.get_current_tracing_context()
|
||||
if tracing_context:
|
||||
tracing_context.log_model(Model(key=key, version=actual_version))
|
||||
return func(*args, **kwargs, **{model_kwarg_name: model})
|
||||
|
||||
return cast(F, wrapper)
|
||||
|
||||
return decorator
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
from .classification_output import ClassificationOutput
|
||||
from .multi_label_classification_output import MultiLabelClassificationOutput
|
||||
from .regression_output import RegressionOutput
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
from typing import Any, Optional, Union
|
||||
|
||||
from ..helper import HashableBaseModel
|
||||
|
||||
|
||||
class ClassificationOutput(HashableBaseModel):
|
||||
label: Union[str, int]
|
||||
confidence: float
|
||||
explanation: Optional[Any]
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
from typing import List
|
||||
|
||||
from ..helper import HashableBaseModel
|
||||
from .classification_output import ClassificationOutput
|
||||
|
||||
|
||||
class MultiLabelClassificationOutput(HashableBaseModel):
|
||||
labels: List[ClassificationOutput] = []
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
from typing import Any, Optional, Union
|
||||
|
||||
from ..helper import HashableBaseModel
|
||||
|
||||
|
||||
class RegressionOutput(HashableBaseModel):
|
||||
value: Union[int, float]
|
||||
explanation: Optional[Any]
|
||||
|
|
@ -1 +0,0 @@
|
|||
# todo
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
from .automatically_decorate_parameters import automatically_decorate_parameters
|
||||
from .log_metric import log_metric
|
||||
from .parameter import parameter
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
import inspect
|
||||
from typing import Any, Callable, TypeVar
|
||||
|
||||
from great_ai.great_ai.helper.get_function_metadata_store import (
|
||||
get_function_metadata_store,
|
||||
)
|
||||
|
||||
from .parameter import parameter
|
||||
|
||||
F = TypeVar("F", bound=Callable[..., Any])
|
||||
|
||||
|
||||
def automatically_decorate_parameters(func: F) -> F:
|
||||
signature = inspect.signature(func)
|
||||
parameter_names = [
|
||||
param.name
|
||||
for param in signature.parameters.values()
|
||||
if param.kind == param.POSITIONAL_OR_KEYWORD
|
||||
]
|
||||
|
||||
metadata = get_function_metadata_store(func)
|
||||
|
||||
for name in parameter_names:
|
||||
if (
|
||||
name not in metadata.model_parameter_names
|
||||
and name not in metadata.input_parameter_names
|
||||
):
|
||||
func = parameter(name)(func)
|
||||
|
||||
return func
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
import inspect
|
||||
from typing import Any
|
||||
|
||||
from ..context import get_context
|
||||
from ..tracing import TracingContext
|
||||
|
||||
|
||||
def log_metric(argument_name: str, value: Any) -> None:
|
||||
tracing_context = TracingContext.get_current_tracing_context()
|
||||
caller = inspect.stack()[1].function
|
||||
actual_name = f"metric:{caller}:{argument_name}"
|
||||
if tracing_context:
|
||||
tracing_context.log_value(name=actual_name, value=value)
|
||||
|
||||
get_context().logger.info(f"{actual_name}={value}")
|
||||
|
|
@ -1,51 +0,0 @@
|
|||
from functools import wraps
|
||||
from typing import Any, Callable, Dict, TypeVar, cast
|
||||
|
||||
from ..exceptions import ArgumentValidationError
|
||||
from ..helper import get_arguments, get_function_metadata_store
|
||||
from ..helper.assert_function_is_not_finalised import assert_function_is_not_finalised
|
||||
from ..tracing.tracing_context import TracingContext
|
||||
|
||||
F = TypeVar("F", bound=Callable[..., Any])
|
||||
|
||||
|
||||
def parameter(
|
||||
parameter_name: str,
|
||||
*,
|
||||
validator: Callable[[Any], bool] = lambda _: True,
|
||||
disable_logging: bool = False,
|
||||
) -> Callable[[F], F]:
|
||||
def decorator(func: F) -> F:
|
||||
get_function_metadata_store(func).input_parameter_names.append(parameter_name)
|
||||
assert_function_is_not_finalised(func)
|
||||
|
||||
actual_name = f"arg:{parameter_name}"
|
||||
|
||||
@wraps(func)
|
||||
def wrapper(*args: Any, **kwargs: Dict[str, Any]) -> Any:
|
||||
arguments = get_arguments(func, args, kwargs)
|
||||
argument = arguments[parameter_name]
|
||||
|
||||
expected_type = func.__annotations__.get(parameter_name)
|
||||
|
||||
if expected_type is not None and not isinstance(argument, expected_type):
|
||||
raise ArgumentValidationError(
|
||||
f"Argument {parameter_name} in {func.__name__} has the wrong type, expected: {expected_type.__name__}, got: {type(argument).__name__}"
|
||||
)
|
||||
|
||||
if not validator(argument):
|
||||
raise ArgumentValidationError(
|
||||
f"Argument {parameter_name} in {func.__name__} did not pass validation"
|
||||
)
|
||||
|
||||
context = TracingContext.get_current_tracing_context()
|
||||
if context and not disable_logging:
|
||||
context.log_value(name=f"{actual_name}:value", value=argument)
|
||||
if isinstance(argument, str):
|
||||
context.log_value(name=f"{actual_name}:length", value=len(argument))
|
||||
|
||||
return func(*args, **kwargs)
|
||||
|
||||
return cast(F, wrapper)
|
||||
|
||||
return decorator
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
from .mongodb_driver import MongodbDriver
|
||||
from .parallel_tinydb_driver import ParallelTinyDbDriver
|
||||
from .tracing_database_driver import TracingDatabaseDriver
|
||||
|
|
@ -1,137 +0,0 @@
|
|||
from datetime import datetime
|
||||
from typing import Any, List, Mapping, Optional, Sequence, Tuple
|
||||
|
||||
from pymongo import MongoClient
|
||||
|
||||
from ..views import Filter, SortBy, Trace
|
||||
from .tracing_database_driver import TracingDatabaseDriver
|
||||
|
||||
operator_mapping = {
|
||||
"=": "$eq",
|
||||
"!=": "$ne",
|
||||
"<": "$lt",
|
||||
"<=": "$lte",
|
||||
">": "$gt",
|
||||
">=": "$gte",
|
||||
"contains": "$regex",
|
||||
}
|
||||
|
||||
|
||||
class MongodbDriver(TracingDatabaseDriver):
|
||||
is_production_ready = True
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
if self.mongo_connection_string is None or self.mongo_database is None:
|
||||
raise ValueError(
|
||||
"Please configure the MongoDB access options by calling MongodbDriver.configure_credentials"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def configure_credentials( # type: ignore
|
||||
cls,
|
||||
*,
|
||||
mongo_connection_string: str,
|
||||
mongo_database: str,
|
||||
**_: Mapping[str, Any],
|
||||
) -> None:
|
||||
cls.mongo_connection_string = mongo_connection_string
|
||||
cls.mongo_database = mongo_database
|
||||
super().configure_credentials()
|
||||
|
||||
def save(self, trace: Trace) -> str:
|
||||
serialized = trace.to_flat_dict()
|
||||
serialized["_id"] = trace.trace_id
|
||||
|
||||
with MongoClient(self.mongo_connection_string) as client:
|
||||
return client[self.mongo_database].traces.insert_one(serialized)
|
||||
|
||||
def save_batch(self, documents: List[Trace]) -> List[str]:
|
||||
serialized = [d.to_flat_dict() for d in documents]
|
||||
for s in serialized:
|
||||
s["_id"] = s["trace_id"]
|
||||
|
||||
with MongoClient(self.mongo_connection_string) as client:
|
||||
return client[self.mongo_database].traces.insert_many(
|
||||
serialized, ordered=False
|
||||
)
|
||||
|
||||
def get(self, id: str) -> Optional[Trace]:
|
||||
with MongoClient(self.mongo_connection_string) as client:
|
||||
value = client[self.mongo_database].traces.find_one(id)
|
||||
|
||||
if value:
|
||||
value = Trace.parse_obj(value)
|
||||
|
||||
return value
|
||||
|
||||
def _get_operator(self, filter: Filter) -> str:
|
||||
if filter.operator == "contains" and not isinstance(filter.value, str):
|
||||
return operator_mapping["="]
|
||||
return operator_mapping[filter.operator]
|
||||
|
||||
def query(
|
||||
self,
|
||||
*,
|
||||
skip: int = 0,
|
||||
take: Optional[int] = None,
|
||||
conjunctive_filters: Sequence[Filter] = [],
|
||||
conjunctive_tags: Sequence[str] = [],
|
||||
since: Optional[datetime] = None,
|
||||
until: Optional[datetime] = None,
|
||||
has_feedback: Optional[bool] = None,
|
||||
sort_by: Sequence[SortBy] = [],
|
||||
) -> Tuple[List[Trace], int]:
|
||||
|
||||
query = {
|
||||
"filter": {
|
||||
"$and": [{"tags": tag} for tag in conjunctive_tags]
|
||||
+ [
|
||||
{f.property: {self._get_operator(f): f.value}}
|
||||
for f in conjunctive_filters
|
||||
]
|
||||
+ [{}]
|
||||
},
|
||||
"sort": [
|
||||
(col.column_id, 1 if col.direction == "asc" else -1) for col in sort_by
|
||||
],
|
||||
}
|
||||
|
||||
if skip:
|
||||
query["skip"] = skip
|
||||
|
||||
if take:
|
||||
query["limit"] = take
|
||||
|
||||
if since:
|
||||
query["filter"]["$and"].append({"created": {"$gte": since}})
|
||||
|
||||
if until:
|
||||
query["filter"]["$and"].append({"created": {"$lte": until}})
|
||||
|
||||
if has_feedback is not None:
|
||||
query["filter"]["$and"].append(
|
||||
{"feedback": {"$ne": None}} if has_feedback else {"feedback": None}
|
||||
)
|
||||
|
||||
with MongoClient(self.mongo_connection_string) as client:
|
||||
values = client[self.mongo_database].traces.find(**query)
|
||||
documents = [Trace.parse_obj(t) for t in values]
|
||||
|
||||
return documents, len(documents)
|
||||
|
||||
def update(self, id: str, new_version: Trace) -> None:
|
||||
serialized = new_version.dict()
|
||||
serialized["_id"] = new_version.trace_id
|
||||
with MongoClient(self.mongo_connection_string) as client:
|
||||
client[self.mongo_database].traces.update_one(id, new_version)
|
||||
|
||||
def delete(self, id: str) -> None:
|
||||
with MongoClient(self.mongo_connection_string) as client:
|
||||
client[self.mongo_database].traces.delete_one(id)
|
||||
|
||||
def delete_batch(self, ids: List[str]) -> List[str]:
|
||||
delete_filter = {"_id": {"$in": ids}}
|
||||
|
||||
with MongoClient(self.mongo_connection_string) as client:
|
||||
return client[self.mongo_database].traces.delete_many(delete_filter)
|
||||
|
|
@ -1,112 +0,0 @@
|
|||
from datetime import datetime
|
||||
from multiprocessing import Lock
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, cast
|
||||
|
||||
import pandas as pd
|
||||
from tinydb import TinyDB
|
||||
|
||||
from ..views import Filter, SortBy, Trace
|
||||
from .tracing_database_driver import TracingDatabaseDriver
|
||||
|
||||
DEFAULT_TRACING_DB_FILENAME = "tracing_database.json"
|
||||
lock = Lock()
|
||||
|
||||
|
||||
operator_mapping = {"=": "eq", "!=": "ne", "<": "lt", "<=": "le", ">": "gt", ">=": "ge"}
|
||||
|
||||
|
||||
class ParallelTinyDbDriver(TracingDatabaseDriver):
|
||||
is_production_ready = False
|
||||
path_to_db = Path(DEFAULT_TRACING_DB_FILENAME)
|
||||
|
||||
def save(self, trace: Trace) -> str:
|
||||
return self._safe_execute(lambda db: db.insert(trace.dict()))
|
||||
|
||||
def save_batch(self, documents: List[Trace]) -> List[str]:
|
||||
traces = [d.dict() for d in documents]
|
||||
return self._safe_execute(lambda db: db.insert_multiple(traces))
|
||||
|
||||
def get(self, id: str) -> Optional[Trace]:
|
||||
value = self._safe_execute(lambda db: db.get(lambda d: d["trace_id"] == id))
|
||||
if value:
|
||||
value = Trace.parse_obj(value)
|
||||
return value
|
||||
|
||||
def query(
|
||||
self,
|
||||
*,
|
||||
skip: int = 0,
|
||||
take: Optional[int] = None,
|
||||
conjunctive_filters: Sequence[Filter] = [],
|
||||
conjunctive_tags: Sequence[str] = [],
|
||||
since: Optional[datetime] = None,
|
||||
until: Optional[datetime] = None,
|
||||
has_feedback: Optional[bool] = None,
|
||||
sort_by: Sequence[SortBy] = []
|
||||
) -> Tuple[List[Trace], int]:
|
||||
def does_match(d: Dict[str, Any]) -> bool:
|
||||
return (
|
||||
not set(conjunctive_tags) - set(d["tags"])
|
||||
and (
|
||||
since is None
|
||||
or cast(datetime, datetime.fromisoformat(d["created"])) >= since
|
||||
)
|
||||
and (
|
||||
until is None
|
||||
or cast(datetime, datetime.fromisoformat(d["created"])) <= until
|
||||
)
|
||||
and (
|
||||
has_feedback is None or has_feedback == (d["feedback"] is not None)
|
||||
)
|
||||
)
|
||||
|
||||
documents: List[Trace] = [
|
||||
Trace.parse_obj(t)
|
||||
for t in self._safe_execute(lambda db: db.search(does_match))
|
||||
]
|
||||
|
||||
if not documents:
|
||||
return [], 0
|
||||
|
||||
df = pd.DataFrame([d.to_flat_dict() for d in documents])
|
||||
|
||||
for f in conjunctive_filters:
|
||||
operator = f.operator.lower()
|
||||
if operator in operator_mapping:
|
||||
df = df.loc[
|
||||
getattr(df[f.property], operator_mapping[f.operator])(f.value)
|
||||
]
|
||||
elif operator == "contains":
|
||||
df = df.loc[df[f.property].str.contains(f.value, case=False)]
|
||||
|
||||
if sort_by:
|
||||
df.sort_values(
|
||||
[col["column_id"] for col in sort_by],
|
||||
ascending=[col["direction"] == "asc" for col in sort_by],
|
||||
inplace=True,
|
||||
)
|
||||
|
||||
count = len(df)
|
||||
result = df.iloc[skip:] if take is None else df.iloc[skip : skip + take]
|
||||
return [
|
||||
next(d for d in documents if d.trace_id == trace_id)
|
||||
for trace_id in result["trace_id"]
|
||||
], count
|
||||
|
||||
def update(self, id: str, new_version: Trace) -> None:
|
||||
self._safe_execute(
|
||||
lambda db: db.update(new_version.dict(), lambda d: d["trace_id"] == id)
|
||||
)
|
||||
|
||||
def delete(self, id: str) -> None:
|
||||
self._safe_execute(lambda db: db.remove(lambda d: d["trace_id"] == id))
|
||||
|
||||
def delete_batch(self, ids: List[str]) -> List[str]:
|
||||
for i in ids:
|
||||
self.delete(i)
|
||||
|
||||
def _safe_execute(self, func: Callable[[TinyDB], Any]) -> Any:
|
||||
with lock:
|
||||
with TinyDB(self.path_to_db) as db:
|
||||
return func(db)
|
||||
|
|
@ -1,71 +0,0 @@
|
|||
from abc import ABC, abstractclassmethod, abstractmethod
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Sequence, Tuple, Union
|
||||
|
||||
from great_ai.utilities import ConfigFile
|
||||
|
||||
from ..views import Filter, SortBy, Trace
|
||||
|
||||
|
||||
class TracingDatabaseDriver(ABC):
|
||||
is_production_ready: bool
|
||||
initialized: bool = False
|
||||
|
||||
@classmethod
|
||||
def configure_credentials_from_file(
|
||||
cls,
|
||||
secrets_path: Union[Path, str],
|
||||
) -> None:
|
||||
cls.configure_credentials(**ConfigFile(secrets_path))
|
||||
|
||||
@abstractclassmethod
|
||||
def configure_credentials(
|
||||
cls,
|
||||
) -> None:
|
||||
cls.initialized = True
|
||||
|
||||
@abstractmethod
|
||||
def save(self, document: Trace) -> str:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def save_batch(
|
||||
self,
|
||||
documents: List[Trace],
|
||||
) -> List[str]:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get(self, id: str) -> Optional[Trace]:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def query(
|
||||
self,
|
||||
*,
|
||||
skip: int = 0,
|
||||
take: Optional[int] = None,
|
||||
conjunctive_filters: Sequence[Filter] = [],
|
||||
conjunctive_tags: Sequence[str] = [],
|
||||
until: Optional[datetime] = None,
|
||||
since: Optional[datetime] = None,
|
||||
has_feedback: Optional[bool] = None,
|
||||
sort_by: Sequence[SortBy] = []
|
||||
) -> Tuple[List[Trace], int]:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def update(self, id: str, new_version: Trace) -> None:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def delete(self, id: str) -> None:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def delete_batch(
|
||||
self,
|
||||
ids: List[str],
|
||||
) -> None:
|
||||
pass
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
from .add_ground_truth import add_ground_truth
|
||||
from .delete_ground_truth import delete_ground_truth
|
||||
from .query_ground_truth import query_ground_truth
|
||||
from .tracing_context import TracingContext
|
||||
|
|
@ -1,67 +0,0 @@
|
|||
from datetime import datetime
|
||||
from math import ceil
|
||||
from random import shuffle
|
||||
from typing import Any, Iterable, List, TypeVar
|
||||
|
||||
from ..constants import (
|
||||
GROUND_TRUTH_TAG_NAME,
|
||||
TEST_SPLIT_TAG_NAME,
|
||||
TRAIN_SPLIT_TAG_NAME,
|
||||
VALIDATION_SPLIT_TAG_NAME,
|
||||
)
|
||||
from ..context import get_context
|
||||
from ..views import Trace
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
def add_ground_truth(
|
||||
inputs: Iterable[Any],
|
||||
expected_outputs: Iterable[T],
|
||||
*,
|
||||
tags: List[str] = [],
|
||||
train_split_ratio: float,
|
||||
test_split_ratio: float,
|
||||
validation_split_ratio: float = 0
|
||||
) -> None:
|
||||
get_context() # this resets the seed
|
||||
|
||||
inputs = list(inputs)
|
||||
expected_outputs = list(expected_outputs)
|
||||
assert len(inputs) == len(
|
||||
expected_outputs
|
||||
), "The length of the inputs and expected_outputs must be equal"
|
||||
|
||||
sum_ratio = train_split_ratio + test_split_ratio + validation_split_ratio
|
||||
assert sum_ratio > 0, "The sum of the split ratios must be a positive number"
|
||||
|
||||
train_split_ratio /= sum_ratio
|
||||
test_split_ratio /= sum_ratio
|
||||
validation_split_ratio /= sum_ratio
|
||||
|
||||
values = list(zip(inputs, expected_outputs))
|
||||
shuffle(values)
|
||||
|
||||
split_tags = (
|
||||
[TRAIN_SPLIT_TAG_NAME] * ceil(train_split_ratio * len(inputs))
|
||||
+ [TEST_SPLIT_TAG_NAME] * ceil(test_split_ratio * len(inputs))
|
||||
+ [VALIDATION_SPLIT_TAG_NAME] * ceil(validation_split_ratio * len(inputs))
|
||||
)
|
||||
shuffle(split_tags)
|
||||
|
||||
created = datetime.utcnow().isoformat()
|
||||
traces = [
|
||||
Trace(
|
||||
created=created,
|
||||
original_execution_time_ms=0,
|
||||
logged_values=X if isinstance(X, dict) else {"input": X},
|
||||
models=[],
|
||||
output=y,
|
||||
feedback=y,
|
||||
exception=None,
|
||||
tags=[GROUND_TRUTH_TAG_NAME, split_tag, *tags],
|
||||
)
|
||||
for ((X, y), split_tag) in zip(values, split_tags)
|
||||
]
|
||||
|
||||
get_context().tracing_database.save_batch(traces)
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
from datetime import datetime
|
||||
from typing import List, Optional, Union
|
||||
|
||||
from ..context import get_context
|
||||
|
||||
|
||||
def delete_ground_truth(
|
||||
conjunctive_tags: Union[List[str], str] = [],
|
||||
*,
|
||||
until: Optional[datetime] = None,
|
||||
since: Optional[datetime] = None,
|
||||
) -> None:
|
||||
tags = (
|
||||
conjunctive_tags if isinstance(conjunctive_tags, list) else [conjunctive_tags]
|
||||
)
|
||||
db = get_context().tracing_database
|
||||
|
||||
items, length = db.query(
|
||||
conjunctive_tags=tags, until=until, since=since, has_feedback=True
|
||||
)
|
||||
|
||||
db.delete_batch([i.trace_id for i in items])
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
from datetime import datetime
|
||||
from typing import List, Optional, Union
|
||||
|
||||
from ..context import get_context
|
||||
from ..views import Trace
|
||||
|
||||
|
||||
def query_ground_truth(
|
||||
conjunctive_tags: Union[List[str], str] = [],
|
||||
*,
|
||||
since: Optional[datetime] = None,
|
||||
return_max_count: Optional[int] = None
|
||||
) -> List[Trace]:
|
||||
tags = (
|
||||
conjunctive_tags if isinstance(conjunctive_tags, list) else [conjunctive_tags]
|
||||
)
|
||||
db = get_context().tracing_database
|
||||
|
||||
items, length = db.query(
|
||||
conjunctive_tags=tags, since=since, take=return_max_count, has_feedback=True
|
||||
)
|
||||
return items
|
||||
|
|
@ -1,95 +0,0 @@
|
|||
import threading
|
||||
from collections import defaultdict
|
||||
from datetime import datetime
|
||||
from types import TracebackType
|
||||
from typing import (
|
||||
Any,
|
||||
DefaultDict,
|
||||
Dict,
|
||||
Generic,
|
||||
List,
|
||||
Literal,
|
||||
Optional,
|
||||
Type,
|
||||
TypeVar,
|
||||
)
|
||||
|
||||
from ..constants import DEVELOPMENT_TAG_NAME, ONLINE_TAG_NAME, PRODUCTION_TAG_NAME
|
||||
from ..context import get_context
|
||||
from ..views import Model, Trace
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
class TracingContext(Generic[T]):
|
||||
_contexts: DefaultDict[int, List["TracingContext"]] = defaultdict(lambda: [])
|
||||
|
||||
def __init__(self, function_name: str) -> None:
|
||||
self._models: List[Model] = []
|
||||
self._values: Dict[str, Any] = {}
|
||||
self._trace: Optional[Trace[T]] = None
|
||||
self._start_time = datetime.utcnow()
|
||||
self._name = function_name
|
||||
|
||||
def log_value(self, name: str, value: Any) -> None:
|
||||
self._values[name] = value
|
||||
|
||||
def log_model(self, model: Model) -> None:
|
||||
self._models.append(model)
|
||||
|
||||
def finalise(self, output: T = None, exception: BaseException = None) -> Trace[T]:
|
||||
assert self._trace is None, "has been already finalised"
|
||||
|
||||
delta_time = (datetime.utcnow() - self._start_time).microseconds / 1000
|
||||
self._trace = Trace(
|
||||
created=self._start_time.isoformat(),
|
||||
original_execution_time_ms=delta_time,
|
||||
logged_values=self._values,
|
||||
models=self._models,
|
||||
output=output,
|
||||
exception=None
|
||||
if exception is None
|
||||
else f"{type(exception).__name__}: {exception}",
|
||||
tags=[
|
||||
self._name,
|
||||
ONLINE_TAG_NAME,
|
||||
PRODUCTION_TAG_NAME
|
||||
if get_context().is_production
|
||||
else DEVELOPMENT_TAG_NAME,
|
||||
],
|
||||
)
|
||||
|
||||
return self._trace
|
||||
|
||||
@classmethod
|
||||
def get_current_tracing_context(cls) -> Optional["TracingContext"]:
|
||||
if cls._contexts[threading.get_ident()]:
|
||||
return cls._contexts[threading.get_ident()][-1]
|
||||
return None
|
||||
|
||||
def __enter__(self) -> "TracingContext":
|
||||
self._contexts[threading.get_ident()].append(self)
|
||||
return self
|
||||
|
||||
def __exit__(
|
||||
self,
|
||||
type: Optional[Type[BaseException]],
|
||||
exception: Optional[BaseException],
|
||||
traceback: Optional[TracebackType],
|
||||
) -> Literal[False]:
|
||||
assert self._contexts[threading.get_ident()][-1] == self
|
||||
self._contexts[threading.get_ident()].remove(self)
|
||||
|
||||
if exception is not None and type is not None:
|
||||
self.finalise(exception=exception)
|
||||
if get_context().should_log_exception_stack:
|
||||
get_context().logger.exception("Could not finish operation")
|
||||
else:
|
||||
get_context().logger.error(
|
||||
f"Could not finish operation because of {type.__name__}: {exception}"
|
||||
)
|
||||
|
||||
assert self._trace is not None
|
||||
get_context().tracing_database.save(self._trace)
|
||||
|
||||
return False
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
from .api_metadata import ApiMetadata
|
||||
from .cache_statistics import CacheStatistics
|
||||
from .evaluation_feedback_request import EvaluationFeedbackRequest
|
||||
from .filter import Filter
|
||||
from .function_metadata import FunctionMetadata
|
||||
from .health_check_response import HealthCheckResponse
|
||||
from .model import Model
|
||||
from .operators import operators
|
||||
from .query import Query
|
||||
from .sort_by import SortBy
|
||||
from .trace import Trace
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class ApiMetadata(BaseModel):
|
||||
name: str
|
||||
version: str
|
||||
documentation: str
|
||||
configuration: Any
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class CacheStatistics(BaseModel):
|
||||
hits: int
|
||||
misses: int
|
||||
size: int
|
||||
max_size: int
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class EvaluationFeedbackRequest(BaseModel):
|
||||
feedback: Any
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
from typing import Union
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from .operators import Operator
|
||||
|
||||
|
||||
class Filter(BaseModel):
|
||||
property: str
|
||||
operator: Operator
|
||||
value: Union[float, str]
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
from typing import List
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class FunctionMetadata(BaseModel):
|
||||
input_parameter_names: List[str] = []
|
||||
model_parameter_names: List[str] = []
|
||||
model_versions: str = ""
|
||||
is_finalised: bool = False
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
from pydantic import BaseModel
|
||||
|
||||
from .cache_statistics import CacheStatistics
|
||||
|
||||
|
||||
class HealthCheckResponse(BaseModel):
|
||||
is_healthy: bool
|
||||
cache_statistics: CacheStatistics
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class Model(BaseModel):
|
||||
key: str
|
||||
version: int
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
from typing import List, Literal
|
||||
|
||||
Operator = Literal[">=", "<=", "<", ">", "!=", "=", "contains"]
|
||||
|
||||
operators: List[Operator] = [">=", "<=", "<", ">", "!=", "=", "contains"]
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
from datetime import datetime
|
||||
from typing import List, Optional, Sequence
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from .filter import Filter
|
||||
from .sort_by import SortBy
|
||||
|
||||
|
||||
class Query(BaseModel):
|
||||
filter: List[Filter] = []
|
||||
sort: List[SortBy] = []
|
||||
conjunctive_tags: Sequence[str] = []
|
||||
since: Optional[datetime] = None
|
||||
until: Optional[datetime] = None
|
||||
has_feedback: Optional[bool] = None
|
||||
|
||||
class Config:
|
||||
schema_extra = {
|
||||
"example": {
|
||||
"filter": [
|
||||
{
|
||||
"property": "original_execution_time_ms",
|
||||
"operator": ">",
|
||||
"value": 100,
|
||||
}
|
||||
],
|
||||
"sort": [
|
||||
{"column_id": "original_execution_time_ms", "direction": "asc"},
|
||||
{"column_id": "id", "direction": "desc"},
|
||||
],
|
||||
"conjunctive_tags": ["online"],
|
||||
"has_feedback": False,
|
||||
}
|
||||
}
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class SortBy(BaseModel):
|
||||
column_id: str
|
||||
direction: Literal["asc", "desc"]
|
||||
|
|
@ -1,86 +0,0 @@
|
|||
from pprint import pformat
|
||||
from typing import Any, Dict, Generic, List, Optional, TypeVar
|
||||
from uuid import uuid4
|
||||
|
||||
from pydantic import Extra, validator
|
||||
|
||||
from ..helper import HashableBaseModel
|
||||
from .model import Model
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
class Trace(Generic[T], HashableBaseModel):
|
||||
trace_id: Optional[str]
|
||||
created: str
|
||||
original_execution_time_ms: float
|
||||
logged_values: Dict[str, Any]
|
||||
models: List[Model]
|
||||
exception: Optional[str]
|
||||
output: T
|
||||
feedback: Any = None
|
||||
tags: List[str]
|
||||
|
||||
class Config:
|
||||
extra = Extra.ignore
|
||||
|
||||
@validator("trace_id", always=True)
|
||||
def generate_id(cls, v: Optional[str], values: Dict[str, Any]) -> Optional[str]:
|
||||
if not v:
|
||||
return str(uuid4())
|
||||
return v
|
||||
|
||||
@property
|
||||
def input(self) -> Any:
|
||||
return (
|
||||
self.logged_values["input"]
|
||||
if list(self.logged_values.keys()) == ["input"]
|
||||
else self.logged_values
|
||||
)
|
||||
|
||||
@property
|
||||
def models_flat(self) -> str:
|
||||
return ", ".join(f"{m.key}:{m.version}" for m in self.models)
|
||||
|
||||
@property
|
||||
def output_flat(self) -> str:
|
||||
return pformat(self.output, indent=2, compact=True)
|
||||
|
||||
@property
|
||||
def exception_flat(self) -> str:
|
||||
return (
|
||||
"null"
|
||||
if self.exception is None
|
||||
else pformat(self.exception, indent=2, compact=True)
|
||||
)
|
||||
|
||||
@property
|
||||
def feedback_flat(self) -> str:
|
||||
return (
|
||||
"null"
|
||||
if self.feedback is None
|
||||
else pformat(self.feedback, indent=2, compact=True)
|
||||
)
|
||||
|
||||
@property
|
||||
def tags_flat(self) -> str:
|
||||
return ",\n".join(self.tags)
|
||||
|
||||
def to_flat_dict(self, include_original: bool = True) -> Dict[str, Any]:
|
||||
return {
|
||||
**(
|
||||
self.dict()
|
||||
if include_original
|
||||
else {
|
||||
"trace_id": self.trace_id,
|
||||
"created": self.created,
|
||||
"original_execution_time_ms": self.original_execution_time_ms,
|
||||
}
|
||||
),
|
||||
**self.logged_values,
|
||||
"models_flat": self.models_flat,
|
||||
"exception_flat": self.exception_flat,
|
||||
"output_flat": self.output_flat,
|
||||
"feedback_flat": self.feedback_flat,
|
||||
"tags_flat": self.tags_flat,
|
||||
}
|
||||
|
|
@ -1 +0,0 @@
|
|||
from .large_file import LargeFile, LargeFileLocal, LargeFileMongo, LargeFileS3
|
||||
|
|
@ -1,72 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
from argparse import Namespace
|
||||
from pathlib import Path
|
||||
from typing import Mapping, Type
|
||||
|
||||
from great_ai.utilities import get_logger
|
||||
|
||||
from .large_file import LargeFile, LargeFileLocal, LargeFileMongo, LargeFileS3
|
||||
from .parse_arguments import parse_arguments
|
||||
|
||||
logger = get_logger("large_file")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser, args = parse_arguments()
|
||||
|
||||
large_file = get_class(args)
|
||||
|
||||
if not args.cache and not args.push and not args.delete:
|
||||
logger.warning("No action required.")
|
||||
parser.print_help()
|
||||
|
||||
if args.cache:
|
||||
for c in args.cache:
|
||||
split = c.split(":")
|
||||
file_name = split[0]
|
||||
version = None if len(split) == 1 else int(split[1])
|
||||
large_file(file_name, "r", version=version).get()
|
||||
|
||||
if args.push:
|
||||
for p in args.push:
|
||||
path = Path(p)
|
||||
large_file(path.name, "w").push(path)
|
||||
|
||||
if args.delete:
|
||||
for f in args.delete:
|
||||
large_file(f).delete()
|
||||
|
||||
|
||||
def get_class(args: Namespace) -> Type[LargeFile]:
|
||||
factory: Mapping[str, Type[LargeFile]] = {
|
||||
"s3": LargeFileS3,
|
||||
"local": LargeFileLocal,
|
||||
"mongodb": LargeFileMongo,
|
||||
}
|
||||
|
||||
if args.backend not in factory:
|
||||
raise ValueError(
|
||||
f"Backend {args.backend} does not exits, available options: {' ,'.join(factory.keys())}"
|
||||
)
|
||||
|
||||
large_file = factory[args.backend]
|
||||
|
||||
if args.backend != "local":
|
||||
if args.secrets is None:
|
||||
raise ValueError(
|
||||
"Providing a credentials file is required when the backend mode is not `local`."
|
||||
)
|
||||
large_file.configure_credentials_from_file(args.secrets)
|
||||
|
||||
return large_file
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except KeyboardInterrupt:
|
||||
logger.warning("Exiting")
|
||||
exit()
|
||||
except Exception as e:
|
||||
logger.exception(e)
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
from .human_readable_to_byte import human_readable_to_byte
|
||||
from .progress_bar import DownloadProgressBar, UploadProgressBar
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
def bytes_to_megabytes(bytes: int) -> str:
|
||||
return f"{round(bytes / 1000 / 1000, 2):.2f}"
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
import re
|
||||
|
||||
|
||||
def human_readable_to_byte(size: str) -> int:
|
||||
"""Case is ignored, kb, kB, Kb, and KB are all treated as kilobyte."""
|
||||
|
||||
if size.strip() == "0":
|
||||
return 0
|
||||
|
||||
possible_units = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
|
||||
units_re = "|".join(possible_units)
|
||||
regex = re.compile(
|
||||
rf"""
|
||||
\s* # trim
|
||||
(?P<scalar>\d+(.\d+)?) # get scalar, it might be a float
|
||||
\s* # ignore optional whitespace
|
||||
(?P<unit>{units_re}) # capture the unit
|
||||
""",
|
||||
flags=re.VERBOSE | re.IGNORECASE,
|
||||
)
|
||||
|
||||
match = regex.match(size)
|
||||
if not match:
|
||||
raise ValueError(f'Could not find values in "{size}"')
|
||||
|
||||
results = match.groupdict()
|
||||
|
||||
scalar = float(results["scalar"])
|
||||
idx = possible_units.index(results["unit"].upper())
|
||||
factor = 1024**idx
|
||||
return round(scalar * factor)
|
||||
|
|
@ -1,51 +0,0 @@
|
|||
import os
|
||||
import threading
|
||||
from logging import Logger
|
||||
from pathlib import Path
|
||||
|
||||
from .bytes_to_megabytes import bytes_to_megabytes
|
||||
|
||||
|
||||
class ProgressBar:
|
||||
min_progress_percentage_change = 10
|
||||
|
||||
def __init__(self, file_size: int, logger: Logger, prefix: str):
|
||||
self._file_size = file_size
|
||||
self._logger = logger
|
||||
self._prefix = prefix
|
||||
|
||||
self._last_percentage: float = 0
|
||||
self._seen_so_far = 0
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def __call__(self, bytes_amount: int) -> None:
|
||||
with self._lock:
|
||||
self._seen_so_far += bytes_amount
|
||||
percentage = (self._seen_so_far / float(self._file_size)) * 100
|
||||
|
||||
if (
|
||||
percentage != 100
|
||||
and percentage - self._last_percentage
|
||||
< self.min_progress_percentage_change
|
||||
):
|
||||
return
|
||||
|
||||
self._last_percentage += self.min_progress_percentage_change
|
||||
|
||||
file_size_mb = bytes_to_megabytes(self._file_size)
|
||||
seen_so_far_mb = bytes_to_megabytes(self._seen_so_far)
|
||||
progress = seen_so_far_mb.rjust(len(file_size_mb))
|
||||
self._logger.info(
|
||||
f"{self._prefix} {progress}/{file_size_mb} MB ({percentage:.1f}%)"
|
||||
)
|
||||
|
||||
|
||||
class DownloadProgressBar(ProgressBar):
|
||||
def __init__(self, name: str, size: int, logger: Logger):
|
||||
super().__init__(file_size=size, logger=logger, prefix=f"Downloading {name}")
|
||||
|
||||
|
||||
class UploadProgressBar(ProgressBar):
|
||||
def __init__(self, path: Path, logger: Logger):
|
||||
size = os.path.getsize(path)
|
||||
super().__init__(file_size=size, logger=logger, prefix=f"Uploading {path.name}")
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
from .large_file import LargeFile
|
||||
from .large_file_local import LargeFileLocal
|
||||
from .large_file_mongo import LargeFileMongo
|
||||
from .large_file_s3 import LargeFileS3
|
||||
|
|
@ -1,314 +0,0 @@
|
|||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
from abc import ABC, abstractmethod
|
||||
from pathlib import Path
|
||||
from types import TracebackType
|
||||
from typing import IO, Any, List, Optional, Type, Union, cast
|
||||
|
||||
from great_ai.utilities import ConfigFile, get_logger
|
||||
|
||||
from ..helper import human_readable_to_byte
|
||||
from ..models import DataInstance
|
||||
|
||||
logger = get_logger("large_file")
|
||||
|
||||
|
||||
CACHE_NAME_VERSION_SEPARATOR = "-"
|
||||
COMPRESSION_ALGORITHM = "gztar"
|
||||
ARCHIVE_EXTENSION = ".tar.gz"
|
||||
|
||||
|
||||
class LargeFile(ABC):
|
||||
"""
|
||||
Store large files remotely. Use local cache for speed up.
|
||||
|
||||
Examples:
|
||||
|
||||
```
|
||||
with LargeFile("test.txt", "w", keep_last_n=3) as f:
|
||||
for i in range(1000000):
|
||||
f.write('test\n')
|
||||
|
||||
with LargeFile("test.txt", "r") as f:
|
||||
print(f.readlines()[0])
|
||||
|
||||
path_to_cached_text_file = LargeFile("test.txt", version=0).get()
|
||||
```
|
||||
|
||||
By default, files are stored in the ".cache" folder and the
|
||||
least recently use is deleted after the overall size reaches 30 GBs.
|
||||
|
||||
Change it with the following properties.
|
||||
|
||||
```
|
||||
LargeFile.cache_path = Path(".cache")
|
||||
LargeFile.max_cache_size = "30GB"
|
||||
```
|
||||
"""
|
||||
|
||||
initialized = False
|
||||
cache_path = Path(".cache")
|
||||
max_cache_size: Optional[str] = "30GB"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
mode: str = "r",
|
||||
*,
|
||||
buffering: int = -1,
|
||||
encoding: Optional[str] = None,
|
||||
errors: Optional[str] = None,
|
||||
newline: Optional[str] = None,
|
||||
version: Optional[int] = None,
|
||||
keep_last_n: Optional[int] = None,
|
||||
cache_only_mode: bool = False,
|
||||
):
|
||||
self._name = name
|
||||
self._version = version
|
||||
self._mode = mode
|
||||
self._keep_last_n = keep_last_n
|
||||
self._cache_only_mode = cache_only_mode
|
||||
|
||||
self._buffering = buffering
|
||||
self._encoding = encoding
|
||||
self._errors = errors
|
||||
self._newline = newline
|
||||
|
||||
LargeFile.cache_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
self._find_instances()
|
||||
self._check_mode_and_set_version()
|
||||
|
||||
@classmethod
|
||||
def configure_credentials_from_file(
|
||||
cls,
|
||||
secrets_path: Union[Path, str],
|
||||
) -> None:
|
||||
cls.configure_credentials(**ConfigFile(secrets_path))
|
||||
|
||||
@classmethod
|
||||
def configure_credentials(
|
||||
cls,
|
||||
) -> None:
|
||||
cls.initialized = True
|
||||
|
||||
def __enter__(self) -> IO:
|
||||
self._file: IO[Any] = (
|
||||
tempfile.NamedTemporaryFile(
|
||||
mode=self._mode,
|
||||
buffering=self._buffering,
|
||||
encoding=self._encoding,
|
||||
newline=self._newline,
|
||||
errors=self._errors,
|
||||
delete=False,
|
||||
prefix="large_file-",
|
||||
)
|
||||
if "w" in self._mode
|
||||
else open(
|
||||
self.get(),
|
||||
mode=self._mode,
|
||||
buffering=self._buffering,
|
||||
encoding=self._encoding,
|
||||
newline=self._newline,
|
||||
errors=self._errors,
|
||||
)
|
||||
)
|
||||
|
||||
return self._file
|
||||
|
||||
def __exit__(
|
||||
self,
|
||||
type: Optional[Type[BaseException]],
|
||||
exc: Optional[BaseException],
|
||||
traceback: Optional[TracebackType],
|
||||
) -> bool:
|
||||
self._file.close()
|
||||
|
||||
if type is None:
|
||||
if "w" in self._mode:
|
||||
self.push(Path(self._file.name))
|
||||
os.unlink(self._file.name)
|
||||
else:
|
||||
logger.exception("Could not finish operation.")
|
||||
|
||||
return True
|
||||
|
||||
@property
|
||||
def _local_name(self) -> str:
|
||||
return f"{self._name}{CACHE_NAME_VERSION_SEPARATOR}{self.version}"
|
||||
|
||||
@property
|
||||
def version(self) -> int:
|
||||
return cast(int, self._version)
|
||||
|
||||
def get(self, hide_progress: bool = False) -> Path:
|
||||
remote_path = next(
|
||||
i.remote_path for i in self._instances if i.version == self._version
|
||||
)
|
||||
|
||||
destination = self.cache_path / self._local_name
|
||||
if not destination.exists():
|
||||
logger.info(f"File {self._local_name} does not exist locally")
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
local_root_path = Path(tmp)
|
||||
tmp_file_archive = (
|
||||
local_root_path / f"{self._local_name}{ARCHIVE_EXTENSION}"
|
||||
)
|
||||
self._download(
|
||||
remote_path, tmp_file_archive, hide_progress=hide_progress
|
||||
)
|
||||
|
||||
logger.info(f"Decompressing {self._local_name}")
|
||||
shutil.unpack_archive(str(tmp_file_archive), tmp, COMPRESSION_ALGORITHM)
|
||||
shutil.move(str(local_root_path / self._local_name), str(destination))
|
||||
else:
|
||||
logger.info(f"File {self._local_name} found in cache")
|
||||
|
||||
return destination
|
||||
|
||||
def push(self, path: Union[Path, str], hide_progress: bool = False) -> None:
|
||||
if isinstance(path, str):
|
||||
path = Path(path)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
if path.is_file():
|
||||
logger.info(f"Copying file for {self._local_name}")
|
||||
copy: Any = shutil.copy
|
||||
else:
|
||||
logger.info(f"Copying directory for {self._local_name}")
|
||||
copy = shutil.copytree
|
||||
|
||||
try:
|
||||
# Make local copy in the cache
|
||||
shutil.rmtree(self.cache_path / self._local_name, ignore_errors=True)
|
||||
copy(str(path), str(self.cache_path / self._local_name))
|
||||
except shutil.SameFileError:
|
||||
pass # No worries
|
||||
|
||||
copy(str(path), str(Path(tmp) / self._local_name))
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp2:
|
||||
# A directory has to be zipped and it cannot contain the output of the zipping
|
||||
logger.info(f"Compressing {self._local_name}")
|
||||
shutil.make_archive(
|
||||
str(Path(tmp2) / self._local_name),
|
||||
COMPRESSION_ALGORITHM,
|
||||
tmp,
|
||||
)
|
||||
|
||||
file_to_be_uploaded = (
|
||||
Path(tmp2) / f"{self._local_name}{ARCHIVE_EXTENSION}"
|
||||
)
|
||||
self._upload(file_to_be_uploaded, hide_progress=hide_progress)
|
||||
|
||||
self.clean_up()
|
||||
|
||||
def delete(self) -> None:
|
||||
self._keep_last_n = 0
|
||||
self._delete_old_remote_versions()
|
||||
|
||||
def _find_instances(self) -> None:
|
||||
if self._cache_only_mode:
|
||||
self._instances = self._find_instances_from_cache()
|
||||
else:
|
||||
self._instances = self._find_remote_instances()
|
||||
|
||||
self._instances = sorted(self._instances, key=lambda i: i.version)
|
||||
|
||||
def _find_instances_from_cache(self) -> List[DataInstance]:
|
||||
logger.info(f"Fetching cached versions of {self._name}")
|
||||
|
||||
candidates = [
|
||||
DataInstance(
|
||||
name=CACHE_NAME_VERSION_SEPARATOR.join(
|
||||
f.name.split(CACHE_NAME_VERSION_SEPARATOR)[:-1]
|
||||
),
|
||||
version=int(f.name.split(CACHE_NAME_VERSION_SEPARATOR)[-1]),
|
||||
remote_path=f,
|
||||
)
|
||||
for f in self.cache_path.glob(
|
||||
f"{self._name}{CACHE_NAME_VERSION_SEPARATOR}*"
|
||||
)
|
||||
]
|
||||
|
||||
return [c for c in candidates if c.name == self._name]
|
||||
|
||||
def _check_mode_and_set_version(self) -> None:
|
||||
if "+" in self._mode:
|
||||
raise ValueError(
|
||||
f"File mode `{self._mode}` is not allowed3, remove the `+`."
|
||||
)
|
||||
|
||||
if "w" in self._mode:
|
||||
if self._version is not None:
|
||||
raise ValueError("Providing a version is not allowed in write mode.")
|
||||
|
||||
self._version = self._instances[-1].version + 1 if self._instances else 0
|
||||
|
||||
elif "r" in self._mode:
|
||||
if not self._instances:
|
||||
raise FileNotFoundError(
|
||||
f"File {self._name} not found. No versions are available."
|
||||
)
|
||||
|
||||
if self._version is None:
|
||||
self._version = self._instances[-1].version
|
||||
logger.info(
|
||||
f"Latest version of {self._name} is {self._version} "
|
||||
+ f"(from versions: {self.versions_pretty})"
|
||||
)
|
||||
elif self._version not in [i.version for i in self._instances]:
|
||||
raise FileNotFoundError(
|
||||
f"File {self._name} not found with version {self._version}. "
|
||||
+ f"(from versions: {self.versions_pretty})"
|
||||
)
|
||||
else:
|
||||
raise ValueError("Unsupported file mode.")
|
||||
|
||||
@property
|
||||
def versions_pretty(self) -> str:
|
||||
return ", ".join((str(i.version) for i in self._instances))
|
||||
|
||||
def clean_up(self) -> None:
|
||||
self._delete_old_remote_versions()
|
||||
self._prune_cache()
|
||||
|
||||
def _prune_cache(self) -> None:
|
||||
self.cache_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if self.max_cache_size is None:
|
||||
return
|
||||
|
||||
allowed_size = human_readable_to_byte(self.max_cache_size)
|
||||
assert allowed_size >= 0
|
||||
|
||||
least_recently_read = sorted(
|
||||
[f for f in self.cache_path.glob("*")], key=lambda f: f.stat().st_atime
|
||||
)
|
||||
|
||||
while sum(os.path.getsize(f) for f in least_recently_read) > allowed_size:
|
||||
file = least_recently_read.pop(0)
|
||||
logger.info(
|
||||
f"Deleting file from cache to meet quota (max_cache_size={self.max_cache_size}): {file}"
|
||||
)
|
||||
os.unlink(file)
|
||||
|
||||
@abstractmethod
|
||||
def _find_remote_instances(self) -> List[DataInstance]:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def _download(
|
||||
self, remote_path: Any, local_path: Path, hide_progress: bool
|
||||
) -> None:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def _upload(self, local_path: Path, hide_progress: bool) -> None:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def _delete_old_remote_versions(self) -> None:
|
||||
pass
|
||||
|
|
@ -1,59 +0,0 @@
|
|||
from pathlib import Path
|
||||
from typing import Any, List, Optional
|
||||
|
||||
from great_ai.utilities import get_logger
|
||||
|
||||
from ..models import DataInstance
|
||||
from .large_file import LargeFile
|
||||
|
||||
logger = get_logger("large_file")
|
||||
|
||||
|
||||
class LargeFileLocal(LargeFile):
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
mode: str = "r",
|
||||
*,
|
||||
buffering: int = -1,
|
||||
encoding: Optional[str] = None,
|
||||
errors: Optional[str] = None,
|
||||
newline: Optional[str] = None,
|
||||
version: Optional[int] = None,
|
||||
keep_last_n: Optional[int] = None,
|
||||
):
|
||||
super().__init__(
|
||||
name,
|
||||
mode,
|
||||
buffering=buffering,
|
||||
encoding=encoding,
|
||||
errors=errors,
|
||||
newline=newline,
|
||||
version=version,
|
||||
keep_last_n=keep_last_n,
|
||||
cache_only_mode=True,
|
||||
)
|
||||
super().configure_credentials()
|
||||
|
||||
def _find_remote_instances(self) -> List[DataInstance]:
|
||||
return []
|
||||
|
||||
def _download(
|
||||
self, remote_path: Any, local_path: Path, hide_progress: bool
|
||||
) -> None:
|
||||
raise NotImplementedError()
|
||||
|
||||
def _upload(self, local_path: Path, hide_progress: bool) -> None:
|
||||
pass # the "upload" is already done py the parent's caching mechanism
|
||||
|
||||
def _delete_old_remote_versions(self) -> None:
|
||||
if self._keep_last_n is not None:
|
||||
for i in (
|
||||
self._instances[: -self._keep_last_n]
|
||||
if self._keep_last_n > 0
|
||||
else self._instances
|
||||
):
|
||||
logger.info(
|
||||
f"Removing old version (keep_last_n={self._keep_last_n}): {i.remote_path}"
|
||||
)
|
||||
i.remote_path.unlink()
|
||||
|
|
@ -1,122 +0,0 @@
|
|||
import re
|
||||
from functools import cached_property
|
||||
from pathlib import Path
|
||||
from typing import Any, List, Mapping
|
||||
|
||||
from gridfs import DEFAULT_CHUNK_SIZE, Database, GridFSBucket
|
||||
from pymongo import MongoClient
|
||||
|
||||
from great_ai.utilities import get_logger
|
||||
|
||||
from ..helper import DownloadProgressBar, UploadProgressBar
|
||||
from ..models import DataInstance
|
||||
from .large_file import LargeFile
|
||||
|
||||
logger = get_logger("large_file")
|
||||
|
||||
|
||||
MONGO_NAME_VERSION_SEPARATOR = "_"
|
||||
|
||||
|
||||
class LargeFileMongo(LargeFile):
|
||||
mongo_connection_string = None
|
||||
mongo_database = None
|
||||
|
||||
@classmethod
|
||||
def configure_credentials( # type: ignore
|
||||
cls,
|
||||
*,
|
||||
mongo_connection_string: str,
|
||||
mongo_database: str,
|
||||
**_: Mapping[str, Any],
|
||||
) -> None:
|
||||
cls.mongo_connection_string = mongo_connection_string
|
||||
cls.mongo_database = mongo_database
|
||||
super().configure_credentials()
|
||||
|
||||
@cached_property
|
||||
def _client(self) -> GridFSBucket:
|
||||
if self.mongo_connection_string is None or self.mongo_database is None:
|
||||
raise ValueError(
|
||||
"Please configure the MongoDB access options by calling LargeFileMongo.configure_credentials or set offline_mode=True in the constructor."
|
||||
)
|
||||
|
||||
db: Database = MongoClient(self.mongo_connection_string)[self.mongo_database]
|
||||
return GridFSBucket(db)
|
||||
|
||||
def _find_remote_instances(self) -> List[DataInstance]:
|
||||
logger.debug(f"Fetching Mongo (GridFS) versions of {self._name}")
|
||||
|
||||
return [
|
||||
DataInstance(
|
||||
name=MONGO_NAME_VERSION_SEPARATOR.join(
|
||||
f.name.split(MONGO_NAME_VERSION_SEPARATOR)[:-1]
|
||||
),
|
||||
version=int(f.name.split(MONGO_NAME_VERSION_SEPARATOR)[-1]),
|
||||
remote_path=(f._id, f.length),
|
||||
origin="mongodb",
|
||||
)
|
||||
for f in self._client.find(
|
||||
{
|
||||
"filename": re.compile(
|
||||
re.escape(self._name + MONGO_NAME_VERSION_SEPARATOR) + ".*"
|
||||
)
|
||||
}
|
||||
)
|
||||
]
|
||||
|
||||
def _download(
|
||||
self, remote_path: Any, local_path: Path, hide_progress: bool
|
||||
) -> None:
|
||||
logger.info(f"Downloading {remote_path[0]} from Mongo (GridFS)")
|
||||
|
||||
progress = (
|
||||
DownloadProgressBar(
|
||||
name=str(remote_path[0]), size=remote_path[1], logger=logger
|
||||
)
|
||||
if not hide_progress
|
||||
else None
|
||||
)
|
||||
with self._client.open_download_stream(remote_path[0]) as stream:
|
||||
with open(local_path, "wb") as f:
|
||||
while True:
|
||||
content = stream.read(DEFAULT_CHUNK_SIZE)
|
||||
f.write(content)
|
||||
|
||||
if progress:
|
||||
progress(len(content))
|
||||
if len(content) < DEFAULT_CHUNK_SIZE:
|
||||
break
|
||||
|
||||
def _upload(self, local_path: Path, hide_progress: bool) -> None:
|
||||
logger.info(f"Uploading {local_path} to Mongo (GridFS)")
|
||||
|
||||
progress = (
|
||||
UploadProgressBar(path=local_path, logger=logger)
|
||||
if not hide_progress
|
||||
else None
|
||||
)
|
||||
with self._client.open_upload_stream(
|
||||
f"{self._name}{MONGO_NAME_VERSION_SEPARATOR}{self.version}"
|
||||
) as stream:
|
||||
with open(local_path, "rb") as f:
|
||||
while True:
|
||||
content = f.read(DEFAULT_CHUNK_SIZE)
|
||||
stream.write(content)
|
||||
|
||||
if progress:
|
||||
progress(len(content))
|
||||
if len(content) < DEFAULT_CHUNK_SIZE:
|
||||
break
|
||||
|
||||
def _delete_old_remote_versions(self) -> None:
|
||||
if self._keep_last_n is not None:
|
||||
for i in (
|
||||
self._instances[: -self._keep_last_n]
|
||||
if self._keep_last_n > 0
|
||||
else self._instances
|
||||
):
|
||||
logger.info(
|
||||
f"Removing old version from MongoDB (GridFS) (keep_last_n={self._keep_last_n}): {i.name}{MONGO_NAME_VERSION_SEPARATOR}{i.version}"
|
||||
)
|
||||
self._client.delete(i.remote_path[0])
|
||||
|
|
@ -1,152 +0,0 @@
|
|||
from functools import cached_property
|
||||
from pathlib import Path
|
||||
from typing import Any, List, Mapping, Optional
|
||||
|
||||
import boto3
|
||||
|
||||
from great_ai.utilities import get_logger
|
||||
|
||||
from ..helper import DownloadProgressBar, UploadProgressBar
|
||||
from ..models import DataInstance
|
||||
from .large_file import LargeFile
|
||||
|
||||
logger = get_logger("large_file")
|
||||
|
||||
|
||||
S3_NAME_VERSION_SEPARATOR = "/"
|
||||
|
||||
|
||||
class LargeFileS3(LargeFile):
|
||||
"""
|
||||
Store large files in S3. Use local cache for speed up.
|
||||
|
||||
Examples:
|
||||
|
||||
```
|
||||
with LargeFile("test.txt", "w", keep_last_n=3) as f:
|
||||
for i in range(1000000):
|
||||
f.write('test\n')
|
||||
|
||||
with LargeFile("test.txt", "r") as f:
|
||||
print(f.readlines()[0])
|
||||
|
||||
path_to_cached_text_file = LargeFile("test.txt", version=0).get()
|
||||
```
|
||||
|
||||
By default, files are stored in the ".cache" folder and the
|
||||
least recently use is deleted after the overall size reaches 30 GBs.
|
||||
|
||||
Change it with the following properties.
|
||||
|
||||
```
|
||||
LargeFile.cache_path = Path(".cache")
|
||||
LargeFile.max_cache_size = "30GB"
|
||||
```
|
||||
"""
|
||||
|
||||
region_name = None
|
||||
access_key_id = None
|
||||
secret_access_key = None
|
||||
bucket_name = None
|
||||
endpoint_url = None
|
||||
|
||||
@classmethod
|
||||
def configure_credentials( # type: ignore
|
||||
cls,
|
||||
*,
|
||||
aws_region_name: str,
|
||||
aws_access_key_id: str,
|
||||
aws_secret_access_key: str,
|
||||
large_files_bucket_name: str,
|
||||
aws_endpoint_url: Optional[str] = None,
|
||||
**_: Mapping[str, Any],
|
||||
) -> None:
|
||||
cls.region_name = aws_region_name
|
||||
cls.access_key_id = aws_access_key_id
|
||||
cls.secret_access_key = aws_secret_access_key
|
||||
cls.bucket_name = large_files_bucket_name
|
||||
cls.endpoint_url = aws_endpoint_url
|
||||
super().configure_credentials()
|
||||
|
||||
@cached_property
|
||||
def _client(self) -> boto3.client:
|
||||
if (
|
||||
self.region_name is None
|
||||
or self.access_key_id is None
|
||||
or self.secret_access_key is None
|
||||
or self.bucket_name is None
|
||||
):
|
||||
raise ValueError(
|
||||
"Please configure the S3 access options by calling LargeFileS3.configure_credentials or set offline_mode=True in the constructor."
|
||||
)
|
||||
|
||||
return boto3.client(
|
||||
"s3",
|
||||
aws_access_key_id=self.access_key_id,
|
||||
aws_secret_access_key=self.secret_access_key,
|
||||
region_name=self.region_name,
|
||||
endpoint_url=self.endpoint_url,
|
||||
)
|
||||
|
||||
def _find_remote_instances(self) -> List[DataInstance]:
|
||||
logger.debug(f"Fetching S3 versions of {self._name}")
|
||||
|
||||
found_objects = self._client.list_objects_v2(
|
||||
Bucket=self.bucket_name, Prefix=self._name
|
||||
)
|
||||
return (
|
||||
[
|
||||
DataInstance(
|
||||
name=o["Key"].split(S3_NAME_VERSION_SEPARATOR)[0],
|
||||
version=int(o["Key"].split(S3_NAME_VERSION_SEPARATOR)[-1]),
|
||||
remote_path=o["Key"],
|
||||
)
|
||||
for o in found_objects["Contents"]
|
||||
if o["Key"].split(S3_NAME_VERSION_SEPARATOR)[0] == self._name
|
||||
]
|
||||
if "Contents" in found_objects
|
||||
else []
|
||||
)
|
||||
|
||||
def _download(
|
||||
self, remote_path: Any, local_path: Path, hide_progress: bool
|
||||
) -> None:
|
||||
logger.info(f"Downloading {remote_path} from S3")
|
||||
|
||||
size = self._client.head_object(Bucket=self.bucket_name, Key=remote_path)[
|
||||
"ContentLength"
|
||||
]
|
||||
|
||||
self._client.download_file(
|
||||
Bucket=self.bucket_name,
|
||||
Key=remote_path,
|
||||
Filename=str(local_path),
|
||||
Callback=None
|
||||
if hide_progress
|
||||
else DownloadProgressBar(name=str(remote_path), size=size, logger=logger),
|
||||
)
|
||||
|
||||
def _upload(self, local_path: Path, hide_progress: bool) -> None:
|
||||
key = f"{self._name}/{self.version}"
|
||||
logger.info(f"Uploading {self._local_name} to S3 as {key}")
|
||||
|
||||
self._client.upload_file(
|
||||
Filename=str(local_path),
|
||||
Bucket=self.bucket_name,
|
||||
Key=key,
|
||||
Callback=None
|
||||
if hide_progress
|
||||
else UploadProgressBar(path=local_path, logger=logger),
|
||||
)
|
||||
|
||||
def _delete_old_remote_versions(self) -> None:
|
||||
if self._keep_last_n is not None:
|
||||
for i in (
|
||||
self._instances[: -self._keep_last_n]
|
||||
if self._keep_last_n > 0
|
||||
else self._instances
|
||||
):
|
||||
logger.info(
|
||||
f"Removing old version from S3 (keep_last_n={self._keep_last_n}): {i.remote_path}"
|
||||
)
|
||||
self._client.delete_object(Bucket=self.bucket_name, Key=i.remote_path)
|
||||
|
|
@ -1 +0,0 @@
|
|||
from .data_instance import DataInstance
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class DataInstance(BaseModel):
|
||||
name: str
|
||||
version: int
|
||||
remote_path: Any
|
||||
|
|
@ -1,56 +0,0 @@
|
|||
from argparse import ArgumentParser, Namespace
|
||||
from typing import Tuple
|
||||
|
||||
|
||||
def parse_arguments() -> Tuple[ArgumentParser, Namespace]:
|
||||
parser = ArgumentParser(
|
||||
description="Store and version large files in S3; open them like regular files. Caching included.",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"-b",
|
||||
"--backend",
|
||||
type=str,
|
||||
help="choose which backend to use, available options: `local`, `s3`, `mongodb`",
|
||||
required=True,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"-s",
|
||||
"--secrets",
|
||||
type=str,
|
||||
help="path to an .ini configuration file with your S3 credentials",
|
||||
required=False,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"-c",
|
||||
"--cache",
|
||||
nargs="+",
|
||||
type=str,
|
||||
help="download file into local cache, example: file_name.txt:version",
|
||||
required=False,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"-p",
|
||||
"--push",
|
||||
nargs="+",
|
||||
type=str,
|
||||
help="push a local file into S3 and set it as the most recent version",
|
||||
required=False,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"-d",
|
||||
"--delete",
|
||||
nargs="+",
|
||||
type=str,
|
||||
help="delete every version of file from S3",
|
||||
required=False,
|
||||
)
|
||||
|
||||
parser.print_usage = parser.print_help # type: ignore
|
||||
args = parser.parse_args()
|
||||
|
||||
return parser, args
|
||||
|
|
@ -1,47 +0,0 @@
|
|||
from argparse import ArgumentParser, Namespace
|
||||
|
||||
|
||||
def parse_arguments() -> Namespace:
|
||||
parser = ArgumentParser(
|
||||
description="GreatAI-Server for deploying you AI applications with ease.",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"file_name",
|
||||
type=str,
|
||||
help="the name of the file containing your to-be-served function such as `main.py`\n",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--host",
|
||||
type=str,
|
||||
help="it is passed to uvicorn which starts a server listening on this address",
|
||||
default="0.0.0.0",
|
||||
required=False,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--port",
|
||||
type=int,
|
||||
help="it is passed to uvicorn which starts a server listening on this port",
|
||||
default=6060,
|
||||
required=False,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--timeout_keep_alive",
|
||||
type=int,
|
||||
help="it is passed to uvicorn which uses it for timing out requests taking longer than this many seconds",
|
||||
default=600,
|
||||
required=False,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--workers",
|
||||
type=int,
|
||||
help="it is passed to uvicorn which starts this many server processes",
|
||||
default=1,
|
||||
required=False,
|
||||
)
|
||||
|
||||
return parser.parse_args()
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
from .clean import clean
|
||||
from .config_file import ConfigFile
|
||||
from .evaluate_ranking import evaluate_ranking
|
||||
from .get_sentences import get_sentences
|
||||
from .language import english_name_of_language, is_english, predict_language
|
||||
from .lemmatize_text import lemmatize_text
|
||||
from .lemmatize_token import lemmatize_token
|
||||
from .logger import get_logger
|
||||
from .match_names import match_names
|
||||
from .matcher import fast_tokenize, filter_sentences, normalize
|
||||
from .nlp import nlp
|
||||
from .parallel_map import parallel_map
|
||||
from .publication_tei import PublicationTEI
|
||||
from .unique import unique
|
||||
|
|
@ -1,68 +0,0 @@
|
|||
import html
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
import unidecode
|
||||
|
||||
from .data import left_regular_punctuations, right_regular_punctuations
|
||||
from .external.pylatexenc.latex2text import LatexNodes2Text
|
||||
from .logger import get_logger
|
||||
|
||||
logger = get_logger("clean")
|
||||
latex = LatexNodes2Text()
|
||||
|
||||
|
||||
joined_left_punctuations = "".join(left_regular_punctuations).replace("]", r"\]")
|
||||
joined_right_punctuations = "".join(right_regular_punctuations).replace("[", r"\[")
|
||||
|
||||
|
||||
def clean(
|
||||
text: str,
|
||||
ignore_xml: bool = False,
|
||||
ignore_latex: bool = False,
|
||||
remove_brackets: bool = False,
|
||||
convert_to_ascii: bool = False,
|
||||
) -> str:
|
||||
if not ignore_xml:
|
||||
text = re.sub(r"<[^>]*>", " ", text)
|
||||
text = html.unescape(text)
|
||||
|
||||
if not ignore_latex:
|
||||
text = text.replace("%", "\\%") # escape LaTeX comments before parsing as LaTeX
|
||||
|
||||
try:
|
||||
text = latex.latex_to_text(text, tolerant_parsing=True, strict_braces=False)
|
||||
text = text.replace("%s", " ")
|
||||
except:
|
||||
logger.exception("Latex parsing error")
|
||||
|
||||
if convert_to_ascii:
|
||||
text = unicodedata.normalize("NFKD", text)
|
||||
|
||||
try:
|
||||
text.encode("ASCII", errors="strict")
|
||||
except UnicodeEncodeError:
|
||||
text = "".join([c for c in text if not unicodedata.combining(c)])
|
||||
text = unidecode.unidecode(text)
|
||||
|
||||
text = re.sub(
|
||||
r"\b[a-zA-Z](?:[\t ]+[a-zA-Z]\b)+", lambda m: re.sub(r"[\t ]", "", m[0]), text
|
||||
) # A R T I C L E => ARTICLE
|
||||
|
||||
if remove_brackets:
|
||||
text = re.sub(r"\[[^\]]*\]", " ", text)
|
||||
|
||||
# fix hypens: break- word => break-word
|
||||
text = re.sub(r"(\S)-\s+", r"\1-", text)
|
||||
text = re.sub(r"\s+-(\S)", r"-\1", text)
|
||||
|
||||
# collapse whitespace
|
||||
text = re.sub(r"\s+", " ", text)
|
||||
|
||||
# fix punctuation
|
||||
text = re.sub(rf" ([{joined_left_punctuations}])", r"\1", text)
|
||||
text = re.sub(rf"([{joined_right_punctuations}]) ", r"\1", text)
|
||||
|
||||
text = text.strip()
|
||||
|
||||
return text
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
from .config_file import ConfigFile
|
||||
from .parse_error import ParseError
|
||||
|
|
@ -1,87 +0,0 @@
|
|||
import os
|
||||
from pathlib import Path
|
||||
from typing import Dict, Iterable, Tuple, Union
|
||||
|
||||
from ..logger import get_logger
|
||||
from .parse_error import ParseError
|
||||
from .pattern import pattern
|
||||
|
||||
ENVIRONMENT_VARIABLE_KEY_PREFIX = "ENV"
|
||||
|
||||
logger = get_logger("ConfigFile")
|
||||
|
||||
|
||||
class ConfigFile:
|
||||
def __init__(
|
||||
self, path: Union[Path, str], ignore_missing_environment_variables: bool = False
|
||||
) -> None:
|
||||
if not isinstance(path, Path):
|
||||
path = Path(path)
|
||||
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(path.absolute())
|
||||
|
||||
self._path = path
|
||||
self._ignore_missing_environment_variables = (
|
||||
ignore_missing_environment_variables
|
||||
)
|
||||
self._key_values: Dict[str, str] = {}
|
||||
|
||||
self._parse()
|
||||
|
||||
def _parse(self):
|
||||
with open(self._path, encoding="utf-8") as f:
|
||||
lines: str = f.read()
|
||||
|
||||
matches = pattern.findall(lines)
|
||||
for key, *values in matches:
|
||||
if key in self._key_values:
|
||||
raise KeyError(
|
||||
f"Key `{key}` has been already defined and its value is `{self._key_values[key]}`"
|
||||
)
|
||||
|
||||
try:
|
||||
value = next(v for v in values if v)
|
||||
except StopIteration:
|
||||
raise ParseError(
|
||||
f"Cannot parse config file ({self._path.absolute()}), error at key `{key}`"
|
||||
)
|
||||
|
||||
if value.startswith(f"{ENVIRONMENT_VARIABLE_KEY_PREFIX}:"):
|
||||
_, value = value.split(":")
|
||||
if value not in os.environ:
|
||||
issue = f'The value of `{key}` contains the "{ENVIRONMENT_VARIABLE_KEY_PREFIX}` prefix but `{value}` is not defined as an environment variable'
|
||||
if self._ignore_missing_environment_variables:
|
||||
logger.warning(f"{issue}, defaulting to `None`")
|
||||
else:
|
||||
raise KeyError(issue)
|
||||
value = os.environ[value]
|
||||
|
||||
self._key_values[key] = value
|
||||
|
||||
def __getattr__(self, key: str) -> str:
|
||||
if key in self._key_values:
|
||||
return self._key_values[key]
|
||||
raise KeyError(
|
||||
f"Key `{key}` is not found in configuration file ({self._path.absolute()})"
|
||||
)
|
||||
|
||||
__getitem__ = __getattr__
|
||||
|
||||
def __iter__(self) -> Iterable[Tuple[str, str]]:
|
||||
return iter(self._key_values)
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self._key_values)
|
||||
|
||||
def keys(self):
|
||||
return self._key_values.keys()
|
||||
|
||||
def values(self):
|
||||
return self._key_values.values()
|
||||
|
||||
def items(self):
|
||||
return self._key_values.items()
|
||||
|
||||
def __repr__(self):
|
||||
return f"{type(self).__name__}(\n path={self._path},\n ignore_missing_environment_variables={self._ignore_missing_environment_variables}\n) {{{self._key_values}}}"
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
class ParseError(Exception):
|
||||
pass
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
import re
|
||||
|
||||
pattern = re.compile(
|
||||
r"""
|
||||
\s* # leading whitespace is allowed
|
||||
(\w+?) # then comes the key
|
||||
\s*=\s* # the key and value are separated by an equal sign
|
||||
(?: # then comes the value
|
||||
"([^"]*)" # the value can be surrounded by quotes: "value"
|
||||
| '([^']*)' # the value can be surrounded by quotes: 'value'
|
||||
| `([^`]*)` # the value can be surrounded by quotes: `value`
|
||||
| ([^#\n]*?) # or it is bare, in that case, the trailing whitespace is ignored
|
||||
)
|
||||
\s*(?:\#.*)? # comments can be added with the `#` symbol
|
||||
(?:\n|$) # a key-value pairs are separated by new lines
|
||||
""",
|
||||
flags=re.UNICODE | re.VERBOSE,
|
||||
)
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
from .american_spellings import american_spellings
|
||||
from .punctuations import (
|
||||
left_regular_punctuations,
|
||||
right_regular_punctuations,
|
||||
sentence_ending_punctuations,
|
||||
)
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,22 +0,0 @@
|
|||
sentence_ending_punctuations = [".", "?", "!", ":", ";"]
|
||||
|
||||
# punctuations that usually have no space between them and the character to their left
|
||||
left_regular_punctuations = [
|
||||
*sentence_ending_punctuations,
|
||||
",",
|
||||
"'",
|
||||
"%",
|
||||
")",
|
||||
"}",
|
||||
"]",
|
||||
]
|
||||
|
||||
# punctuations that usually have no space between them and the character to their right
|
||||
right_regular_punctuations = [
|
||||
"(",
|
||||
"{",
|
||||
"[",
|
||||
"$",
|
||||
"€",
|
||||
"#",
|
||||
]
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
from .draw_f1_iso_lines import draw_f1_iso_lines
|
||||
from .evaluate_ranking import evaluate_ranking
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue