From 22696c7f6fb24eda83bae1993c52d262f15ec712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A1s=20Schmelczer?= Date: Wed, 25 May 2022 10:02:16 +0200 Subject: [PATCH] Add auto-reload, evaluation endpoint, show docs --- .vscode/settings.json | 1 + .vscode/tasks.json | 4 +- example/main_service.py | 9 -- {example => examples}/README.md | 2 +- {example => examples}/main_batch.py | 0 examples/main_service.py | 9 ++ {example => examples}/predict_domain.py | 5 +- {example => examples}/preprocess.py | 0 {example => examples}/train.ipynb | 2 +- great_ai/src/great_ai/__main__.py | 42 ++++++ great_ai/src/great_ai/great_ai/__init__.py | 4 +- .../src/great_ai/great_ai/context/__init__.py | 2 +- .../great_ai/great_ai/context/get_context.py | 19 ++- .../src/great_ai/great_ai/deploy/__init__.py | 2 +- ...reate_fastapi_app.py => create_service.py} | 44 +++++- .../src/great_ai/great_ai/deploy/serve.py | 48 ------ .../great_ai/great_ai/exceptions/__init__.py | 1 + .../exceptions/missing_argument_error.py | 2 + .../src/great_ai/great_ai/helper/__init__.py | 1 + .../great_ai/great_ai/helper/strip_lines.py | 2 + .../great_ai/metrics/create_dash_app.py | 6 +- .../great_ai/metrics/get_description.py | 39 +++-- .../great_ai/persistence/mongodb_driver.py | 2 - .../persistence/parallel_tinydb_driver.py | 18 ++- .../persistence/persistence_driver.py | 10 +- .../great_ai/tracing/tracing_context.py | 2 +- .../src/great_ai/great_ai/views/__init__.py | 1 + .../views/evaluation_feedback_request.py | 7 + great_ai/src/great_ai/great_ai/views/trace.py | 1 + great_ai/src/sus.egg-info/PKG-INFO | 50 +++++++ great_ai/src/sus.egg-info/SOURCES.txt | 138 ++++++++++++++++++ .../src/sus.egg-info/dependency_links.txt | 1 + great_ai/src/sus.egg-info/requires.txt | 14 ++ great_ai/src/sus.egg-info/top_level.txt | 1 + 34 files changed, 389 insertions(+), 100 deletions(-) delete mode 100755 example/main_service.py rename {example => examples}/README.md (79%) rename {example => examples}/main_batch.py (100%) create mode 100755 examples/main_service.py rename {example => examples}/predict_domain.py (93%) rename {example => examples}/preprocess.py (100%) rename {example => examples}/train.ipynb (99%) create mode 100644 great_ai/src/great_ai/__main__.py rename great_ai/src/great_ai/great_ai/deploy/{create_fastapi_app.py => create_service.py} (54%) delete mode 100644 great_ai/src/great_ai/great_ai/deploy/serve.py create mode 100644 great_ai/src/great_ai/great_ai/exceptions/missing_argument_error.py create mode 100644 great_ai/src/great_ai/great_ai/helper/strip_lines.py create mode 100644 great_ai/src/great_ai/great_ai/views/evaluation_feedback_request.py create mode 100644 great_ai/src/sus.egg-info/PKG-INFO create mode 100644 great_ai/src/sus.egg-info/SOURCES.txt create mode 100644 great_ai/src/sus.egg-info/dependency_links.txt create mode 100644 great_ai/src/sus.egg-info/requires.txt create mode 100644 great_ai/src/sus.egg-info/top_level.txt diff --git a/.vscode/settings.json b/.vscode/settings.json index d9a251a..fbf0fd2 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,6 +2,7 @@ "cSpell.words": [ "boto", "botocore", + "fastapi", "iloc", "inplace", "levelno", diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 9fd7faf..6e1f719 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -4,9 +4,9 @@ { "label": "Format and lint Python", "type": "shell", - "command": "source .env/bin/activate && scripts/format-python.sh great_ai && scripts/format-python.sh example", + "command": "source .env/bin/activate && scripts/format-python.sh great_ai && scripts/format-python.sh examples", "windows": { - "command": ".env/bin/activate.bat; scripts/format-python.sh great_ai; scripts\\format-python.sh example" + "command": ".env/bin/activate.bat; scripts/format-python.sh great_ai; scripts\\format-python.sh examples" }, "group": "test", "presentation": { diff --git a/example/main_service.py b/example/main_service.py deleted file mode 100755 index f033e74..0000000 --- a/example/main_service.py +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env python3 - - -from predict_domain import predict_domain - -from great_ai import serve - -if __name__ == "__main__": - serve(predict_domain) diff --git a/example/README.md b/examples/README.md similarity index 79% rename from example/README.md rename to examples/README.md index 40a4926..627f957 100644 --- a/example/README.md +++ b/examples/README.md @@ -1,4 +1,4 @@ -# Train Domain classifier from the [semantic scholar dataset](https://api.semanticscholar.org/corpus) +# Train Domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus) ## Upload the dataset (or a part of it) to shared infrastructure diff --git a/example/main_batch.py b/examples/main_batch.py similarity index 100% rename from example/main_batch.py rename to examples/main_batch.py diff --git a/examples/main_service.py b/examples/main_service.py new file mode 100755 index 0000000..29049d5 --- /dev/null +++ b/examples/main_service.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 + +from great_ai import configure, create_service + +configure(development_mode_override=True) + +from predict_domain import predict_domain + +app = create_service(predict_domain) diff --git a/example/predict_domain.py b/examples/predict_domain.py similarity index 93% rename from example/predict_domain.py rename to examples/predict_domain.py index 353a8f1..cf1550b 100644 --- a/example/predict_domain.py +++ b/examples/predict_domain.py @@ -20,7 +20,10 @@ class DomainPrediction(BaseModel): def predict_domain( text: str, model: Pipeline, cut_off_probability: float = 0.2 ) -> List[DomainPrediction]: - assert 0 <= cut_off_probability <= 1 + """ + Predict the scientific domain of the input text. + Return labels until their sum likelihood is larger than cut_off_probability. + """ log_metric("text_length", len(text)) cleaned = clean(text, convert_to_ascii=True) diff --git a/example/preprocess.py b/examples/preprocess.py similarity index 100% rename from example/preprocess.py rename to examples/preprocess.py diff --git a/example/train.ipynb b/examples/train.ipynb similarity index 99% rename from example/train.ipynb rename to examples/train.ipynb index a9bd6d0..5ad5315 100644 --- a/example/train.ipynb +++ b/examples/train.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Train Domain classifier from the [semantic scholar dataset](https://api.semanticscholar.org/corpus)" + "# Train Domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)" ] }, { diff --git a/great_ai/src/great_ai/__main__.py b/great_ai/src/great_ai/__main__.py new file mode 100644 index 0000000..fd9eb0e --- /dev/null +++ b/great_ai/src/great_ai/__main__.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +import re +from importlib import import_module +from sys import argv + +import uvicorn +from uvicorn.config import LOGGING_CONFIG + +from great_ai.great_ai.context import get_context + +if __name__ == "__main__": + if len(argv) < 2: + raise ValueError(f"Provide a filename such as: {argv[0]} my_app.py") + + module_name = re.sub(r"\.py\b", "", argv[1]) + if ":" not in module_name: + module_name += ":app" + + base = module_name.split(":")[0] + module = import_module(base) + get_context().logger.info("Starting server") + + uvicorn.run( + module_name, + host="0.0.0.0", + port=6060, + workers=1, + reload=not get_context().is_production, + log_config={ + **LOGGING_CONFIG, + "formatters": { + "default": { + "()": "great_ai.logger.CustomFormatter", + "fmt": "%(asctime)s | %(levelname)8s | %(message)s", + }, + "access": { + "()": "great_ai.logger.CustomFormatter", + "fmt": "%(asctime)s | %(levelname)8s | %(message)s", # noqa: E501 + }, + }, + }, + ) diff --git a/great_ai/src/great_ai/great_ai/__init__.py b/great_ai/src/great_ai/great_ai/__init__.py index c2f67f4..ad88332 100644 --- a/great_ai/src/great_ai/great_ai/__init__.py +++ b/great_ai/src/great_ai/great_ai/__init__.py @@ -1,4 +1,4 @@ -from .context import set_default_config -from .deploy import process_batch, process_single, serve +from .context import configure +from .deploy import create_service, process_batch, process_single from .metrics import log_argument, log_metric from .models import save_model, use_model diff --git a/great_ai/src/great_ai/great_ai/context/__init__.py b/great_ai/src/great_ai/great_ai/context/__init__.py index 58cd719..a1d9540 100644 --- a/great_ai/src/great_ai/great_ai/context/__init__.py +++ b/great_ai/src/great_ai/great_ai/context/__init__.py @@ -1,2 +1,2 @@ from .context import Context -from .get_context import get_context, set_default_config +from .get_context import configure, get_context diff --git a/great_ai/src/great_ai/great_ai/context/get_context.py b/great_ai/src/great_ai/great_ai/context/get_context.py index 6949b78..24b1e40 100644 --- a/great_ai/src/great_ai/great_ai/context/get_context.py +++ b/great_ai/src/great_ai/great_ai/context/get_context.py @@ -16,26 +16,35 @@ PRODUCTION_KEY = "production" def get_context() -> Context: if _context is None: - set_default_config() + configure() return cast(Context, _context) -def set_default_config( +def configure( log_level: int = INFO, s3_config: Path = Path("s3.ini"), seed: int = 42, persistence_driver: PersistenceDriver = ParallelTinyDbDriver( Path("tracing_database.json") ), - is_production_mode_override: Optional[bool] = None, + development_mode_override: Optional[bool] = None, ) -> None: global _context logger = create_logger("great_ai", level=log_level) + if _context is not None: + logger.warn( + "Configuration has been already initialised, overwriting.\n" + + 'Make sure to call "configure()" before importing your application code.' + ) + is_production = _is_in_production_mode( - override=is_production_mode_override, logger=logger + override=None + if development_mode_override is None + else not development_mode_override, + logger=logger, ) _initialize_large_file(s3_config, logger=logger) _set_seed(seed) @@ -51,7 +60,7 @@ def set_default_config( logger=logger, ) - logger.info("Defaults: configured ✅") + logger.info("Options: configured ✅") def _is_in_production_mode(override: Optional[bool], logger: Logger) -> bool: diff --git a/great_ai/src/great_ai/great_ai/deploy/__init__.py b/great_ai/src/great_ai/great_ai/deploy/__init__.py index c274dbd..456db68 100644 --- a/great_ai/src/great_ai/great_ai/deploy/__init__.py +++ b/great_ai/src/great_ai/great_ai/deploy/__init__.py @@ -1,3 +1,3 @@ +from .create_service import create_service from .process_batch import process_batch from .process_single import process_single -from .serve import serve diff --git a/great_ai/src/great_ai/great_ai/deploy/create_fastapi_app.py b/great_ai/src/great_ai/great_ai/deploy/create_service.py similarity index 54% rename from great_ai/src/great_ai/great_ai/deploy/create_fastapi_app.py rename to great_ai/src/great_ai/great_ai/deploy/create_service.py index 55c03b4..3c70eac 100644 --- a/great_ai/src/great_ai/great_ai/deploy/create_fastapi_app.py +++ b/great_ai/src/great_ai/great_ai/deploy/create_service.py @@ -1,7 +1,7 @@ from pathlib import Path -from typing import Any, Dict, List +from typing import Any, Callable, Dict, List -from fastapi import FastAPI, status +from fastapi import FastAPI, HTTPException, status from fastapi.middleware.wsgi import WSGIMiddleware from fastapi.openapi.docs import get_swagger_ui_html from fastapi.responses import RedirectResponse @@ -11,21 +11,51 @@ from starlette.responses import HTMLResponse from ..context import get_context from ..helper import snake_case_to_text from ..metrics import create_dash_app -from ..views import HealthCheckResponse, Query +from ..tracing import TracingContext +from ..views import EvaluationFeedbackRequest, HealthCheckResponse, Query, Trace PATH = Path(__file__).parent.resolve() -def create_fastapi_app( - function_name: str, disable_docs: bool, disable_metrics: bool +def create_service( + func: Callable[..., Any], disable_docs: bool = False, disable_metrics: bool = False ) -> FastAPI: + function_name = func.__name__ + function_docs = func.__doc__ + + documentation = ( + f"REST API wrapper for interacting with the '{function_name}' function.\n" + ) + if function_docs: + documentation += function_docs + app = FastAPI( title=snake_case_to_text(function_name), - description=f"REST API wrapper for interacting with the '{function_name}' function.", + description=documentation, docs_url=None, redoc_url=None, ) + @app.post("/evaluations", status_code=status.HTTP_200_OK, response_model=Trace) + def score(input: Any) -> Trace: + with TracingContext() as t: + result = func(input) + output = t.log_output(result) + return output + + @app.get("/evaluations/:evaluation_id", status_code=status.HTTP_200_OK) + def get_evaluation(evaluation_id: str) -> Trace: + result = get_context().persistence.get_trace(evaluation_id) + if result is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND) + return result + + @app.post( + "/evaluations/:evaluation_id/feedback", status_code=status.HTTP_202_ACCEPTED + ) + def give_feedback(evaluation_id: str, input: EvaluationFeedbackRequest) -> None: + get_context().persistence.add_evaluation(evaluation_id, input.evaluation) + if not disable_docs: @app.get("/docs", include_in_schema=False) @@ -37,7 +67,7 @@ def create_fastapi_app( return RedirectResponse("/docs") if not disable_metrics: - dash_app = create_dash_app(function_name) + dash_app = create_dash_app(function_name, documentation) app.mount(get_context().metrics_path, WSGIMiddleware(dash_app)) @app.get("/", include_in_schema=False) diff --git a/great_ai/src/great_ai/great_ai/deploy/serve.py b/great_ai/src/great_ai/great_ai/deploy/serve.py deleted file mode 100644 index b9117a9..0000000 --- a/great_ai/src/great_ai/great_ai/deploy/serve.py +++ /dev/null @@ -1,48 +0,0 @@ -from typing import Any, Callable - -import uvicorn -from fastapi import FastAPI, status -from uvicorn.config import LOGGING_CONFIG - -from ..tracing import TracingContext -from ..views import Trace -from .create_fastapi_app import create_fastapi_app - - -def serve( - function: Callable[..., Any], - disable_docs: bool = False, - disable_metrics: bool = False, - configure: Callable[[FastAPI], None] = lambda _: None, -) -> None: - app = create_fastapi_app( - function.__name__, disable_docs=disable_docs, disable_metrics=disable_metrics - ) - - @app.post("/score", status_code=status.HTTP_200_OK, response_model=Trace) - def process(input: Any) -> Trace: - with TracingContext() as t: - result = function(input) - output = t.log_output(result) - return output - - configure(app) - - uvicorn.run( - app, - host="0.0.0.0", - port=5050, - log_config={ - **LOGGING_CONFIG, - "formatters": { - "default": { - "()": "great_ai.logger.CustomFormatter", - "fmt": "%(asctime)s | %(levelname)8s | %(message)s", - }, - "access": { - "()": "great_ai.logger.CustomFormatter", - "fmt": "%(asctime)s | %(levelname)8s | %(message)s", # noqa: E501 - }, - }, - }, - ) diff --git a/great_ai/src/great_ai/great_ai/exceptions/__init__.py b/great_ai/src/great_ai/great_ai/exceptions/__init__.py index ed0a91a..037b1aa 100644 --- a/great_ai/src/great_ai/great_ai/exceptions/__init__.py +++ b/great_ai/src/great_ai/great_ai/exceptions/__init__.py @@ -1 +1,2 @@ from .argument_validation_error import ArgumentValidationError +from .missing_argument_error import MissingArgumentError diff --git a/great_ai/src/great_ai/great_ai/exceptions/missing_argument_error.py b/great_ai/src/great_ai/great_ai/exceptions/missing_argument_error.py new file mode 100644 index 0000000..38786e9 --- /dev/null +++ b/great_ai/src/great_ai/great_ai/exceptions/missing_argument_error.py @@ -0,0 +1,2 @@ +class MissingArgumentError(Exception): + pass diff --git a/great_ai/src/great_ai/great_ai/helper/__init__.py b/great_ai/src/great_ai/great_ai/helper/__init__.py index 32e14ef..ed156a0 100644 --- a/great_ai/src/great_ai/great_ai/helper/__init__.py +++ b/great_ai/src/great_ai/great_ai/helper/__init__.py @@ -1,3 +1,4 @@ from .get_args import get_args from .snake_case_to_text import snake_case_to_text +from .strip_lines import strip_lines from .text_to_hex_color import text_to_hex_color diff --git a/great_ai/src/great_ai/great_ai/helper/strip_lines.py b/great_ai/src/great_ai/great_ai/helper/strip_lines.py new file mode 100644 index 0000000..3cfbbba --- /dev/null +++ b/great_ai/src/great_ai/great_ai/helper/strip_lines.py @@ -0,0 +1,2 @@ +def strip_lines(text: str) -> str: + return "\n".join(line.strip() for line in text.split("\n")) diff --git a/great_ai/src/great_ai/great_ai/metrics/create_dash_app.py b/great_ai/src/great_ai/great_ai/metrics/create_dash_app.py index d11ec8c..cab63a3 100644 --- a/great_ai/src/great_ai/great_ai/metrics/create_dash_app.py +++ b/great_ai/src/great_ai/great_ai/metrics/create_dash_app.py @@ -17,7 +17,7 @@ from .get_filter_from_datatable import get_filter_from_datatable from .get_footer import get_footer -def create_dash_app(function_name: str) -> Flask: +def create_dash_app(function_name: str, function_docs: str) -> Flask: accent_color = text_to_hex_color(function_name) flask_app = Flask(__name__) @@ -55,7 +55,9 @@ def create_dash_app(function_name: str) -> Flask: html.Div( [ get_description( - function_name=function_name, accent_color=accent_color + function_name=function_name, + function_docs=function_docs, + accent_color=accent_color, ), execution_time_histogram, ], diff --git a/great_ai/src/great_ai/great_ai/metrics/get_description.py b/great_ai/src/great_ai/great_ai/metrics/get_description.py index 4059688..dce6a48 100644 --- a/great_ai/src/great_ai/great_ai/metrics/get_description.py +++ b/great_ai/src/great_ai/great_ai/metrics/get_description.py @@ -1,27 +1,36 @@ from dash import dcc, html -from ..helper import snake_case_to_text +from ..helper import snake_case_to_text, strip_lines -def get_description(function_name: str, accent_color: str) -> html.Div: - markdown_text = f""" - View the live data of your deployments here. - - ## Using the API - - You can find the available endpoints at [/docs](/docs). - - ## Metrics - - Recent traces and aggregated metrics are presented below. Try filtering the table. - """ - +def get_description( + function_name: str, function_docs: str, accent_color: str +) -> html.Div: return html.Div( [ html.H1( f"{snake_case_to_text(function_name)} - metrics", style={"color": accent_color}, ), - dcc.Markdown(markdown_text, className="description"), + dcc.Markdown( + strip_lines( + f""" + > View the live data of your deployments here. + + ## Using the API + + You can find the available endpoints at [/docs](/docs). + + ### Details + + {function_docs} + + ## Metrics + + Recent traces and aggregated metrics are presented below. Try filtering the table. + """ + ), + className="description", + ), ] ) diff --git a/great_ai/src/great_ai/great_ai/persistence/mongodb_driver.py b/great_ai/src/great_ai/great_ai/persistence/mongodb_driver.py index cd0812f..7b27a82 100644 --- a/great_ai/src/great_ai/great_ai/persistence/mongodb_driver.py +++ b/great_ai/src/great_ai/great_ai/persistence/mongodb_driver.py @@ -1,7 +1,5 @@ -from math import remainder from .persistence_driver import PersistenceDriver class MongoDbDriver(PersistenceDriver): pass - diff --git a/great_ai/src/great_ai/great_ai/persistence/parallel_tinydb_driver.py b/great_ai/src/great_ai/great_ai/persistence/parallel_tinydb_driver.py index 239a80a..0f1bfc4 100644 --- a/great_ai/src/great_ai/great_ai/persistence/parallel_tinydb_driver.py +++ b/great_ai/src/great_ai/great_ai/persistence/parallel_tinydb_driver.py @@ -22,9 +22,25 @@ class ParallelTinyDbDriver(PersistenceDriver): super().__init__() self._path_to_db = path_to_db - def save_document(self, trace: Trace) -> str: + def save_trace(self, trace: Trace) -> str: return self._safe_execute(lambda db: db.insert(trace.dict())) + def add_evaluation(self, id: str, evaluation: Any) -> None: + self._safe_execute( + lambda db: db.update( + fields={"evaluation": evaluation}, + cond=lambda d: d["evaluation_id"] == id, + ) + ) + + def get_trace(self, id: str) -> Optional[Trace]: + value = self._safe_execute( + lambda db: db.get(lambda d: d["evaluation_id"] == id) + ) + if value: + value = Trace.parse_obj(value) + return value + def get_traces(self) -> List[Trace]: return self._safe_execute(lambda db: [Trace.parse_obj(t) for t in db.all()]) diff --git a/great_ai/src/great_ai/great_ai/persistence/persistence_driver.py b/great_ai/src/great_ai/great_ai/persistence/persistence_driver.py index 8f9a1f4..7d2695f 100644 --- a/great_ai/src/great_ai/great_ai/persistence/persistence_driver.py +++ b/great_ai/src/great_ai/great_ai/persistence/persistence_driver.py @@ -10,7 +10,15 @@ class PersistenceDriver(ABC): is_threadsafe: bool @abstractmethod - def save_document(self, document: Trace) -> str: + def save_trace(self, document: Trace) -> str: + pass + + @abstractmethod + def add_evaluation(self, id: str, evaluation: Any) -> None: + pass + + @abstractmethod + def get_trace(self, id: str) -> Optional[Trace]: pass @abstractmethod diff --git a/great_ai/src/great_ai/great_ai/tracing/tracing_context.py b/great_ai/src/great_ai/great_ai/tracing/tracing_context.py index a770c3a..d2eefb4 100644 --- a/great_ai/src/great_ai/great_ai/tracing/tracing_context.py +++ b/great_ai/src/great_ai/great_ai/tracing/tracing_context.py @@ -59,7 +59,7 @@ class TracingContext: if type is None: assert self._trace is not None - get_context().persistence.save_document(self._trace) + get_context().persistence.save_trace(self._trace) else: get_context().logger.exception(f"Could not finish operation: {exception}") diff --git a/great_ai/src/great_ai/great_ai/views/__init__.py b/great_ai/src/great_ai/great_ai/views/__init__.py index 668939f..0ffd0c2 100644 --- a/great_ai/src/great_ai/great_ai/views/__init__.py +++ b/great_ai/src/great_ai/great_ai/views/__init__.py @@ -1,3 +1,4 @@ +from .evaluation_feedback_request import EvaluationFeedbackRequest from .filter import Filter from .health_check_response import HealthCheckResponse from .model import Model diff --git a/great_ai/src/great_ai/great_ai/views/evaluation_feedback_request.py b/great_ai/src/great_ai/great_ai/views/evaluation_feedback_request.py new file mode 100644 index 0000000..dc5db85 --- /dev/null +++ b/great_ai/src/great_ai/great_ai/views/evaluation_feedback_request.py @@ -0,0 +1,7 @@ +from typing import Any + +from pydantic import BaseModel + + +class EvaluationFeedbackRequest(BaseModel): + evaluation: Any diff --git a/great_ai/src/great_ai/great_ai/views/trace.py b/great_ai/src/great_ai/great_ai/views/trace.py index a6bb359..8357c65 100644 --- a/great_ai/src/great_ai/great_ai/views/trace.py +++ b/great_ai/src/great_ai/great_ai/views/trace.py @@ -29,6 +29,7 @@ class Trace(BaseModel): "created": self.created, "execution_time_ms": self.execution_time_ms, "models": ", ".join(f"{m.key}:{m.version}" for m in self.models), + "output": self.output, "evaluation": self.evaluation, **self.logged_values, } diff --git a/great_ai/src/sus.egg-info/PKG-INFO b/great_ai/src/sus.egg-info/PKG-INFO new file mode 100644 index 0000000..13a14e2 --- /dev/null +++ b/great_ai/src/sus.egg-info/PKG-INFO @@ -0,0 +1,50 @@ +Metadata-Version: 2.1 +Name: sus +Version: 0.0.3 +Summary: [S]coutinScience [u]tilitie[s]: reusable utilities for text processing +Home-page: https://github.com/ScoutinScience/platform +Author: ScoutinScience B.V. +Author-email: andras@scoutinscience.com +License: UNKNOWN +Project-URL: Bug Tracker, https://github.com/ScoutinScience/platform/issues +Platform: UNKNOWN +Classifier: Programming Language :: Python :: 3 +Classifier: Operating System :: OS Independent +Requires-Python: >=3.8 +Description-Content-Type: text/markdown + +# **S**coutinScience **U**tilitie**S** for text processing [![Lint and test ScoutinScience utilities](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml/badge.svg)](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml) + +> amogus + +## Exports + +- [clean](src/sus/clean.py) +- [unique](src/sus/unique.py) +- [parallel_map](src/sus/parallel_map.py) +- [match_names](src/sus/match_names/match_names.py) +- [evaluate_ranking](src/sus/evaluate_ranking/evaluate_ranking.py) + +### Requires loading spacy model + +> This is automatic but will require some time. + +> Add this to the Dockerfile for caching the spaCy model: +> +> ```docker +> RUN pip install --no-cache-dir en-core-web-lg@https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.2.0/en_core_web_lg-3.2.0-py3-none-any.whl +> ``` + +- [spacy model (nlp)](src/sus/nlp.py) +- [get_sentences](src/sus/get_sentences.py) +- [lemmatize_text](src/sus/lemmatize_text.py) +- [lemmatize_token](src/sus/lemmatize_token.py) +- [publication TEI](src/sus/publication_tei/publication_tei.py) + +## Development + +- Optional booleans must have a default value of `False`. +- No imports in top-level `__init__.py`, in order to not load anything unnecessary automatically +- Should only be updated through a PR + + diff --git a/great_ai/src/sus.egg-info/SOURCES.txt b/great_ai/src/sus.egg-info/SOURCES.txt new file mode 100644 index 0000000..6442db9 --- /dev/null +++ b/great_ai/src/sus.egg-info/SOURCES.txt @@ -0,0 +1,138 @@ +.gitignore +README.md +example_secrets.ini +open_s3.md +pyproject.toml +requirements.txt +setup.cfg +src/__init__.py +src/great_ai/__init__.py +src/great_ai/great_ai/__init__.py +src/great_ai/great_ai/context/__init__.py +src/great_ai/great_ai/context/context.py +src/great_ai/great_ai/context/get_context.py +src/great_ai/great_ai/deploy/__init__.py +src/great_ai/great_ai/deploy/create_fastapi_app.py +src/great_ai/great_ai/deploy/process_batch.py +src/great_ai/great_ai/deploy/process_single.py +src/great_ai/great_ai/deploy/serve.py +src/great_ai/great_ai/exceptions/__init__.py +src/great_ai/great_ai/exceptions/argument_validation_error.py +src/great_ai/great_ai/helper/__init__.py +src/great_ai/great_ai/helper/get_args.py +src/great_ai/great_ai/helper/snake_case_to_text.py +src/great_ai/great_ai/helper/text_to_hex_color.py +src/great_ai/great_ai/metrics/__init__.py +src/great_ai/great_ai/metrics/create_dash_app.py +src/great_ai/great_ai/metrics/get_description.py +src/great_ai/great_ai/metrics/get_filter_from_datatable.py +src/great_ai/great_ai/metrics/get_footer.py +src/great_ai/great_ai/metrics/log_argument.py +src/great_ai/great_ai/metrics/log_metric.py +src/great_ai/great_ai/metrics/assets/github.png +src/great_ai/great_ai/metrics/assets/index.css +src/great_ai/great_ai/models/__init__.py +src/great_ai/great_ai/models/load_model.py +src/great_ai/great_ai/models/save_model.py +src/great_ai/great_ai/models/use_model.py +src/great_ai/great_ai/persistence/__init__.py +src/great_ai/great_ai/persistence/mongodb_driver.py +src/great_ai/great_ai/persistence/parallel_tinydb_driver.py +src/great_ai/great_ai/persistence/persistence_driver.py +src/great_ai/great_ai/tracing/__init__.py +src/great_ai/great_ai/tracing/tracing_context.py +src/great_ai/great_ai/views/__init__.py +src/great_ai/great_ai/views/filter.py +src/great_ai/great_ai/views/health_check_response.py +src/great_ai/great_ai/views/model.py +src/great_ai/great_ai/views/operators.py +src/great_ai/great_ai/views/query.py +src/great_ai/great_ai/views/sort_by.py +src/great_ai/great_ai/views/trace.py +src/great_ai/open_s3/__init__.py +src/great_ai/open_s3/__main__.py +src/great_ai/open_s3/large_file.py +src/great_ai/open_s3/parse_arguments.py +src/great_ai/open_s3/helper/__init__.py +src/great_ai/open_s3/helper/bytes_to_megabytes.py +src/great_ai/open_s3/helper/human_readable_to_byte.py +src/great_ai/open_s3/helper/progress_bar.py +src/great_ai/utilities/__init__.py +src/great_ai/utilities/clean.py +src/great_ai/utilities/get_sentences.py +src/great_ai/utilities/lemmatize_text.py +src/great_ai/utilities/lemmatize_token.py +src/great_ai/utilities/nlp.py +src/great_ai/utilities/parallel_map.py +src/great_ai/utilities/unique.py +src/great_ai/utilities/data/__init__.py +src/great_ai/utilities/data/american_spellings.py +src/great_ai/utilities/data/punctuations.py +src/great_ai/utilities/evaluate_ranking/__init__.py +src/great_ai/utilities/evaluate_ranking/draw_f1_iso_lines.py +src/great_ai/utilities/evaluate_ranking/evaluate_ranking.py +src/great_ai/utilities/external/__init__.py +src/great_ai/utilities/external/negspacy/README.md +src/great_ai/utilities/external/negspacy/__init__.py +src/great_ai/utilities/external/negspacy/negation.py +src/great_ai/utilities/external/negspacy/termsets.py +src/great_ai/utilities/external/pylatexenc/README.md +src/great_ai/utilities/external/pylatexenc/__init__.py +src/great_ai/utilities/external/pylatexenc/_util.py +src/great_ai/utilities/external/pylatexenc/version.py +src/great_ai/utilities/external/pylatexenc/latex2text/__init__.py +src/great_ai/utilities/external/pylatexenc/latex2text/__main__.py +src/great_ai/utilities/external/pylatexenc/latex2text/_defaultspecs.py +src/great_ai/utilities/external/pylatexenc/latexencode/__init__.py +src/great_ai/utilities/external/pylatexenc/latexencode/__main__.py +src/great_ai/utilities/external/pylatexenc/latexencode/_partial_latex_encoder.py +src/great_ai/utilities/external/pylatexenc/latexencode/_uni2latexmap.py +src/great_ai/utilities/external/pylatexenc/latexencode/_uni2latexmap_xml.py +src/great_ai/utilities/external/pylatexenc/latexencode/_unicode_to_latex_encoder.py +src/great_ai/utilities/external/pylatexenc/latexwalker/__init__.py +src/great_ai/utilities/external/pylatexenc/latexwalker/_defaultspecs.py +src/great_ai/utilities/external/pylatexenc/macrospec/__init__.py +src/great_ai/utilities/external/pylatexenc/macrospec/_argparsers.py +src/great_ai/utilities/language/__init__.py +src/great_ai/utilities/language/english_name_of_language.py +src/great_ai/utilities/language/is_english.py +src/great_ai/utilities/language/predict_language.py +src/great_ai/utilities/logger/__init__.py +src/great_ai/utilities/logger/colors.py +src/great_ai/utilities/logger/create_logger.py +src/great_ai/utilities/logger/custom_formatter.py +src/great_ai/utilities/match_names/__init__.py +src/great_ai/utilities/match_names/config.py +src/great_ai/utilities/match_names/match_names.py +src/great_ai/utilities/match_names/name_parts.py +src/great_ai/utilities/publication_tei/__init__.py +src/great_ai/utilities/publication_tei/publication_tei.py +src/great_ai/utilities/publication_tei/models/__init__.py +src/great_ai/utilities/publication_tei/models/affiliation.py +src/great_ai/utilities/publication_tei/models/author.py +src/great_ai/utilities/publication_tei/models/element.py +src/great_ai/utilities/publication_tei/models/publication_metadata.py +src/great_ai/utilities/publication_tei/models/text.py +src/sus.egg-info/PKG-INFO +src/sus.egg-info/SOURCES.txt +src/sus.egg-info/dependency_links.txt +src/sus.egg-info/requires.txt +src/sus.egg-info/top_level.txt +tests/__init__.py +tests/open_s3/__init__.py +tests/open_s3/test_human_readable_to_byte.py +tests/open_s3/test_large_file.py +tests/sus/__init__.py +tests/sus/test_clean.py +tests/sus/test_evaluate_ranking.py +tests/sus/test_get_sentences.py +tests/sus/test_language.py +tests/sus/test_lemmatize_text.py +tests/sus/test_lemmatize_token.py +tests/sus/test_match_names.py +tests/sus/test_parallel_map.py +tests/sus/test_publication_tei.py +tests/sus/test_unique.py +tests/sus/data/10.1136_bmjspcare-2021-003026.pdf.tei.xml +tests/sus/data/bad.tei.xml +tests/sus/data/parsed.py \ No newline at end of file diff --git a/great_ai/src/sus.egg-info/dependency_links.txt b/great_ai/src/sus.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/great_ai/src/sus.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/great_ai/src/sus.egg-info/requires.txt b/great_ai/src/sus.egg-info/requires.txt new file mode 100644 index 0000000..1f9aec7 --- /dev/null +++ b/great_ai/src/sus.egg-info/requires.txt @@ -0,0 +1,14 @@ +click<8.1.0 +unidecode>=1.3.0 +multiprocess>=0.70.0.0 +tqdm>=4.0.0 +psutil>=5.9.0 +beautifulsoup4>=4.10.0 +lxml>=4.6.0 +spacy>=3.2.0 +pydantic>=1.8.0 +scikit-learn>=1.0.0 +matplotlib>=3.5.0 +numpy>=1.22.0 +langcodes[data]>=3.3.0 +langdetect>=1.0.9 diff --git a/great_ai/src/sus.egg-info/top_level.txt b/great_ai/src/sus.egg-info/top_level.txt new file mode 100644 index 0000000..f081433 --- /dev/null +++ b/great_ai/src/sus.egg-info/top_level.txt @@ -0,0 +1 @@ +great_ai