Add auto-reload, evaluation endpoint, show docs

.vscode/settings.json

@@ -2,6 +2,7 @@
   "cSpell.words": [
     "boto",
     "botocore",
+    "fastapi",
     "iloc",
     "inplace",
     "levelno",

.vscode/tasks.json

@@ -4,9 +4,9 @@
     {
       "label": "Format and lint Python",
       "type": "shell",
-      "command": "source .env/bin/activate && scripts/format-python.sh great_ai && scripts/format-python.sh example",
+      "command": "source .env/bin/activate && scripts/format-python.sh great_ai && scripts/format-python.sh examples",
       "windows": {
-        "command": ".env/bin/activate.bat; scripts/format-python.sh great_ai; scripts\\format-python.sh example"
+        "command": ".env/bin/activate.bat; scripts/format-python.sh great_ai; scripts\\format-python.sh examples"
       },
       "group": "test",
       "presentation": {

example/main_service.py

@@ -1,9 +0,0 @@
-#!/usr/bin/env python3
-
-
-from predict_domain import predict_domain
-
-from great_ai import serve
-
-if __name__ == "__main__":
-    serve(predict_domain)

examples/README.md

@@ -1,4 +1,4 @@
-# Train Domain classifier from the [semantic scholar dataset](https://api.semanticscholar.org/corpus)
+# Train Domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)
 
 ## Upload the dataset (or a part of it) to shared infrastructure
 

examples/main_service.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python3
+
+from great_ai import configure, create_service
+
+configure(development_mode_override=True)
+
+from predict_domain import predict_domain
+
+app = create_service(predict_domain)

examples/predict_domain.py

@@ -20,7 +20,10 @@ class DomainPrediction(BaseModel):
 def predict_domain(
     text: str, model: Pipeline, cut_off_probability: float = 0.2
 ) -> List[DomainPrediction]:
-    assert 0 <= cut_off_probability <= 1
+    """
+    Predict the scientific domain of the input text.
+    Return labels until their sum likelihood is larger than cut_off_probability.
+    """
     log_metric("text_length", len(text))
 
     cleaned = clean(text, convert_to_ascii=True)

examples/train.ipynb

@@ -4,7 +4,7 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "# Train Domain classifier from the [semantic scholar dataset](https://api.semanticscholar.org/corpus)"
+        "# Train Domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)"
       ]
     },
     {

great_ai/src/great_ai/__main__.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+import re
+from importlib import import_module
+from sys import argv
+
+import uvicorn
+from uvicorn.config import LOGGING_CONFIG
+
+from great_ai.great_ai.context import get_context
+
+if __name__ == "__main__":
+    if len(argv) < 2:
+        raise ValueError(f"Provide a filename such as: {argv[0]} my_app.py")
+
+    module_name = re.sub(r"\.py\b", "", argv[1])
+    if ":" not in module_name:
+        module_name += ":app"
+
+    base = module_name.split(":")[0]
+    module = import_module(base)
+    get_context().logger.info("Starting server")
+
+    uvicorn.run(
+        module_name,
+        host="0.0.0.0",
+        port=6060,
+        workers=1,
+        reload=not get_context().is_production,
+        log_config={
+            **LOGGING_CONFIG,
+            "formatters": {
+                "default": {
+                    "()": "great_ai.logger.CustomFormatter",
+                    "fmt": "%(asctime)s | %(levelname)8s | %(message)s",
+                },
+                "access": {
+                    "()": "great_ai.logger.CustomFormatter",
+                    "fmt": "%(asctime)s | %(levelname)8s | %(message)s",  # noqa: E501
+                },
+            },
+        },
+    )

great_ai/src/great_ai/great_ai/__init__.py

@@ -1,4 +1,4 @@
-from .context import set_default_config
+from .context import configure
-from .deploy import process_batch, process_single, serve
+from .deploy import create_service, process_batch, process_single
 from .metrics import log_argument, log_metric
 from .models import save_model, use_model

great_ai/src/great_ai/great_ai/context/__init__.py

@@ -1,2 +1,2 @@
 from .context import Context
-from .get_context import get_context, set_default_config
+from .get_context import configure, get_context

great_ai/src/great_ai/great_ai/context/get_context.py

@@ -16,26 +16,35 @@ PRODUCTION_KEY = "production"
 
 def get_context() -> Context:
     if _context is None:
-        set_default_config()
+        configure()
 
     return cast(Context, _context)
 
 
-def set_default_config(
+def configure(
     log_level: int = INFO,
     s3_config: Path = Path("s3.ini"),
     seed: int = 42,
     persistence_driver: PersistenceDriver = ParallelTinyDbDriver(
         Path("tracing_database.json")
     ),
-    is_production_mode_override: Optional[bool] = None,
+    development_mode_override: Optional[bool] = None,
 ) -> None:
     global _context
 
     logger = create_logger("great_ai", level=log_level)
 
+    if _context is not None:
+        logger.warn(
+            "Configuration has been already initialised, overwriting.\n"
+            + 'Make sure to call "configure()" before importing your application code.'
+        )
+
     is_production = _is_in_production_mode(
-        override=is_production_mode_override, logger=logger
+        override=None
+        if development_mode_override is None
+        else not development_mode_override,
+        logger=logger,
     )
     _initialize_large_file(s3_config, logger=logger)
     _set_seed(seed)
@@ -51,7 +60,7 @@ def set_default_config(
         logger=logger,
     )
 
-    logger.info("Defaults: configured ✅")
+    logger.info("Options: configured ✅")
 
 
 def _is_in_production_mode(override: Optional[bool], logger: Logger) -> bool:

great_ai/src/great_ai/great_ai/deploy/__init__.py

@@ -1,3 +1,3 @@
+from .create_service import create_service
 from .process_batch import process_batch
 from .process_single import process_single
-from .serve import serve

great_ai/src/great_ai/great_ai/deploy/create_service.py

@@ -1,7 +1,7 @@
 from pathlib import Path
-from typing import Any, Dict, List
+from typing import Any, Callable, Dict, List
 
-from fastapi import FastAPI, status
+from fastapi import FastAPI, HTTPException, status
 from fastapi.middleware.wsgi import WSGIMiddleware
 from fastapi.openapi.docs import get_swagger_ui_html
 from fastapi.responses import RedirectResponse
@@ -11,21 +11,51 @@ from starlette.responses import HTMLResponse
 from ..context import get_context
 from ..helper import snake_case_to_text
 from ..metrics import create_dash_app
-from ..views import HealthCheckResponse, Query
+from ..tracing import TracingContext
+from ..views import EvaluationFeedbackRequest, HealthCheckResponse, Query, Trace
 
 PATH = Path(__file__).parent.resolve()
 
 
-def create_fastapi_app(
+def create_service(
-    function_name: str, disable_docs: bool, disable_metrics: bool
+    func: Callable[..., Any], disable_docs: bool = False, disable_metrics: bool = False
 ) -> FastAPI:
+    function_name = func.__name__
+    function_docs = func.__doc__
+
+    documentation = (
+        f"REST API wrapper for interacting with the '{function_name}' function.\n"
+    )
+    if function_docs:
+        documentation += function_docs
+
     app = FastAPI(
         title=snake_case_to_text(function_name),
-        description=f"REST API wrapper for interacting with the '{function_name}' function.",
+        description=documentation,
         docs_url=None,
         redoc_url=None,
     )
 
+    @app.post("/evaluations", status_code=status.HTTP_200_OK, response_model=Trace)
+    def score(input: Any) -> Trace:
+        with TracingContext() as t:
+            result = func(input)
+            output = t.log_output(result)
+        return output
+
+    @app.get("/evaluations/:evaluation_id", status_code=status.HTTP_200_OK)
+    def get_evaluation(evaluation_id: str) -> Trace:
+        result = get_context().persistence.get_trace(evaluation_id)
+        if result is None:
+            raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
+        return result
+
+    @app.post(
+        "/evaluations/:evaluation_id/feedback", status_code=status.HTTP_202_ACCEPTED
+    )
+    def give_feedback(evaluation_id: str, input: EvaluationFeedbackRequest) -> None:
+        get_context().persistence.add_evaluation(evaluation_id, input.evaluation)
+
     if not disable_docs:
 
         @app.get("/docs", include_in_schema=False)
@@ -37,7 +67,7 @@ def create_fastapi_app(
             return RedirectResponse("/docs")
 
     if not disable_metrics:
-        dash_app = create_dash_app(function_name)
+        dash_app = create_dash_app(function_name, documentation)
         app.mount(get_context().metrics_path, WSGIMiddleware(dash_app))
 
         @app.get("/", include_in_schema=False)

great_ai/src/great_ai/great_ai/deploy/serve.py

@@ -1,48 +0,0 @@
-from typing import Any, Callable
-
-import uvicorn
-from fastapi import FastAPI, status
-from uvicorn.config import LOGGING_CONFIG
-
-from ..tracing import TracingContext
-from ..views import Trace
-from .create_fastapi_app import create_fastapi_app
-
-
-def serve(
-    function: Callable[..., Any],
-    disable_docs: bool = False,
-    disable_metrics: bool = False,
-    configure: Callable[[FastAPI], None] = lambda _: None,
-) -> None:
-    app = create_fastapi_app(
-        function.__name__, disable_docs=disable_docs, disable_metrics=disable_metrics
-    )
-
-    @app.post("/score", status_code=status.HTTP_200_OK, response_model=Trace)
-    def process(input: Any) -> Trace:
-        with TracingContext() as t:
-            result = function(input)
-            output = t.log_output(result)
-        return output
-
-    configure(app)
-
-    uvicorn.run(
-        app,
-        host="0.0.0.0",
-        port=5050,
-        log_config={
-            **LOGGING_CONFIG,
-            "formatters": {
-                "default": {
-                    "()": "great_ai.logger.CustomFormatter",
-                    "fmt": "%(asctime)s | %(levelname)8s | %(message)s",
-                },
-                "access": {
-                    "()": "great_ai.logger.CustomFormatter",
-                    "fmt": "%(asctime)s | %(levelname)8s | %(message)s",  # noqa: E501
-                },
-            },
-        },
-    )

great_ai/src/great_ai/great_ai/exceptions/__init__.py

@@ -1 +1,2 @@
 from .argument_validation_error import ArgumentValidationError
+from .missing_argument_error import MissingArgumentError

great_ai/src/great_ai/great_ai/exceptions/missing_argument_error.py

@@ -0,0 +1,2 @@
+class MissingArgumentError(Exception):
+    pass

great_ai/src/great_ai/great_ai/helper/__init__.py

@@ -1,3 +1,4 @@
 from .get_args import get_args
 from .snake_case_to_text import snake_case_to_text
+from .strip_lines import strip_lines
 from .text_to_hex_color import text_to_hex_color

great_ai/src/great_ai/great_ai/helper/strip_lines.py

@@ -0,0 +1,2 @@
+def strip_lines(text: str) -> str:
+    return "\n".join(line.strip() for line in text.split("\n"))

great_ai/src/great_ai/great_ai/metrics/create_dash_app.py

@@ -17,7 +17,7 @@ from .get_filter_from_datatable import get_filter_from_datatable
 from .get_footer import get_footer
 
 
-def create_dash_app(function_name: str) -> Flask:
+def create_dash_app(function_name: str, function_docs: str) -> Flask:
     accent_color = text_to_hex_color(function_name)
 
     flask_app = Flask(__name__)
@@ -55,7 +55,9 @@ def create_dash_app(function_name: str) -> Flask:
             html.Div(
                 [
                     get_description(
-                        function_name=function_name, accent_color=accent_color
+                        function_name=function_name,
+                        function_docs=function_docs,
+                        accent_color=accent_color,
                     ),
                     execution_time_histogram,
                 ],

great_ai/src/great_ai/great_ai/metrics/get_description.py

@@ -1,27 +1,36 @@
 from dash import dcc, html
 
-from ..helper import snake_case_to_text
+from ..helper import snake_case_to_text, strip_lines
 
 
-def get_description(function_name: str, accent_color: str) -> html.Div:
+def get_description(
-    markdown_text = f"""
+    function_name: str, function_docs: str, accent_color: str
-    View the live data of your deployments here.
+) -> html.Div:
-
-    ## Using the API
-
-    You can find the available endpoints at [/docs](/docs).
-
-    ## Metrics
-
-    Recent traces and aggregated metrics are presented below. Try filtering the table.
-    """
-
     return html.Div(
         [
             html.H1(
                 f"{snake_case_to_text(function_name)} - metrics",
                 style={"color": accent_color},
             ),
-            dcc.Markdown(markdown_text, className="description"),
+            dcc.Markdown(
+                strip_lines(
+                    f"""
+                > View the live data of your deployments here.
+
+                ## Using the API
+
+                You can find the available endpoints at [/docs](/docs).
+
+                ### Details
+
+                {function_docs}
+
+                ## Metrics
+
+                Recent traces and aggregated metrics are presented below. Try filtering the table.
+                """
+                ),
+                className="description",
+            ),
         ]
     )

great_ai/src/great_ai/great_ai/persistence/mongodb_driver.py

@@ -1,7 +1,5 @@
-from math import remainder
 from .persistence_driver import PersistenceDriver
 
 
 class MongoDbDriver(PersistenceDriver):
     pass
-

great_ai/src/great_ai/great_ai/persistence/parallel_tinydb_driver.py

@@ -22,9 +22,25 @@ class ParallelTinyDbDriver(PersistenceDriver):
         super().__init__()
         self._path_to_db = path_to_db
 
-    def save_document(self, trace: Trace) -> str:
+    def save_trace(self, trace: Trace) -> str:
         return self._safe_execute(lambda db: db.insert(trace.dict()))
 
+    def add_evaluation(self, id: str, evaluation: Any) -> None:
+        self._safe_execute(
+            lambda db: db.update(
+                fields={"evaluation": evaluation},
+                cond=lambda d: d["evaluation_id"] == id,
+            )
+        )
+
+    def get_trace(self, id: str) -> Optional[Trace]:
+        value = self._safe_execute(
+            lambda db: db.get(lambda d: d["evaluation_id"] == id)
+        )
+        if value:
+            value = Trace.parse_obj(value)
+        return value
+
     def get_traces(self) -> List[Trace]:
         return self._safe_execute(lambda db: [Trace.parse_obj(t) for t in db.all()])
 

great_ai/src/great_ai/great_ai/persistence/persistence_driver.py

@@ -10,7 +10,15 @@ class PersistenceDriver(ABC):
     is_threadsafe: bool
 
     @abstractmethod
-    def save_document(self, document: Trace) -> str:
+    def save_trace(self, document: Trace) -> str:
+        pass
+
+    @abstractmethod
+    def add_evaluation(self, id: str, evaluation: Any) -> None:
+        pass
+
+    @abstractmethod
+    def get_trace(self, id: str) -> Optional[Trace]:
         pass
 
     @abstractmethod

great_ai/src/great_ai/great_ai/tracing/tracing_context.py

@@ -59,7 +59,7 @@ class TracingContext:
 
         if type is None:
             assert self._trace is not None
-            get_context().persistence.save_document(self._trace)
+            get_context().persistence.save_trace(self._trace)
         else:
             get_context().logger.exception(f"Could not finish operation: {exception}")
 

great_ai/src/great_ai/great_ai/views/__init__.py

@@ -1,3 +1,4 @@
+from .evaluation_feedback_request import EvaluationFeedbackRequest
 from .filter import Filter
 from .health_check_response import HealthCheckResponse
 from .model import Model

great_ai/src/great_ai/great_ai/views/evaluation_feedback_request.py

@@ -0,0 +1,7 @@
+from typing import Any
+
+from pydantic import BaseModel
+
+
+class EvaluationFeedbackRequest(BaseModel):
+    evaluation: Any

great_ai/src/great_ai/great_ai/views/trace.py

@@ -29,6 +29,7 @@ class Trace(BaseModel):
             "created": self.created,
             "execution_time_ms": self.execution_time_ms,
             "models": ", ".join(f"{m.key}:{m.version}" for m in self.models),
+            "output": self.output,
             "evaluation": self.evaluation,
             **self.logged_values,
         }

great_ai/src/sus.egg-info/PKG-INFO

@@ -0,0 +1,50 @@
+Metadata-Version: 2.1
+Name: sus
+Version: 0.0.3
+Summary: [S]coutinScience [u]tilitie[s]: reusable utilities for text processing
+Home-page: https://github.com/ScoutinScience/platform
+Author: ScoutinScience B.V.
+Author-email: andras@scoutinscience.com
+License: UNKNOWN
+Project-URL: Bug Tracker, https://github.com/ScoutinScience/platform/issues
+Platform: UNKNOWN
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+
+# **S**coutinScience **U**tilitie**S** for text processing [![Lint and test ScoutinScience utilities](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml/badge.svg)](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml)
+
+> amogus
+
+## Exports
+
+- [clean](src/sus/clean.py)
+- [unique](src/sus/unique.py)
+- [parallel_map](src/sus/parallel_map.py)
+- [match_names](src/sus/match_names/match_names.py)
+- [evaluate_ranking](src/sus/evaluate_ranking/evaluate_ranking.py)
+
+### Requires loading spacy model
+
+> This is automatic but will require some time.
+
+> Add this to the Dockerfile for caching the spaCy model:
+>
+> ```docker
+> RUN pip install --no-cache-dir en-core-web-lg@https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.2.0/en_core_web_lg-3.2.0-py3-none-any.whl
+> ```
+
+- [spacy model (nlp)](src/sus/nlp.py)
+- [get_sentences](src/sus/get_sentences.py)
+- [lemmatize_text](src/sus/lemmatize_text.py)
+- [lemmatize_token](src/sus/lemmatize_token.py)
+- [publication TEI](src/sus/publication_tei/publication_tei.py)
+
+## Development
+
+- Optional booleans must have a default value of `False`.
+- No imports in top-level `__init__.py`, in order to not load anything unnecessary automatically
+- Should only be updated through a PR
+
+

great_ai/src/sus.egg-info/SOURCES.txt

@@ -0,0 +1,138 @@
+.gitignore
+README.md
+example_secrets.ini
+open_s3.md
+pyproject.toml
+requirements.txt
+setup.cfg
+src/__init__.py
+src/great_ai/__init__.py
+src/great_ai/great_ai/__init__.py
+src/great_ai/great_ai/context/__init__.py
+src/great_ai/great_ai/context/context.py
+src/great_ai/great_ai/context/get_context.py
+src/great_ai/great_ai/deploy/__init__.py
+src/great_ai/great_ai/deploy/create_fastapi_app.py
+src/great_ai/great_ai/deploy/process_batch.py
+src/great_ai/great_ai/deploy/process_single.py
+src/great_ai/great_ai/deploy/serve.py
+src/great_ai/great_ai/exceptions/__init__.py
+src/great_ai/great_ai/exceptions/argument_validation_error.py
+src/great_ai/great_ai/helper/__init__.py
+src/great_ai/great_ai/helper/get_args.py
+src/great_ai/great_ai/helper/snake_case_to_text.py
+src/great_ai/great_ai/helper/text_to_hex_color.py
+src/great_ai/great_ai/metrics/__init__.py
+src/great_ai/great_ai/metrics/create_dash_app.py
+src/great_ai/great_ai/metrics/get_description.py
+src/great_ai/great_ai/metrics/get_filter_from_datatable.py
+src/great_ai/great_ai/metrics/get_footer.py
+src/great_ai/great_ai/metrics/log_argument.py
+src/great_ai/great_ai/metrics/log_metric.py
+src/great_ai/great_ai/metrics/assets/github.png
+src/great_ai/great_ai/metrics/assets/index.css
+src/great_ai/great_ai/models/__init__.py
+src/great_ai/great_ai/models/load_model.py
+src/great_ai/great_ai/models/save_model.py
+src/great_ai/great_ai/models/use_model.py
+src/great_ai/great_ai/persistence/__init__.py
+src/great_ai/great_ai/persistence/mongodb_driver.py
+src/great_ai/great_ai/persistence/parallel_tinydb_driver.py
+src/great_ai/great_ai/persistence/persistence_driver.py
+src/great_ai/great_ai/tracing/__init__.py
+src/great_ai/great_ai/tracing/tracing_context.py
+src/great_ai/great_ai/views/__init__.py
+src/great_ai/great_ai/views/filter.py
+src/great_ai/great_ai/views/health_check_response.py
+src/great_ai/great_ai/views/model.py
+src/great_ai/great_ai/views/operators.py
+src/great_ai/great_ai/views/query.py
+src/great_ai/great_ai/views/sort_by.py
+src/great_ai/great_ai/views/trace.py
+src/great_ai/open_s3/__init__.py
+src/great_ai/open_s3/__main__.py
+src/great_ai/open_s3/large_file.py
+src/great_ai/open_s3/parse_arguments.py
+src/great_ai/open_s3/helper/__init__.py
+src/great_ai/open_s3/helper/bytes_to_megabytes.py
+src/great_ai/open_s3/helper/human_readable_to_byte.py
+src/great_ai/open_s3/helper/progress_bar.py
+src/great_ai/utilities/__init__.py
+src/great_ai/utilities/clean.py
+src/great_ai/utilities/get_sentences.py
+src/great_ai/utilities/lemmatize_text.py
+src/great_ai/utilities/lemmatize_token.py
+src/great_ai/utilities/nlp.py
+src/great_ai/utilities/parallel_map.py
+src/great_ai/utilities/unique.py
+src/great_ai/utilities/data/__init__.py
+src/great_ai/utilities/data/american_spellings.py
+src/great_ai/utilities/data/punctuations.py
+src/great_ai/utilities/evaluate_ranking/__init__.py
+src/great_ai/utilities/evaluate_ranking/draw_f1_iso_lines.py
+src/great_ai/utilities/evaluate_ranking/evaluate_ranking.py
+src/great_ai/utilities/external/__init__.py
+src/great_ai/utilities/external/negspacy/README.md
+src/great_ai/utilities/external/negspacy/__init__.py
+src/great_ai/utilities/external/negspacy/negation.py
+src/great_ai/utilities/external/negspacy/termsets.py
+src/great_ai/utilities/external/pylatexenc/README.md
+src/great_ai/utilities/external/pylatexenc/__init__.py
+src/great_ai/utilities/external/pylatexenc/_util.py
+src/great_ai/utilities/external/pylatexenc/version.py
+src/great_ai/utilities/external/pylatexenc/latex2text/__init__.py
+src/great_ai/utilities/external/pylatexenc/latex2text/__main__.py
+src/great_ai/utilities/external/pylatexenc/latex2text/_defaultspecs.py
+src/great_ai/utilities/external/pylatexenc/latexencode/__init__.py
+src/great_ai/utilities/external/pylatexenc/latexencode/__main__.py
+src/great_ai/utilities/external/pylatexenc/latexencode/_partial_latex_encoder.py
+src/great_ai/utilities/external/pylatexenc/latexencode/_uni2latexmap.py
+src/great_ai/utilities/external/pylatexenc/latexencode/_uni2latexmap_xml.py
+src/great_ai/utilities/external/pylatexenc/latexencode/_unicode_to_latex_encoder.py
+src/great_ai/utilities/external/pylatexenc/latexwalker/__init__.py
+src/great_ai/utilities/external/pylatexenc/latexwalker/_defaultspecs.py
+src/great_ai/utilities/external/pylatexenc/macrospec/__init__.py
+src/great_ai/utilities/external/pylatexenc/macrospec/_argparsers.py
+src/great_ai/utilities/language/__init__.py
+src/great_ai/utilities/language/english_name_of_language.py
+src/great_ai/utilities/language/is_english.py
+src/great_ai/utilities/language/predict_language.py
+src/great_ai/utilities/logger/__init__.py
+src/great_ai/utilities/logger/colors.py
+src/great_ai/utilities/logger/create_logger.py
+src/great_ai/utilities/logger/custom_formatter.py
+src/great_ai/utilities/match_names/__init__.py
+src/great_ai/utilities/match_names/config.py
+src/great_ai/utilities/match_names/match_names.py
+src/great_ai/utilities/match_names/name_parts.py
+src/great_ai/utilities/publication_tei/__init__.py
+src/great_ai/utilities/publication_tei/publication_tei.py
+src/great_ai/utilities/publication_tei/models/__init__.py
+src/great_ai/utilities/publication_tei/models/affiliation.py
+src/great_ai/utilities/publication_tei/models/author.py
+src/great_ai/utilities/publication_tei/models/element.py
+src/great_ai/utilities/publication_tei/models/publication_metadata.py
+src/great_ai/utilities/publication_tei/models/text.py
+src/sus.egg-info/PKG-INFO
+src/sus.egg-info/SOURCES.txt
+src/sus.egg-info/dependency_links.txt
+src/sus.egg-info/requires.txt
+src/sus.egg-info/top_level.txt
+tests/__init__.py
+tests/open_s3/__init__.py
+tests/open_s3/test_human_readable_to_byte.py
+tests/open_s3/test_large_file.py
+tests/sus/__init__.py
+tests/sus/test_clean.py
+tests/sus/test_evaluate_ranking.py
+tests/sus/test_get_sentences.py
+tests/sus/test_language.py
+tests/sus/test_lemmatize_text.py
+tests/sus/test_lemmatize_token.py
+tests/sus/test_match_names.py
+tests/sus/test_parallel_map.py
+tests/sus/test_publication_tei.py
+tests/sus/test_unique.py
+tests/sus/data/10.1136_bmjspcare-2021-003026.pdf.tei.xml
+tests/sus/data/bad.tei.xml
+tests/sus/data/parsed.py

great_ai/src/sus.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

great_ai/src/sus.egg-info/requires.txt

@@ -0,0 +1,14 @@
+click<8.1.0
+unidecode>=1.3.0
+multiprocess>=0.70.0.0
+tqdm>=4.0.0
+psutil>=5.9.0
+beautifulsoup4>=4.10.0
+lxml>=4.6.0
+spacy>=3.2.0
+pydantic>=1.8.0
+scikit-learn>=1.0.0
+matplotlib>=3.5.0
+numpy>=1.22.0
+langcodes[data]>=3.3.0
+langdetect>=1.0.9

great_ai/src/sus.egg-info/top_level.txt

@@ -0,0 +1 @@
+great_ai