diff --git a/great_ai/setup.cfg b/great_ai/setup.cfg index 709bd51..120311f 100644 --- a/great_ai/setup.cfg +++ b/great_ai/setup.cfg @@ -35,6 +35,7 @@ install_requires = langcodes[data] >= 3.3.0 segtok >= 1.5.11 langdetect >= 1.0.9 + pyaml >= 21.0.0 [options.package_data] * = *.json, *.yaml, *.yml diff --git a/great_ai/src/great_ai/__main__.py b/great_ai/src/great_ai/__main__.py index 29a1bdb..a94af48 100644 --- a/great_ai/src/great_ai/__main__.py +++ b/great_ai/src/great_ai/__main__.py @@ -2,20 +2,20 @@ import re from importlib import import_module from sys import argv + import uvicorn from uvicorn.config import LOGGING_CONFIG -from .great_ai.exceptions import MissingArgumentError from .great_ai.context import get_context +from .great_ai.exceptions import MissingArgumentError - -def main(): +def main() -> None: if len(argv) < 2: raise MissingArgumentError(f"Provide a filename such as: {argv[0]} my_app.py") module_name = re.sub(r"\.py\b", "", argv[1]) - + base = module_name.split(":")[0] import_module(base) @@ -25,7 +25,7 @@ def main(): module_name += ":app" get_context().logger.warning( 'Service name (name of variable) is assumed to be "app",' - + ' such as: `app = create_service(predict_domain)`' + + " such as: `app = create_service(predict_domain)`" ) uvicorn.run( @@ -50,6 +50,7 @@ def main(): }, ) + if __name__ == "__main__": try: main() diff --git a/great_ai/src/great_ai/utilities/parallel_map.py b/great_ai/src/great_ai/utilities/parallel_map.py index 8e0e6fc..d62ed6c 100644 --- a/great_ai/src/great_ai/utilities/parallel_map.py +++ b/great_ai/src/great_ai/utilities/parallel_map.py @@ -9,6 +9,7 @@ from .logger import create_logger logger = create_logger("parallel_map") + def parallel_map( function: Callable[[Any], Any], values: Iterable[Any], @@ -27,7 +28,7 @@ def parallel_map( logger.info( f"Starting parallel map, concurrency: {concurrency}, chunk size: {chunk_size}" ) - + if concurrency == 1 or len(values) <= chunk_size: logger.warning(f"Running in series, there is no reason for parallelism") iterable = values if disable_progress else tqdm(values) diff --git a/great_ai/src/great_ai/utilities/publication_tei/publication_tei.py b/great_ai/src/great_ai/utilities/publication_tei/publication_tei.py index c6a079e..49eeaa5 100644 --- a/great_ai/src/great_ai/utilities/publication_tei/publication_tei.py +++ b/great_ai/src/great_ai/utilities/publication_tei/publication_tei.py @@ -6,7 +6,6 @@ from typing import Any, List, Optional, Pattern, Tuple, Union from bs4 import BeautifulSoup from bs4.element import NavigableString, Tag -from sus.publication_tei.models.element import Paragraph from ..clean import clean from ..lemmatize_text import lemmatize_text @@ -20,6 +19,7 @@ from .models import ( Element, Meta, MetaType, + Paragraph, PublicationMetadata, Text, Title, @@ -30,8 +30,8 @@ THIS_FOLDER = Path(os.path.dirname(os.path.abspath(__file__))) class PublicationTEI: - # remove template sentenaces, such as copyright notices - agressive_cleaning_enabled = True + # remove template sentences, such as copyright notices + aggressive_cleaning_enabled = True def __init__(self, tei: str): self._document_order_counter = 0 @@ -324,7 +324,7 @@ class PublicationTEI: if text is None: return None - if self.agressive_cleaning_enabled: + if self.aggressive_cleaning_enabled: filtered = filter_sentences( text, THIS_FOLDER / "templates.yaml", diff --git a/great_ai/tests/sus/__init__.py b/great_ai/tests/utilities/__init__.py similarity index 100% rename from great_ai/tests/sus/__init__.py rename to great_ai/tests/utilities/__init__.py diff --git a/great_ai/tests/sus/data/10.1136_bmjspcare-2021-003026.pdf.tei.xml b/great_ai/tests/utilities/data/10.1136_bmjspcare-2021-003026.pdf.tei.xml similarity index 100% rename from great_ai/tests/sus/data/10.1136_bmjspcare-2021-003026.pdf.tei.xml rename to great_ai/tests/utilities/data/10.1136_bmjspcare-2021-003026.pdf.tei.xml diff --git a/great_ai/tests/sus/data/bad.tei.xml b/great_ai/tests/utilities/data/bad.tei.xml similarity index 100% rename from great_ai/tests/sus/data/bad.tei.xml rename to great_ai/tests/utilities/data/bad.tei.xml diff --git a/great_ai/tests/sus/data/parsed.py b/great_ai/tests/utilities/data/parsed.py similarity index 99% rename from great_ai/tests/sus/data/parsed.py rename to great_ai/tests/utilities/data/parsed.py index 9ce83cf..b861406 100644 --- a/great_ai/tests/sus/data/parsed.py +++ b/great_ai/tests/utilities/data/parsed.py @@ -1,4 +1,4 @@ -from sus.publication_tei.models import ( +from great_ai.utilities.publication_tei.models import ( Affiliation, Author, Bookmark, diff --git a/great_ai/tests/sus/test_clean.py b/great_ai/tests/utilities/test_clean.py similarity index 99% rename from great_ai/tests/sus/test_clean.py rename to great_ai/tests/utilities/test_clean.py index cb590be..0fdf9e4 100644 --- a/great_ai/tests/sus/test_clean.py +++ b/great_ai/tests/utilities/test_clean.py @@ -3,7 +3,6 @@ import unittest from src.great_ai.utilities.clean import clean - class TestClean(unittest.TestCase): def test_xml_handling(self) -> None: xml = 'Hi, my name
is AndrĂ¡s! <3 <> < ><> <> <|' diff --git a/great_ai/tests/sus/test_evaluate_ranking.py b/great_ai/tests/utilities/test_evaluate_ranking.py similarity index 100% rename from great_ai/tests/sus/test_evaluate_ranking.py rename to great_ai/tests/utilities/test_evaluate_ranking.py diff --git a/great_ai/tests/sus/test_get_sentences.py b/great_ai/tests/utilities/test_get_sentences.py similarity index 99% rename from great_ai/tests/sus/test_get_sentences.py rename to great_ai/tests/utilities/test_get_sentences.py index b03a091..51ffdec 100644 --- a/great_ai/tests/sus/test_get_sentences.py +++ b/great_ai/tests/utilities/test_get_sentences.py @@ -3,7 +3,6 @@ import unittest from src.great_ai.utilities.get_sentences import get_sentences - class TestGetSentences(unittest.TestCase): def test_default(self) -> None: text = "This is a complete sentence. So is this. However this is n" # ot. diff --git a/great_ai/tests/sus/test_language.py b/great_ai/tests/utilities/test_language.py similarity index 100% rename from great_ai/tests/sus/test_language.py rename to great_ai/tests/utilities/test_language.py diff --git a/great_ai/tests/sus/test_lemmatize_text.py b/great_ai/tests/utilities/test_lemmatize_text.py similarity index 100% rename from great_ai/tests/sus/test_lemmatize_text.py rename to great_ai/tests/utilities/test_lemmatize_text.py diff --git a/great_ai/tests/sus/test_lemmatize_token.py b/great_ai/tests/utilities/test_lemmatize_token.py similarity index 100% rename from great_ai/tests/sus/test_lemmatize_token.py rename to great_ai/tests/utilities/test_lemmatize_token.py diff --git a/great_ai/tests/sus/test_match_names.py b/great_ai/tests/utilities/test_match_names.py similarity index 99% rename from great_ai/tests/sus/test_match_names.py rename to great_ai/tests/utilities/test_match_names.py index f2fbd89..b4e5a4e 100644 --- a/great_ai/tests/sus/test_match_names.py +++ b/great_ai/tests/utilities/test_match_names.py @@ -3,7 +3,6 @@ import unittest from src.great_ai.utilities.match_names.match_names import match_names - class TestMatchNames(unittest.TestCase): def test_grid(self) -> None: names = [ diff --git a/great_ai/tests/sus/test_parallel_map.py b/great_ai/tests/utilities/test_parallel_map.py similarity index 100% rename from great_ai/tests/sus/test_parallel_map.py rename to great_ai/tests/utilities/test_parallel_map.py diff --git a/great_ai/tests/sus/test_publication_tei.py b/great_ai/tests/utilities/test_publication_tei.py similarity index 99% rename from great_ai/tests/sus/test_publication_tei.py rename to great_ai/tests/utilities/test_publication_tei.py index adbd6b1..390f73f 100644 --- a/great_ai/tests/sus/test_publication_tei.py +++ b/great_ai/tests/utilities/test_publication_tei.py @@ -3,7 +3,6 @@ from pathlib import Path from src.great_ai.utilities.publication_tei import PublicationTEI - from .data.parsed import ( abstract, authors, diff --git a/great_ai/tests/sus/test_unique.py b/great_ai/tests/utilities/test_unique.py similarity index 99% rename from great_ai/tests/sus/test_unique.py rename to great_ai/tests/utilities/test_unique.py index 21a9f5f..fc020c7 100644 --- a/great_ai/tests/sus/test_unique.py +++ b/great_ai/tests/utilities/test_unique.py @@ -2,7 +2,6 @@ import unittest from src.great_ai.utilities.unique import unique - original = [ ("a", 1), ("b", 5),