diff --git a/great_ai/setup.cfg b/great_ai/setup.cfg
index 709bd51..120311f 100644
--- a/great_ai/setup.cfg
+++ b/great_ai/setup.cfg
@@ -35,6 +35,7 @@ install_requires =
langcodes[data] >= 3.3.0
segtok >= 1.5.11
langdetect >= 1.0.9
+ pyaml >= 21.0.0
[options.package_data]
* = *.json, *.yaml, *.yml
diff --git a/great_ai/src/great_ai/__main__.py b/great_ai/src/great_ai/__main__.py
index 29a1bdb..a94af48 100644
--- a/great_ai/src/great_ai/__main__.py
+++ b/great_ai/src/great_ai/__main__.py
@@ -2,20 +2,20 @@
import re
from importlib import import_module
from sys import argv
+
import uvicorn
from uvicorn.config import LOGGING_CONFIG
-from .great_ai.exceptions import MissingArgumentError
from .great_ai.context import get_context
+from .great_ai.exceptions import MissingArgumentError
-
-def main():
+def main() -> None:
if len(argv) < 2:
raise MissingArgumentError(f"Provide a filename such as: {argv[0]} my_app.py")
module_name = re.sub(r"\.py\b", "", argv[1])
-
+
base = module_name.split(":")[0]
import_module(base)
@@ -25,7 +25,7 @@ def main():
module_name += ":app"
get_context().logger.warning(
'Service name (name of variable) is assumed to be "app",'
- + ' such as: `app = create_service(predict_domain)`'
+ + " such as: `app = create_service(predict_domain)`"
)
uvicorn.run(
@@ -50,6 +50,7 @@ def main():
},
)
+
if __name__ == "__main__":
try:
main()
diff --git a/great_ai/src/great_ai/utilities/parallel_map.py b/great_ai/src/great_ai/utilities/parallel_map.py
index 8e0e6fc..d62ed6c 100644
--- a/great_ai/src/great_ai/utilities/parallel_map.py
+++ b/great_ai/src/great_ai/utilities/parallel_map.py
@@ -9,6 +9,7 @@ from .logger import create_logger
logger = create_logger("parallel_map")
+
def parallel_map(
function: Callable[[Any], Any],
values: Iterable[Any],
@@ -27,7 +28,7 @@ def parallel_map(
logger.info(
f"Starting parallel map, concurrency: {concurrency}, chunk size: {chunk_size}"
)
-
+
if concurrency == 1 or len(values) <= chunk_size:
logger.warning(f"Running in series, there is no reason for parallelism")
iterable = values if disable_progress else tqdm(values)
diff --git a/great_ai/src/great_ai/utilities/publication_tei/publication_tei.py b/great_ai/src/great_ai/utilities/publication_tei/publication_tei.py
index c6a079e..49eeaa5 100644
--- a/great_ai/src/great_ai/utilities/publication_tei/publication_tei.py
+++ b/great_ai/src/great_ai/utilities/publication_tei/publication_tei.py
@@ -6,7 +6,6 @@ from typing import Any, List, Optional, Pattern, Tuple, Union
from bs4 import BeautifulSoup
from bs4.element import NavigableString, Tag
-from sus.publication_tei.models.element import Paragraph
from ..clean import clean
from ..lemmatize_text import lemmatize_text
@@ -20,6 +19,7 @@ from .models import (
Element,
Meta,
MetaType,
+ Paragraph,
PublicationMetadata,
Text,
Title,
@@ -30,8 +30,8 @@ THIS_FOLDER = Path(os.path.dirname(os.path.abspath(__file__)))
class PublicationTEI:
- # remove template sentenaces, such as copyright notices
- agressive_cleaning_enabled = True
+ # remove template sentences, such as copyright notices
+ aggressive_cleaning_enabled = True
def __init__(self, tei: str):
self._document_order_counter = 0
@@ -324,7 +324,7 @@ class PublicationTEI:
if text is None:
return None
- if self.agressive_cleaning_enabled:
+ if self.aggressive_cleaning_enabled:
filtered = filter_sentences(
text,
THIS_FOLDER / "templates.yaml",
diff --git a/great_ai/tests/sus/__init__.py b/great_ai/tests/utilities/__init__.py
similarity index 100%
rename from great_ai/tests/sus/__init__.py
rename to great_ai/tests/utilities/__init__.py
diff --git a/great_ai/tests/sus/data/10.1136_bmjspcare-2021-003026.pdf.tei.xml b/great_ai/tests/utilities/data/10.1136_bmjspcare-2021-003026.pdf.tei.xml
similarity index 100%
rename from great_ai/tests/sus/data/10.1136_bmjspcare-2021-003026.pdf.tei.xml
rename to great_ai/tests/utilities/data/10.1136_bmjspcare-2021-003026.pdf.tei.xml
diff --git a/great_ai/tests/sus/data/bad.tei.xml b/great_ai/tests/utilities/data/bad.tei.xml
similarity index 100%
rename from great_ai/tests/sus/data/bad.tei.xml
rename to great_ai/tests/utilities/data/bad.tei.xml
diff --git a/great_ai/tests/sus/data/parsed.py b/great_ai/tests/utilities/data/parsed.py
similarity index 99%
rename from great_ai/tests/sus/data/parsed.py
rename to great_ai/tests/utilities/data/parsed.py
index 9ce83cf..b861406 100644
--- a/great_ai/tests/sus/data/parsed.py
+++ b/great_ai/tests/utilities/data/parsed.py
@@ -1,4 +1,4 @@
-from sus.publication_tei.models import (
+from great_ai.utilities.publication_tei.models import (
Affiliation,
Author,
Bookmark,
diff --git a/great_ai/tests/sus/test_clean.py b/great_ai/tests/utilities/test_clean.py
similarity index 99%
rename from great_ai/tests/sus/test_clean.py
rename to great_ai/tests/utilities/test_clean.py
index cb590be..0fdf9e4 100644
--- a/great_ai/tests/sus/test_clean.py
+++ b/great_ai/tests/utilities/test_clean.py
@@ -3,7 +3,6 @@ import unittest
from src.great_ai.utilities.clean import clean
-
class TestClean(unittest.TestCase):
def test_xml_handling(self) -> None:
xml = 'Hi, my name
is AndrĂ¡s! <3 <> < >><> <> <|'
diff --git a/great_ai/tests/sus/test_evaluate_ranking.py b/great_ai/tests/utilities/test_evaluate_ranking.py
similarity index 100%
rename from great_ai/tests/sus/test_evaluate_ranking.py
rename to great_ai/tests/utilities/test_evaluate_ranking.py
diff --git a/great_ai/tests/sus/test_get_sentences.py b/great_ai/tests/utilities/test_get_sentences.py
similarity index 99%
rename from great_ai/tests/sus/test_get_sentences.py
rename to great_ai/tests/utilities/test_get_sentences.py
index b03a091..51ffdec 100644
--- a/great_ai/tests/sus/test_get_sentences.py
+++ b/great_ai/tests/utilities/test_get_sentences.py
@@ -3,7 +3,6 @@ import unittest
from src.great_ai.utilities.get_sentences import get_sentences
-
class TestGetSentences(unittest.TestCase):
def test_default(self) -> None:
text = "This is a complete sentence. So is this. However this is n" # ot.
diff --git a/great_ai/tests/sus/test_language.py b/great_ai/tests/utilities/test_language.py
similarity index 100%
rename from great_ai/tests/sus/test_language.py
rename to great_ai/tests/utilities/test_language.py
diff --git a/great_ai/tests/sus/test_lemmatize_text.py b/great_ai/tests/utilities/test_lemmatize_text.py
similarity index 100%
rename from great_ai/tests/sus/test_lemmatize_text.py
rename to great_ai/tests/utilities/test_lemmatize_text.py
diff --git a/great_ai/tests/sus/test_lemmatize_token.py b/great_ai/tests/utilities/test_lemmatize_token.py
similarity index 100%
rename from great_ai/tests/sus/test_lemmatize_token.py
rename to great_ai/tests/utilities/test_lemmatize_token.py
diff --git a/great_ai/tests/sus/test_match_names.py b/great_ai/tests/utilities/test_match_names.py
similarity index 99%
rename from great_ai/tests/sus/test_match_names.py
rename to great_ai/tests/utilities/test_match_names.py
index f2fbd89..b4e5a4e 100644
--- a/great_ai/tests/sus/test_match_names.py
+++ b/great_ai/tests/utilities/test_match_names.py
@@ -3,7 +3,6 @@ import unittest
from src.great_ai.utilities.match_names.match_names import match_names
-
class TestMatchNames(unittest.TestCase):
def test_grid(self) -> None:
names = [
diff --git a/great_ai/tests/sus/test_parallel_map.py b/great_ai/tests/utilities/test_parallel_map.py
similarity index 100%
rename from great_ai/tests/sus/test_parallel_map.py
rename to great_ai/tests/utilities/test_parallel_map.py
diff --git a/great_ai/tests/sus/test_publication_tei.py b/great_ai/tests/utilities/test_publication_tei.py
similarity index 99%
rename from great_ai/tests/sus/test_publication_tei.py
rename to great_ai/tests/utilities/test_publication_tei.py
index adbd6b1..390f73f 100644
--- a/great_ai/tests/sus/test_publication_tei.py
+++ b/great_ai/tests/utilities/test_publication_tei.py
@@ -3,7 +3,6 @@ from pathlib import Path
from src.great_ai.utilities.publication_tei import PublicationTEI
-
from .data.parsed import (
abstract,
authors,
diff --git a/great_ai/tests/sus/test_unique.py b/great_ai/tests/utilities/test_unique.py
similarity index 99%
rename from great_ai/tests/sus/test_unique.py
rename to great_ai/tests/utilities/test_unique.py
index 21a9f5f..fc020c7 100644
--- a/great_ai/tests/sus/test_unique.py
+++ b/great_ai/tests/utilities/test_unique.py
@@ -2,7 +2,6 @@ import unittest
from src.great_ai.utilities.unique import unique
-
original = [
("a", 1),
("b", 5),