Fix tests

This commit is contained in:
Andras Schmelczer 2022-05-25 19:49:14 +02:00
parent 3b6134de2e
commit 2d71fceb54
18 changed files with 14 additions and 16 deletions

View file

@ -2,20 +2,20 @@
import re
from importlib import import_module
from sys import argv
import uvicorn
from uvicorn.config import LOGGING_CONFIG
from .great_ai.exceptions import MissingArgumentError
from .great_ai.context import get_context
from .great_ai.exceptions import MissingArgumentError
def main():
def main() -> None:
if len(argv) < 2:
raise MissingArgumentError(f"Provide a filename such as: {argv[0]} my_app.py")
module_name = re.sub(r"\.py\b", "", argv[1])
base = module_name.split(":")[0]
import_module(base)
@ -25,7 +25,7 @@ def main():
module_name += ":app"
get_context().logger.warning(
'Service name (name of variable) is assumed to be "app",'
+ ' such as: `app = create_service(predict_domain)`'
+ " such as: `app = create_service(predict_domain)`"
)
uvicorn.run(
@ -50,6 +50,7 @@ def main():
},
)
if __name__ == "__main__":
try:
main()

View file

@ -9,6 +9,7 @@ from .logger import create_logger
logger = create_logger("parallel_map")
def parallel_map(
function: Callable[[Any], Any],
values: Iterable[Any],
@ -27,7 +28,7 @@ def parallel_map(
logger.info(
f"Starting parallel map, concurrency: {concurrency}, chunk size: {chunk_size}"
)
if concurrency == 1 or len(values) <= chunk_size:
logger.warning(f"Running in series, there is no reason for parallelism")
iterable = values if disable_progress else tqdm(values)

View file

@ -6,7 +6,6 @@ from typing import Any, List, Optional, Pattern, Tuple, Union
from bs4 import BeautifulSoup
from bs4.element import NavigableString, Tag
from sus.publication_tei.models.element import Paragraph
from ..clean import clean
from ..lemmatize_text import lemmatize_text
@ -20,6 +19,7 @@ from .models import (
Element,
Meta,
MetaType,
Paragraph,
PublicationMetadata,
Text,
Title,
@ -30,8 +30,8 @@ THIS_FOLDER = Path(os.path.dirname(os.path.abspath(__file__)))
class PublicationTEI:
# remove template sentenaces, such as copyright notices
agressive_cleaning_enabled = True
# remove template sentences, such as copyright notices
aggressive_cleaning_enabled = True
def __init__(self, tei: str):
self._document_order_counter = 0
@ -324,7 +324,7 @@ class PublicationTEI:
if text is None:
return None
if self.agressive_cleaning_enabled:
if self.aggressive_cleaning_enabled:
filtered = filter_sentences(
text,
THIS_FOLDER / "templates.yaml",