Fix tests
This commit is contained in:
parent
3b6134de2e
commit
2d71fceb54
18 changed files with 14 additions and 16 deletions
|
|
@ -2,20 +2,20 @@
|
|||
import re
|
||||
from importlib import import_module
|
||||
from sys import argv
|
||||
|
||||
import uvicorn
|
||||
from uvicorn.config import LOGGING_CONFIG
|
||||
|
||||
from .great_ai.exceptions import MissingArgumentError
|
||||
from .great_ai.context import get_context
|
||||
from .great_ai.exceptions import MissingArgumentError
|
||||
|
||||
|
||||
|
||||
def main():
|
||||
def main() -> None:
|
||||
if len(argv) < 2:
|
||||
raise MissingArgumentError(f"Provide a filename such as: {argv[0]} my_app.py")
|
||||
|
||||
module_name = re.sub(r"\.py\b", "", argv[1])
|
||||
|
||||
|
||||
base = module_name.split(":")[0]
|
||||
|
||||
import_module(base)
|
||||
|
|
@ -25,7 +25,7 @@ def main():
|
|||
module_name += ":app"
|
||||
get_context().logger.warning(
|
||||
'Service name (name of variable) is assumed to be "app",'
|
||||
+ ' such as: `app = create_service(predict_domain)`'
|
||||
+ " such as: `app = create_service(predict_domain)`"
|
||||
)
|
||||
|
||||
uvicorn.run(
|
||||
|
|
@ -50,6 +50,7 @@ def main():
|
|||
},
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from .logger import create_logger
|
|||
|
||||
logger = create_logger("parallel_map")
|
||||
|
||||
|
||||
def parallel_map(
|
||||
function: Callable[[Any], Any],
|
||||
values: Iterable[Any],
|
||||
|
|
@ -27,7 +28,7 @@ def parallel_map(
|
|||
logger.info(
|
||||
f"Starting parallel map, concurrency: {concurrency}, chunk size: {chunk_size}"
|
||||
)
|
||||
|
||||
|
||||
if concurrency == 1 or len(values) <= chunk_size:
|
||||
logger.warning(f"Running in series, there is no reason for parallelism")
|
||||
iterable = values if disable_progress else tqdm(values)
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ from typing import Any, List, Optional, Pattern, Tuple, Union
|
|||
|
||||
from bs4 import BeautifulSoup
|
||||
from bs4.element import NavigableString, Tag
|
||||
from sus.publication_tei.models.element import Paragraph
|
||||
|
||||
from ..clean import clean
|
||||
from ..lemmatize_text import lemmatize_text
|
||||
|
|
@ -20,6 +19,7 @@ from .models import (
|
|||
Element,
|
||||
Meta,
|
||||
MetaType,
|
||||
Paragraph,
|
||||
PublicationMetadata,
|
||||
Text,
|
||||
Title,
|
||||
|
|
@ -30,8 +30,8 @@ THIS_FOLDER = Path(os.path.dirname(os.path.abspath(__file__)))
|
|||
|
||||
|
||||
class PublicationTEI:
|
||||
# remove template sentenaces, such as copyright notices
|
||||
agressive_cleaning_enabled = True
|
||||
# remove template sentences, such as copyright notices
|
||||
aggressive_cleaning_enabled = True
|
||||
|
||||
def __init__(self, tei: str):
|
||||
self._document_order_counter = 0
|
||||
|
|
@ -324,7 +324,7 @@ class PublicationTEI:
|
|||
if text is None:
|
||||
return None
|
||||
|
||||
if self.agressive_cleaning_enabled:
|
||||
if self.aggressive_cleaning_enabled:
|
||||
filtered = filter_sentences(
|
||||
text,
|
||||
THIS_FOLDER / "templates.yaml",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue