From ae9521bc297bf61b29ecc7e1dadbe9999328af3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A1s=20Schmelczer?= Date: Thu, 26 May 2022 21:22:00 +0200 Subject: [PATCH] Minor fixes --- .github/workflows/check_example.yml | 26 ---- examples/complex/Dockerfile | 16 +- examples/complex/Dockerfile.development | 2 +- great_ai/requirements.txt | 16 +- great_ai/setup.cfg | 9 +- .../persistence/parallel_tinydb_driver.py | 3 +- .../persistence/persistence_driver.py | 4 +- great_ai/src/sus.egg-info/PKG-INFO | 50 ------- great_ai/src/sus.egg-info/SOURCES.txt | 138 ------------------ .../src/sus.egg-info/dependency_links.txt | 1 - great_ai/src/sus.egg-info/requires.txt | 14 -- great_ai/src/sus.egg-info/top_level.txt | 1 - 12 files changed, 34 insertions(+), 246 deletions(-) delete mode 100644 .github/workflows/check_example.yml delete mode 100644 great_ai/src/sus.egg-info/PKG-INFO delete mode 100644 great_ai/src/sus.egg-info/SOURCES.txt delete mode 100644 great_ai/src/sus.egg-info/dependency_links.txt delete mode 100644 great_ai/src/sus.egg-info/requires.txt delete mode 100644 great_ai/src/sus.egg-info/top_level.txt diff --git a/.github/workflows/check_example.yml b/.github/workflows/check_example.yml deleted file mode 100644 index 3231cfe..0000000 --- a/.github/workflows/check_example.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Check library - -on: - workflow_dispatch: - push: - branches: - - main - paths: - - "great_ai/**" - - ".github/workflows/*" - -jobs: - lint: - uses: ./.github/workflows/generic-lint-python.yml - with: - path_to_project: examples/complex - - sonar: - needs: lint - uses: ./.github/workflows/generic-sonarqube-python.yml - with: - project_name: great_ai - path_to_project: examples/complex - secrets: - sonar_token: ${{ secrets.SONAR_TOKEN }} - sonar_host: ${{ secrets.SONAR_HOST_URL }} diff --git a/examples/complex/Dockerfile b/examples/complex/Dockerfile index 21aa538..27c5f2a 100644 --- a/examples/complex/Dockerfile +++ b/examples/complex/Dockerfile @@ -1,7 +1,9 @@ -FROM python:3.8.12-slim-bullseye +FROM python:3.10.4-alpine3.16 ENV ENVIRONMENT production +ARG ENTRYPOINT main.py + WORKDIR /app RUN apt-get update &&\ @@ -9,17 +11,17 @@ RUN apt-get update &&\ rm -rf /var/lib/apt/lists/* RUN pip install --no-cache-dir en-core-web-sm@https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl -COPY sus sus -RUN pip install --no-cache-dir --use-feature=in-tree-build ./sus +COPY great_ai great_ai +RUN pip install --no-cache-dir ./great_ai COPY requirements.txt ./ RUN pip install --no-cache-dir --requirement requirements.txt COPY . . -RUN rm -rf sus secrets +RUN rm -rf great_ai -EXPOSE 5000 +EXPOSE 6060 -HEALTHCHECK --interval=60s --timeout=60s --start-period=90s --retries=5 CMD [ "curl", "--fail", "http://localhost:5000/health" ] +HEALTHCHECK --interval=60s --timeout=60s --start-period=90s --retries=5 CMD [ "curl", "--fail", "http://localhost:6060/health" ] -CMD ["uvicorn", "main:app", "--proxy-headers", "--backlog", "8196", "--timeout-keep-alive", "900", "--host", "0.0.0.0", "--port", "5000"] +CMD ["great_ai", "$ENTRYPOINT"] diff --git a/examples/complex/Dockerfile.development b/examples/complex/Dockerfile.development index ff9cdff..220447e 100644 --- a/examples/complex/Dockerfile.development +++ b/examples/complex/Dockerfile.development @@ -1,4 +1,4 @@ -FROM python:3.8.12-slim-bullseye +FROM python:3.10.4-slim-bullseye ENV ENVIRONMENT development diff --git a/great_ai/requirements.txt b/great_ai/requirements.txt index 4e916a5..0ac00d1 100644 --- a/great_ai/requirements.txt +++ b/great_ai/requirements.txt @@ -1,11 +1,23 @@ +click < 8.1.0 unidecode >= 1.3.0 multiprocess >= 0.70.0.0 tqdm >= 4.0.0 psutil >= 5.9.0 beautifulsoup4 >= 4.10.0 lxml >= 4.6.0 -spacy >= 3.2.0 +spacy >= 3.3.0 pydantic >= 1.8.0 -scikit-learn >= 1.0.0 +scikit-learn == 1.1.1 matplotlib >= 3.5.0 numpy >= 1.22.0 +langcodes[data] >= 3.3.0 +segtok >= 1.5.11 +langdetect >= 1.0.9 +tinydb >= 4.7.0 +pandas >= 1.4.0 +pyaml >= 21.0.0 +boto3 >= 1.23.0 +fastapi >= 0.70.0 +plotly >= 5.8.0 +dash >= 2.4.0 +uvicorn[standard] >= 0.17.0 diff --git a/great_ai/setup.cfg b/great_ai/setup.cfg index aba112e..cfbcd95 100644 --- a/great_ai/setup.cfg +++ b/great_ai/setup.cfg @@ -29,13 +29,20 @@ install_requires = lxml >= 4.6.0 spacy >= 3.3.0 pydantic >= 1.8.0 - scikit-learn >= 1.0.0 + scikit-learn == 1.1.1 matplotlib >= 3.5.0 numpy >= 1.22.0 langcodes[data] >= 3.3.0 segtok >= 1.5.11 langdetect >= 1.0.9 + tinydb >= 4.7.0 + pandas >= 1.4.0 pyaml >= 21.0.0 + boto3 >= 1.23.0 + fastapi >= 0.70.0 + plotly >= 5.8.0 + dash >= 2.4.0 + uvicorn[standard] >= 0.17.0 [options.package_data] * = *.json, *.yaml, *.yml, *.css diff --git a/great_ai/src/great_ai/great_ai/persistence/parallel_tinydb_driver.py b/great_ai/src/great_ai/great_ai/persistence/parallel_tinydb_driver.py index f3112c1..bd0976c 100644 --- a/great_ai/src/great_ai/great_ai/persistence/parallel_tinydb_driver.py +++ b/great_ai/src/great_ai/great_ai/persistence/parallel_tinydb_driver.py @@ -1,9 +1,8 @@ from multiprocessing import Lock from pathlib import Path -from typing import Any, Callable, Dict, Optional +from typing import Any, Callable, Dict, List, Optional import pandas as pd -from black import List from tinydb import TinyDB from ..views import Filter, SortBy, Trace diff --git a/great_ai/src/great_ai/great_ai/persistence/persistence_driver.py b/great_ai/src/great_ai/great_ai/persistence/persistence_driver.py index 7d2695f..d7137f2 100644 --- a/great_ai/src/great_ai/great_ai/persistence/persistence_driver.py +++ b/great_ai/src/great_ai/great_ai/persistence/persistence_driver.py @@ -1,7 +1,5 @@ from abc import ABC, abstractmethod -from typing import Any, Dict, Optional - -from black import List +from typing import Any, Dict, List, Optional from ..views import Filter, SortBy, Trace diff --git a/great_ai/src/sus.egg-info/PKG-INFO b/great_ai/src/sus.egg-info/PKG-INFO deleted file mode 100644 index 13a14e2..0000000 --- a/great_ai/src/sus.egg-info/PKG-INFO +++ /dev/null @@ -1,50 +0,0 @@ -Metadata-Version: 2.1 -Name: sus -Version: 0.0.3 -Summary: [S]coutinScience [u]tilitie[s]: reusable utilities for text processing -Home-page: https://github.com/ScoutinScience/platform -Author: ScoutinScience B.V. -Author-email: andras@scoutinscience.com -License: UNKNOWN -Project-URL: Bug Tracker, https://github.com/ScoutinScience/platform/issues -Platform: UNKNOWN -Classifier: Programming Language :: Python :: 3 -Classifier: Operating System :: OS Independent -Requires-Python: >=3.8 -Description-Content-Type: text/markdown - -# **S**coutinScience **U**tilitie**S** for text processing [![Lint and test ScoutinScience utilities](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml/badge.svg)](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml) - -> amogus - -## Exports - -- [clean](src/sus/clean.py) -- [unique](src/sus/unique.py) -- [parallel_map](src/sus/parallel_map.py) -- [match_names](src/sus/match_names/match_names.py) -- [evaluate_ranking](src/sus/evaluate_ranking/evaluate_ranking.py) - -### Requires loading spacy model - -> This is automatic but will require some time. - -> Add this to the Dockerfile for caching the spaCy model: -> -> ```docker -> RUN pip install --no-cache-dir en-core-web-lg@https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.2.0/en_core_web_lg-3.2.0-py3-none-any.whl -> ``` - -- [spacy model (nlp)](src/sus/nlp.py) -- [get_sentences](src/sus/get_sentences.py) -- [lemmatize_text](src/sus/lemmatize_text.py) -- [lemmatize_token](src/sus/lemmatize_token.py) -- [publication TEI](src/sus/publication_tei/publication_tei.py) - -## Development - -- Optional booleans must have a default value of `False`. -- No imports in top-level `__init__.py`, in order to not load anything unnecessary automatically -- Should only be updated through a PR - - diff --git a/great_ai/src/sus.egg-info/SOURCES.txt b/great_ai/src/sus.egg-info/SOURCES.txt deleted file mode 100644 index 6442db9..0000000 --- a/great_ai/src/sus.egg-info/SOURCES.txt +++ /dev/null @@ -1,138 +0,0 @@ -.gitignore -README.md -example_secrets.ini -open_s3.md -pyproject.toml -requirements.txt -setup.cfg -src/__init__.py -src/great_ai/__init__.py -src/great_ai/great_ai/__init__.py -src/great_ai/great_ai/context/__init__.py -src/great_ai/great_ai/context/context.py -src/great_ai/great_ai/context/get_context.py -src/great_ai/great_ai/deploy/__init__.py -src/great_ai/great_ai/deploy/create_fastapi_app.py -src/great_ai/great_ai/deploy/process_batch.py -src/great_ai/great_ai/deploy/process_single.py -src/great_ai/great_ai/deploy/serve.py -src/great_ai/great_ai/exceptions/__init__.py -src/great_ai/great_ai/exceptions/argument_validation_error.py -src/great_ai/great_ai/helper/__init__.py -src/great_ai/great_ai/helper/get_args.py -src/great_ai/great_ai/helper/snake_case_to_text.py -src/great_ai/great_ai/helper/text_to_hex_color.py -src/great_ai/great_ai/metrics/__init__.py -src/great_ai/great_ai/metrics/create_dash_app.py -src/great_ai/great_ai/metrics/get_description.py -src/great_ai/great_ai/metrics/get_filter_from_datatable.py -src/great_ai/great_ai/metrics/get_footer.py -src/great_ai/great_ai/metrics/log_argument.py -src/great_ai/great_ai/metrics/log_metric.py -src/great_ai/great_ai/metrics/assets/github.png -src/great_ai/great_ai/metrics/assets/index.css -src/great_ai/great_ai/models/__init__.py -src/great_ai/great_ai/models/load_model.py -src/great_ai/great_ai/models/save_model.py -src/great_ai/great_ai/models/use_model.py -src/great_ai/great_ai/persistence/__init__.py -src/great_ai/great_ai/persistence/mongodb_driver.py -src/great_ai/great_ai/persistence/parallel_tinydb_driver.py -src/great_ai/great_ai/persistence/persistence_driver.py -src/great_ai/great_ai/tracing/__init__.py -src/great_ai/great_ai/tracing/tracing_context.py -src/great_ai/great_ai/views/__init__.py -src/great_ai/great_ai/views/filter.py -src/great_ai/great_ai/views/health_check_response.py -src/great_ai/great_ai/views/model.py -src/great_ai/great_ai/views/operators.py -src/great_ai/great_ai/views/query.py -src/great_ai/great_ai/views/sort_by.py -src/great_ai/great_ai/views/trace.py -src/great_ai/open_s3/__init__.py -src/great_ai/open_s3/__main__.py -src/great_ai/open_s3/large_file.py -src/great_ai/open_s3/parse_arguments.py -src/great_ai/open_s3/helper/__init__.py -src/great_ai/open_s3/helper/bytes_to_megabytes.py -src/great_ai/open_s3/helper/human_readable_to_byte.py -src/great_ai/open_s3/helper/progress_bar.py -src/great_ai/utilities/__init__.py -src/great_ai/utilities/clean.py -src/great_ai/utilities/get_sentences.py -src/great_ai/utilities/lemmatize_text.py -src/great_ai/utilities/lemmatize_token.py -src/great_ai/utilities/nlp.py -src/great_ai/utilities/parallel_map.py -src/great_ai/utilities/unique.py -src/great_ai/utilities/data/__init__.py -src/great_ai/utilities/data/american_spellings.py -src/great_ai/utilities/data/punctuations.py -src/great_ai/utilities/evaluate_ranking/__init__.py -src/great_ai/utilities/evaluate_ranking/draw_f1_iso_lines.py -src/great_ai/utilities/evaluate_ranking/evaluate_ranking.py -src/great_ai/utilities/external/__init__.py -src/great_ai/utilities/external/negspacy/README.md -src/great_ai/utilities/external/negspacy/__init__.py -src/great_ai/utilities/external/negspacy/negation.py -src/great_ai/utilities/external/negspacy/termsets.py -src/great_ai/utilities/external/pylatexenc/README.md -src/great_ai/utilities/external/pylatexenc/__init__.py -src/great_ai/utilities/external/pylatexenc/_util.py -src/great_ai/utilities/external/pylatexenc/version.py -src/great_ai/utilities/external/pylatexenc/latex2text/__init__.py -src/great_ai/utilities/external/pylatexenc/latex2text/__main__.py -src/great_ai/utilities/external/pylatexenc/latex2text/_defaultspecs.py -src/great_ai/utilities/external/pylatexenc/latexencode/__init__.py -src/great_ai/utilities/external/pylatexenc/latexencode/__main__.py -src/great_ai/utilities/external/pylatexenc/latexencode/_partial_latex_encoder.py -src/great_ai/utilities/external/pylatexenc/latexencode/_uni2latexmap.py -src/great_ai/utilities/external/pylatexenc/latexencode/_uni2latexmap_xml.py -src/great_ai/utilities/external/pylatexenc/latexencode/_unicode_to_latex_encoder.py -src/great_ai/utilities/external/pylatexenc/latexwalker/__init__.py -src/great_ai/utilities/external/pylatexenc/latexwalker/_defaultspecs.py -src/great_ai/utilities/external/pylatexenc/macrospec/__init__.py -src/great_ai/utilities/external/pylatexenc/macrospec/_argparsers.py -src/great_ai/utilities/language/__init__.py -src/great_ai/utilities/language/english_name_of_language.py -src/great_ai/utilities/language/is_english.py -src/great_ai/utilities/language/predict_language.py -src/great_ai/utilities/logger/__init__.py -src/great_ai/utilities/logger/colors.py -src/great_ai/utilities/logger/create_logger.py -src/great_ai/utilities/logger/custom_formatter.py -src/great_ai/utilities/match_names/__init__.py -src/great_ai/utilities/match_names/config.py -src/great_ai/utilities/match_names/match_names.py -src/great_ai/utilities/match_names/name_parts.py -src/great_ai/utilities/publication_tei/__init__.py -src/great_ai/utilities/publication_tei/publication_tei.py -src/great_ai/utilities/publication_tei/models/__init__.py -src/great_ai/utilities/publication_tei/models/affiliation.py -src/great_ai/utilities/publication_tei/models/author.py -src/great_ai/utilities/publication_tei/models/element.py -src/great_ai/utilities/publication_tei/models/publication_metadata.py -src/great_ai/utilities/publication_tei/models/text.py -src/sus.egg-info/PKG-INFO -src/sus.egg-info/SOURCES.txt -src/sus.egg-info/dependency_links.txt -src/sus.egg-info/requires.txt -src/sus.egg-info/top_level.txt -tests/__init__.py -tests/open_s3/__init__.py -tests/open_s3/test_human_readable_to_byte.py -tests/open_s3/test_large_file.py -tests/sus/__init__.py -tests/sus/test_clean.py -tests/sus/test_evaluate_ranking.py -tests/sus/test_get_sentences.py -tests/sus/test_language.py -tests/sus/test_lemmatize_text.py -tests/sus/test_lemmatize_token.py -tests/sus/test_match_names.py -tests/sus/test_parallel_map.py -tests/sus/test_publication_tei.py -tests/sus/test_unique.py -tests/sus/data/10.1136_bmjspcare-2021-003026.pdf.tei.xml -tests/sus/data/bad.tei.xml -tests/sus/data/parsed.py \ No newline at end of file diff --git a/great_ai/src/sus.egg-info/dependency_links.txt b/great_ai/src/sus.egg-info/dependency_links.txt deleted file mode 100644 index 8b13789..0000000 --- a/great_ai/src/sus.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/great_ai/src/sus.egg-info/requires.txt b/great_ai/src/sus.egg-info/requires.txt deleted file mode 100644 index 1f9aec7..0000000 --- a/great_ai/src/sus.egg-info/requires.txt +++ /dev/null @@ -1,14 +0,0 @@ -click<8.1.0 -unidecode>=1.3.0 -multiprocess>=0.70.0.0 -tqdm>=4.0.0 -psutil>=5.9.0 -beautifulsoup4>=4.10.0 -lxml>=4.6.0 -spacy>=3.2.0 -pydantic>=1.8.0 -scikit-learn>=1.0.0 -matplotlib>=3.5.0 -numpy>=1.22.0 -langcodes[data]>=3.3.0 -langdetect>=1.0.9 diff --git a/great_ai/src/sus.egg-info/top_level.txt b/great_ai/src/sus.egg-info/top_level.txt deleted file mode 100644 index f081433..0000000 --- a/great_ai/src/sus.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -great_ai