Minor fixes
This commit is contained in:
parent
317d9f535e
commit
8016660276
12 changed files with 34 additions and 246 deletions
26
.github/workflows/check_example.yml
vendored
26
.github/workflows/check_example.yml
vendored
|
|
@ -1,26 +0,0 @@
|
||||||
name: Check library
|
|
||||||
|
|
||||||
on:
|
|
||||||
workflow_dispatch:
|
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- main
|
|
||||||
paths:
|
|
||||||
- "great_ai/**"
|
|
||||||
- ".github/workflows/*"
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
lint:
|
|
||||||
uses: ./.github/workflows/generic-lint-python.yml
|
|
||||||
with:
|
|
||||||
path_to_project: examples/complex
|
|
||||||
|
|
||||||
sonar:
|
|
||||||
needs: lint
|
|
||||||
uses: ./.github/workflows/generic-sonarqube-python.yml
|
|
||||||
with:
|
|
||||||
project_name: great_ai
|
|
||||||
path_to_project: examples/complex
|
|
||||||
secrets:
|
|
||||||
sonar_token: ${{ secrets.SONAR_TOKEN }}
|
|
||||||
sonar_host: ${{ secrets.SONAR_HOST_URL }}
|
|
||||||
|
|
@ -1,7 +1,9 @@
|
||||||
FROM python:3.8.12-slim-bullseye
|
FROM python:3.10.4-alpine3.16
|
||||||
|
|
||||||
ENV ENVIRONMENT production
|
ENV ENVIRONMENT production
|
||||||
|
|
||||||
|
ARG ENTRYPOINT main.py
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
RUN apt-get update &&\
|
RUN apt-get update &&\
|
||||||
|
|
@ -9,17 +11,17 @@ RUN apt-get update &&\
|
||||||
rm -rf /var/lib/apt/lists/*
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
RUN pip install --no-cache-dir en-core-web-sm@https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl
|
RUN pip install --no-cache-dir en-core-web-sm@https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl
|
||||||
COPY sus sus
|
COPY great_ai great_ai
|
||||||
RUN pip install --no-cache-dir --use-feature=in-tree-build ./sus
|
RUN pip install --no-cache-dir ./great_ai
|
||||||
|
|
||||||
COPY requirements.txt ./
|
COPY requirements.txt ./
|
||||||
RUN pip install --no-cache-dir --requirement requirements.txt
|
RUN pip install --no-cache-dir --requirement requirements.txt
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
RUN rm -rf sus secrets
|
RUN rm -rf great_ai
|
||||||
|
|
||||||
EXPOSE 5000
|
EXPOSE 6060
|
||||||
|
|
||||||
HEALTHCHECK --interval=60s --timeout=60s --start-period=90s --retries=5 CMD [ "curl", "--fail", "http://localhost:5000/health" ]
|
HEALTHCHECK --interval=60s --timeout=60s --start-period=90s --retries=5 CMD [ "curl", "--fail", "http://localhost:6060/health" ]
|
||||||
|
|
||||||
CMD ["uvicorn", "main:app", "--proxy-headers", "--backlog", "8196", "--timeout-keep-alive", "900", "--host", "0.0.0.0", "--port", "5000"]
|
CMD ["great_ai", "$ENTRYPOINT"]
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
FROM python:3.8.12-slim-bullseye
|
FROM python:3.10.4-slim-bullseye
|
||||||
|
|
||||||
ENV ENVIRONMENT development
|
ENV ENVIRONMENT development
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,11 +1,23 @@
|
||||||
|
click < 8.1.0
|
||||||
unidecode >= 1.3.0
|
unidecode >= 1.3.0
|
||||||
multiprocess >= 0.70.0.0
|
multiprocess >= 0.70.0.0
|
||||||
tqdm >= 4.0.0
|
tqdm >= 4.0.0
|
||||||
psutil >= 5.9.0
|
psutil >= 5.9.0
|
||||||
beautifulsoup4 >= 4.10.0
|
beautifulsoup4 >= 4.10.0
|
||||||
lxml >= 4.6.0
|
lxml >= 4.6.0
|
||||||
spacy >= 3.2.0
|
spacy >= 3.3.0
|
||||||
pydantic >= 1.8.0
|
pydantic >= 1.8.0
|
||||||
scikit-learn >= 1.0.0
|
scikit-learn == 1.1.1
|
||||||
matplotlib >= 3.5.0
|
matplotlib >= 3.5.0
|
||||||
numpy >= 1.22.0
|
numpy >= 1.22.0
|
||||||
|
langcodes[data] >= 3.3.0
|
||||||
|
segtok >= 1.5.11
|
||||||
|
langdetect >= 1.0.9
|
||||||
|
tinydb >= 4.7.0
|
||||||
|
pandas >= 1.4.0
|
||||||
|
pyaml >= 21.0.0
|
||||||
|
boto3 >= 1.23.0
|
||||||
|
fastapi >= 0.70.0
|
||||||
|
plotly >= 5.8.0
|
||||||
|
dash >= 2.4.0
|
||||||
|
uvicorn[standard] >= 0.17.0
|
||||||
|
|
|
||||||
|
|
@ -29,13 +29,20 @@ install_requires =
|
||||||
lxml >= 4.6.0
|
lxml >= 4.6.0
|
||||||
spacy >= 3.3.0
|
spacy >= 3.3.0
|
||||||
pydantic >= 1.8.0
|
pydantic >= 1.8.0
|
||||||
scikit-learn >= 1.0.0
|
scikit-learn == 1.1.1
|
||||||
matplotlib >= 3.5.0
|
matplotlib >= 3.5.0
|
||||||
numpy >= 1.22.0
|
numpy >= 1.22.0
|
||||||
langcodes[data] >= 3.3.0
|
langcodes[data] >= 3.3.0
|
||||||
segtok >= 1.5.11
|
segtok >= 1.5.11
|
||||||
langdetect >= 1.0.9
|
langdetect >= 1.0.9
|
||||||
|
tinydb >= 4.7.0
|
||||||
|
pandas >= 1.4.0
|
||||||
pyaml >= 21.0.0
|
pyaml >= 21.0.0
|
||||||
|
boto3 >= 1.23.0
|
||||||
|
fastapi >= 0.70.0
|
||||||
|
plotly >= 5.8.0
|
||||||
|
dash >= 2.4.0
|
||||||
|
uvicorn[standard] >= 0.17.0
|
||||||
|
|
||||||
[options.package_data]
|
[options.package_data]
|
||||||
* = *.json, *.yaml, *.yml, *.css
|
* = *.json, *.yaml, *.yml, *.css
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,8 @@
|
||||||
from multiprocessing import Lock
|
from multiprocessing import Lock
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Callable, Dict, Optional
|
from typing import Any, Callable, Dict, List, Optional
|
||||||
|
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
from black import List
|
|
||||||
from tinydb import TinyDB
|
from tinydb import TinyDB
|
||||||
|
|
||||||
from ..views import Filter, SortBy, Trace
|
from ..views import Filter, SortBy, Trace
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,5 @@
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from typing import Any, Dict, Optional
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
from black import List
|
|
||||||
|
|
||||||
from ..views import Filter, SortBy, Trace
|
from ..views import Filter, SortBy, Trace
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,50 +0,0 @@
|
||||||
Metadata-Version: 2.1
|
|
||||||
Name: sus
|
|
||||||
Version: 0.0.3
|
|
||||||
Summary: [S]coutinScience [u]tilitie[s]: reusable utilities for text processing
|
|
||||||
Home-page: https://github.com/ScoutinScience/platform
|
|
||||||
Author: ScoutinScience B.V.
|
|
||||||
Author-email: andras@scoutinscience.com
|
|
||||||
License: UNKNOWN
|
|
||||||
Project-URL: Bug Tracker, https://github.com/ScoutinScience/platform/issues
|
|
||||||
Platform: UNKNOWN
|
|
||||||
Classifier: Programming Language :: Python :: 3
|
|
||||||
Classifier: Operating System :: OS Independent
|
|
||||||
Requires-Python: >=3.8
|
|
||||||
Description-Content-Type: text/markdown
|
|
||||||
|
|
||||||
# **S**coutinScience **U**tilitie**S** for text processing [](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml)
|
|
||||||
|
|
||||||
> amogus
|
|
||||||
|
|
||||||
## Exports
|
|
||||||
|
|
||||||
- [clean](src/sus/clean.py)
|
|
||||||
- [unique](src/sus/unique.py)
|
|
||||||
- [parallel_map](src/sus/parallel_map.py)
|
|
||||||
- [match_names](src/sus/match_names/match_names.py)
|
|
||||||
- [evaluate_ranking](src/sus/evaluate_ranking/evaluate_ranking.py)
|
|
||||||
|
|
||||||
### Requires loading spacy model
|
|
||||||
|
|
||||||
> This is automatic but will require some time.
|
|
||||||
|
|
||||||
> Add this to the Dockerfile for caching the spaCy model:
|
|
||||||
>
|
|
||||||
> ```docker
|
|
||||||
> RUN pip install --no-cache-dir en-core-web-lg@https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.2.0/en_core_web_lg-3.2.0-py3-none-any.whl
|
|
||||||
> ```
|
|
||||||
|
|
||||||
- [spacy model (nlp)](src/sus/nlp.py)
|
|
||||||
- [get_sentences](src/sus/get_sentences.py)
|
|
||||||
- [lemmatize_text](src/sus/lemmatize_text.py)
|
|
||||||
- [lemmatize_token](src/sus/lemmatize_token.py)
|
|
||||||
- [publication TEI](src/sus/publication_tei/publication_tei.py)
|
|
||||||
|
|
||||||
## Development
|
|
||||||
|
|
||||||
- Optional booleans must have a default value of `False`.
|
|
||||||
- No imports in top-level `__init__.py`, in order to not load anything unnecessary automatically
|
|
||||||
- Should only be updated through a PR
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -1,138 +0,0 @@
|
||||||
.gitignore
|
|
||||||
README.md
|
|
||||||
example_secrets.ini
|
|
||||||
open_s3.md
|
|
||||||
pyproject.toml
|
|
||||||
requirements.txt
|
|
||||||
setup.cfg
|
|
||||||
src/__init__.py
|
|
||||||
src/great_ai/__init__.py
|
|
||||||
src/great_ai/great_ai/__init__.py
|
|
||||||
src/great_ai/great_ai/context/__init__.py
|
|
||||||
src/great_ai/great_ai/context/context.py
|
|
||||||
src/great_ai/great_ai/context/get_context.py
|
|
||||||
src/great_ai/great_ai/deploy/__init__.py
|
|
||||||
src/great_ai/great_ai/deploy/create_fastapi_app.py
|
|
||||||
src/great_ai/great_ai/deploy/process_batch.py
|
|
||||||
src/great_ai/great_ai/deploy/process_single.py
|
|
||||||
src/great_ai/great_ai/deploy/serve.py
|
|
||||||
src/great_ai/great_ai/exceptions/__init__.py
|
|
||||||
src/great_ai/great_ai/exceptions/argument_validation_error.py
|
|
||||||
src/great_ai/great_ai/helper/__init__.py
|
|
||||||
src/great_ai/great_ai/helper/get_args.py
|
|
||||||
src/great_ai/great_ai/helper/snake_case_to_text.py
|
|
||||||
src/great_ai/great_ai/helper/text_to_hex_color.py
|
|
||||||
src/great_ai/great_ai/metrics/__init__.py
|
|
||||||
src/great_ai/great_ai/metrics/create_dash_app.py
|
|
||||||
src/great_ai/great_ai/metrics/get_description.py
|
|
||||||
src/great_ai/great_ai/metrics/get_filter_from_datatable.py
|
|
||||||
src/great_ai/great_ai/metrics/get_footer.py
|
|
||||||
src/great_ai/great_ai/metrics/log_argument.py
|
|
||||||
src/great_ai/great_ai/metrics/log_metric.py
|
|
||||||
src/great_ai/great_ai/metrics/assets/github.png
|
|
||||||
src/great_ai/great_ai/metrics/assets/index.css
|
|
||||||
src/great_ai/great_ai/models/__init__.py
|
|
||||||
src/great_ai/great_ai/models/load_model.py
|
|
||||||
src/great_ai/great_ai/models/save_model.py
|
|
||||||
src/great_ai/great_ai/models/use_model.py
|
|
||||||
src/great_ai/great_ai/persistence/__init__.py
|
|
||||||
src/great_ai/great_ai/persistence/mongodb_driver.py
|
|
||||||
src/great_ai/great_ai/persistence/parallel_tinydb_driver.py
|
|
||||||
src/great_ai/great_ai/persistence/persistence_driver.py
|
|
||||||
src/great_ai/great_ai/tracing/__init__.py
|
|
||||||
src/great_ai/great_ai/tracing/tracing_context.py
|
|
||||||
src/great_ai/great_ai/views/__init__.py
|
|
||||||
src/great_ai/great_ai/views/filter.py
|
|
||||||
src/great_ai/great_ai/views/health_check_response.py
|
|
||||||
src/great_ai/great_ai/views/model.py
|
|
||||||
src/great_ai/great_ai/views/operators.py
|
|
||||||
src/great_ai/great_ai/views/query.py
|
|
||||||
src/great_ai/great_ai/views/sort_by.py
|
|
||||||
src/great_ai/great_ai/views/trace.py
|
|
||||||
src/great_ai/open_s3/__init__.py
|
|
||||||
src/great_ai/open_s3/__main__.py
|
|
||||||
src/great_ai/open_s3/large_file.py
|
|
||||||
src/great_ai/open_s3/parse_arguments.py
|
|
||||||
src/great_ai/open_s3/helper/__init__.py
|
|
||||||
src/great_ai/open_s3/helper/bytes_to_megabytes.py
|
|
||||||
src/great_ai/open_s3/helper/human_readable_to_byte.py
|
|
||||||
src/great_ai/open_s3/helper/progress_bar.py
|
|
||||||
src/great_ai/utilities/__init__.py
|
|
||||||
src/great_ai/utilities/clean.py
|
|
||||||
src/great_ai/utilities/get_sentences.py
|
|
||||||
src/great_ai/utilities/lemmatize_text.py
|
|
||||||
src/great_ai/utilities/lemmatize_token.py
|
|
||||||
src/great_ai/utilities/nlp.py
|
|
||||||
src/great_ai/utilities/parallel_map.py
|
|
||||||
src/great_ai/utilities/unique.py
|
|
||||||
src/great_ai/utilities/data/__init__.py
|
|
||||||
src/great_ai/utilities/data/american_spellings.py
|
|
||||||
src/great_ai/utilities/data/punctuations.py
|
|
||||||
src/great_ai/utilities/evaluate_ranking/__init__.py
|
|
||||||
src/great_ai/utilities/evaluate_ranking/draw_f1_iso_lines.py
|
|
||||||
src/great_ai/utilities/evaluate_ranking/evaluate_ranking.py
|
|
||||||
src/great_ai/utilities/external/__init__.py
|
|
||||||
src/great_ai/utilities/external/negspacy/README.md
|
|
||||||
src/great_ai/utilities/external/negspacy/__init__.py
|
|
||||||
src/great_ai/utilities/external/negspacy/negation.py
|
|
||||||
src/great_ai/utilities/external/negspacy/termsets.py
|
|
||||||
src/great_ai/utilities/external/pylatexenc/README.md
|
|
||||||
src/great_ai/utilities/external/pylatexenc/__init__.py
|
|
||||||
src/great_ai/utilities/external/pylatexenc/_util.py
|
|
||||||
src/great_ai/utilities/external/pylatexenc/version.py
|
|
||||||
src/great_ai/utilities/external/pylatexenc/latex2text/__init__.py
|
|
||||||
src/great_ai/utilities/external/pylatexenc/latex2text/__main__.py
|
|
||||||
src/great_ai/utilities/external/pylatexenc/latex2text/_defaultspecs.py
|
|
||||||
src/great_ai/utilities/external/pylatexenc/latexencode/__init__.py
|
|
||||||
src/great_ai/utilities/external/pylatexenc/latexencode/__main__.py
|
|
||||||
src/great_ai/utilities/external/pylatexenc/latexencode/_partial_latex_encoder.py
|
|
||||||
src/great_ai/utilities/external/pylatexenc/latexencode/_uni2latexmap.py
|
|
||||||
src/great_ai/utilities/external/pylatexenc/latexencode/_uni2latexmap_xml.py
|
|
||||||
src/great_ai/utilities/external/pylatexenc/latexencode/_unicode_to_latex_encoder.py
|
|
||||||
src/great_ai/utilities/external/pylatexenc/latexwalker/__init__.py
|
|
||||||
src/great_ai/utilities/external/pylatexenc/latexwalker/_defaultspecs.py
|
|
||||||
src/great_ai/utilities/external/pylatexenc/macrospec/__init__.py
|
|
||||||
src/great_ai/utilities/external/pylatexenc/macrospec/_argparsers.py
|
|
||||||
src/great_ai/utilities/language/__init__.py
|
|
||||||
src/great_ai/utilities/language/english_name_of_language.py
|
|
||||||
src/great_ai/utilities/language/is_english.py
|
|
||||||
src/great_ai/utilities/language/predict_language.py
|
|
||||||
src/great_ai/utilities/logger/__init__.py
|
|
||||||
src/great_ai/utilities/logger/colors.py
|
|
||||||
src/great_ai/utilities/logger/create_logger.py
|
|
||||||
src/great_ai/utilities/logger/custom_formatter.py
|
|
||||||
src/great_ai/utilities/match_names/__init__.py
|
|
||||||
src/great_ai/utilities/match_names/config.py
|
|
||||||
src/great_ai/utilities/match_names/match_names.py
|
|
||||||
src/great_ai/utilities/match_names/name_parts.py
|
|
||||||
src/great_ai/utilities/publication_tei/__init__.py
|
|
||||||
src/great_ai/utilities/publication_tei/publication_tei.py
|
|
||||||
src/great_ai/utilities/publication_tei/models/__init__.py
|
|
||||||
src/great_ai/utilities/publication_tei/models/affiliation.py
|
|
||||||
src/great_ai/utilities/publication_tei/models/author.py
|
|
||||||
src/great_ai/utilities/publication_tei/models/element.py
|
|
||||||
src/great_ai/utilities/publication_tei/models/publication_metadata.py
|
|
||||||
src/great_ai/utilities/publication_tei/models/text.py
|
|
||||||
src/sus.egg-info/PKG-INFO
|
|
||||||
src/sus.egg-info/SOURCES.txt
|
|
||||||
src/sus.egg-info/dependency_links.txt
|
|
||||||
src/sus.egg-info/requires.txt
|
|
||||||
src/sus.egg-info/top_level.txt
|
|
||||||
tests/__init__.py
|
|
||||||
tests/open_s3/__init__.py
|
|
||||||
tests/open_s3/test_human_readable_to_byte.py
|
|
||||||
tests/open_s3/test_large_file.py
|
|
||||||
tests/sus/__init__.py
|
|
||||||
tests/sus/test_clean.py
|
|
||||||
tests/sus/test_evaluate_ranking.py
|
|
||||||
tests/sus/test_get_sentences.py
|
|
||||||
tests/sus/test_language.py
|
|
||||||
tests/sus/test_lemmatize_text.py
|
|
||||||
tests/sus/test_lemmatize_token.py
|
|
||||||
tests/sus/test_match_names.py
|
|
||||||
tests/sus/test_parallel_map.py
|
|
||||||
tests/sus/test_publication_tei.py
|
|
||||||
tests/sus/test_unique.py
|
|
||||||
tests/sus/data/10.1136_bmjspcare-2021-003026.pdf.tei.xml
|
|
||||||
tests/sus/data/bad.tei.xml
|
|
||||||
tests/sus/data/parsed.py
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
|
|
||||||
|
|
@ -1,14 +0,0 @@
|
||||||
click<8.1.0
|
|
||||||
unidecode>=1.3.0
|
|
||||||
multiprocess>=0.70.0.0
|
|
||||||
tqdm>=4.0.0
|
|
||||||
psutil>=5.9.0
|
|
||||||
beautifulsoup4>=4.10.0
|
|
||||||
lxml>=4.6.0
|
|
||||||
spacy>=3.2.0
|
|
||||||
pydantic>=1.8.0
|
|
||||||
scikit-learn>=1.0.0
|
|
||||||
matplotlib>=3.5.0
|
|
||||||
numpy>=1.22.0
|
|
||||||
langcodes[data]>=3.3.0
|
|
||||||
langdetect>=1.0.9
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
great_ai
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue