Minor improvements

Dockerfile

@@ -5,22 +5,31 @@ LABEL org.opencontainers.image.vendor="ScoutinScience B.V."
 LABEL org.opencontainers.image.authors="andras@schmelczer.dev"
 LABEL org.opencontainers.image.source="https://github.com/ScoutinScience/great_ai"
 
-ENV ENVIRONMENT production
+ENV ENVIRONMENT=production
 EXPOSE 6060
 
+# curl is needed for the healthcheck
 RUN DEBIAN_FRONTEND=noninteractive apt update &&\
     apt install curl -y &&\
     rm -rf /var/lib/apt/lists/*
 
+WORKDIR /dependencies
 COPY . great_ai
-RUN pip3 install --no-cache-dir ./great_ai &&\
+RUN python3 -m pip --no-cache-dir install --upgrade pip &&\
+    pip install --no-cache-dir ./great_ai &&\
     rm -rf great_ai
 
+# great_ai.utilities.nlp depends on this
 RUN pip3 install --no-cache-dir en-core-web-sm@https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl
 
-HEALTHCHECK --interval=60s --timeout=60s --start-period=90s --retries=5 CMD [ "curl", "--fail", "http://localhost:6060/health" ]
+HEALTHCHECK \
+    --interval=60s \
+    --timeout=60s  \
+    --start-period=90s \
+    --retries=5 \
+    CMD [ "curl", "--fail", "http://localhost:6060/health" ]
 
 WORKDIR /app
 VOLUME /app
 
-ENTRYPOINT ["/bin/sh", "python3", "-m", "great_ai"]
+ENTRYPOINT ["/usr/local/bin/python3", "-m", "great_ai"]

README.md

@@ -1,34 +1,3 @@
-# **S**coutinScience **U**tilitie**S** for text processing [![Lint and test ScoutinScience utilities](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml/badge.svg)](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml)
+# GreatAI
 
-> amogus
+[![Quality Gate Status](https://sonar.scoutinscience.com/api/project_badges/measure?project=great-ai&metric=alert_status)](https://sonar.scoutinscience.com/dashboard?id=great-ai)
-
-## Exports
-
-- [clean](src/sus/clean.py)
-- [unique](src/sus/unique.py)
-- [parallel_map](src/sus/parallel_map.py)
-- [match_names](src/sus/match_names/match_names.py)
-- [evaluate_ranking](src/sus/evaluate_ranking/evaluate_ranking.py)
-- [get_sentences](src/sus/get_sentences.py)
-
-### Requires loading spacy model
-
-> This is automatic but will require some time.
-
-> Add this to the Dockerfile for caching the spaCy model:
->
-> ```docker
-> RUN pip install --no-cache-dir en-core-web-sm@https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl
-> ```
-
-- [publication TEI](src/sus/publication_tei/publication_tei.py)
-- [lemmatize_text](src/sus/lemmatize_text.py)
-- [lemmatize_token](src/sus/lemmatize_token.py)
-- [spacy model (nlp)](src/sus/nlp.py)
-- [filter_sentences](src/sus/matcher/filter_sentences.py)
-
-## Development
-
-- Optional booleans must have a default value of `False`.
-- No imports in top-level `__init__.py`, in order to not load anything unnecessary automatically
-- Should only be updated through a PR

docs/README.md

@@ -0,0 +1,34 @@
+# **S**coutinScience **U**tilitie**S** for text processing [![Lint and test ScoutinScience utilities](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml/badge.svg)](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml)
+
+> amogus
+
+## Exports
+
+- [clean](src/sus/clean.py)
+- [unique](src/sus/unique.py)
+- [parallel_map](src/sus/parallel_map.py)
+- [match_names](src/sus/match_names/match_names.py)
+- [evaluate_ranking](src/sus/evaluate_ranking/evaluate_ranking.py)
+- [get_sentences](src/sus/get_sentences.py)
+
+### Requires loading spacy model
+
+> This is automatic but will require some time.
+
+> Add this to the Dockerfile for caching the spaCy model:
+>
+> ```docker
+> RUN pip install --no-cache-dir en-core-web-sm@https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl
+> ```
+
+- [publication TEI](src/sus/publication_tei/publication_tei.py)
+- [lemmatize_text](src/sus/lemmatize_text.py)
+- [lemmatize_token](src/sus/lemmatize_token.py)
+- [spacy model (nlp)](src/sus/nlp.py)
+- [filter_sentences](src/sus/matcher/filter_sentences.py)
+
+## Development
+
+- Optional booleans must have a default value of `False`.
+- No imports in top-level `__init__.py`, in order to not load anything unnecessary automatically
+- Should only be updated through a PR

setup.cfg

@@ -23,11 +23,9 @@ install_requires =
     unidecode >= 1.3.0
     multiprocess >= 0.70.0.0
     tqdm >= 4.0.0
-    psutil >= 5.9.0
     beautifulsoup4 >= 4.10.0
     lxml >= 4.6.0
     spacy >= 3.3.0
-    pydantic >= 1.8.0
     scikit-learn == 1.1.1
     matplotlib >= 3.5.0
     numpy >= 1.22.0
@@ -42,6 +40,7 @@ install_requires =
     plotly >= 5.8.0
     dash >= 2.4.0
     nbconvert >= 6.5.0
+    ipython >= 8.0.0
     uvicorn[standard] >= 0.18.0
     watchdog >= 2.1.0
     pymongo >= 4.0.0