Split examples

.vscode/tasks.json

@@ -4,9 +4,9 @@
     {
       "label": "Format and lint Python",
       "type": "shell",
-      "command": "source .env/bin/activate && scripts/format-python.sh great_ai && scripts/format-python.sh examples",
+      "command": "source .env/bin/activate && scripts/format-python.sh great_ai && scripts/format-python.sh examples/simple && scripts/format-python.sh examples/complex",
       "windows": {
-        "command": ".env/bin/activate.bat; scripts/format-python.sh great_ai; scripts\\format-python.sh examples"
+        "command": ".env\\bin\\activate.bat; scripts\\format-python.sh great_ai; scripts\\format-python.sh examples\\simple; scripts\\format-python.sh examples\\complex"
       },
       "group": "test",
       "presentation": {
@@ -22,7 +22,7 @@
       "type": "shell",
       "command": "source .env/bin/activate && python3 -m pytest great_ai",
       "windows": {
-        "command": ".env/bin/activate.bat; python3 -m pytest great_ai"
+        "command": ".env\\bin\\activate.bat; python3 -m pytest great_ai"
       },
       "group": "test",
       "presentation": {

examples/complex/Dockerfile

@@ -0,0 +1,25 @@
+FROM python:3.8.12-slim-bullseye
+
+ENV ENVIRONMENT production
+
+WORKDIR /app
+
+RUN apt-get update &&\
+    apt-get install curl -y &&\
+    rm -rf /var/lib/apt/lists/*
+
+RUN pip install --no-cache-dir en-core-web-sm@https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl
+COPY sus sus
+RUN pip install --no-cache-dir --use-feature=in-tree-build ./sus
+
+COPY requirements.txt ./
+RUN pip install --no-cache-dir --requirement requirements.txt
+
+COPY . .
+RUN rm -rf sus secrets
+
+EXPOSE 5000
+
+HEALTHCHECK --interval=60s --timeout=60s --start-period=90s --retries=5 CMD [ "curl", "--fail", "http://localhost:5000/health" ]
+
+CMD ["uvicorn", "main:app", "--proxy-headers", "--backlog", "8196", "--timeout-keep-alive", "900", "--host", "0.0.0.0", "--port", "5000"]

examples/complex/Dockerfile.development

@@ -0,0 +1,22 @@
+FROM python:3.8.12-slim-bullseye
+
+ENV ENVIRONMENT development
+
+VOLUME /app
+WORKDIR /app
+
+RUN apt-get update &&\
+    apt-get install build-essential -y &&\
+    rm -rf /var/lib/apt/lists/*
+
+RUN python3 -m pip install --no-cache-dir --upgrade pip &&\
+    python3 -m pip install --no-cache-dir en-core-web-sm@https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl
+
+COPY requirements.txt ./
+RUN python3 -m pip install --no-cache-dir -r requirements.txt
+
+EXPOSE 5000
+
+VOLUME /dependencies
+
+CMD ["/bin/bash", "-c", "python3 -m pip install -e /dependencies/sus && python3 -m uvicorn --reload --host 0.0.0.0 --port 5000 main:app"]

examples/complex/main_service.py

@@ -1,8 +1,6 @@
-#!/usr/bin/env python3
-
 from great_ai import configure, create_service
 
-# configure(development_mode_override=True)
+configure(development_mode_override=True)
 
 from predict_domain import predict_domain
 

examples/simple/predict_domain.py

@@ -0,0 +1,86 @@
+from great_ai import configure, create_service, log_argument, log_metric, use_model
+
+configure(development_mode_override=True)
+
+import re
+from typing import Dict, Iterable, List
+
+from preprocess import preprocess
+from pydantic import BaseModel
+from sklearn.pipeline import Pipeline
+
+from great_ai.utilities.clean import clean
+
+
+class DomainPrediction(BaseModel):
+    domain: str
+    probability: float
+    explanation: List[str]
+
+
+@create_service
+@use_model("small-domain-prediction-v2", version="latest")
+@log_argument("text", validator=lambda t: len(t) > 0)
+def predict_domain(
+    text: str, model: Pipeline, cut_off_probability: float = 0.2
+) -> List[DomainPrediction]:
+    assert 0 <= cut_off_probability <= 1
+
+    """
+    Predict the scientific domain of the input text.
+    Return labels until their sum likelihood is larger than cut_off_probability.
+    """
+    log_metric("text_length", len(text))
+
+    cleaned = clean(text, convert_to_ascii=True)
+    text = re.sub(r"[^a-zA-Z0-9]", " ", cleaned)
+
+    feature_names = model.named_steps["vectorizer"].get_feature_names_out()
+
+    token_mapping = {preprocess(original): original for original in text.split(" ")}
+
+    features = model.named_steps["vectorizer"].transform(
+        [" ".join(token_mapping.keys())]
+    )
+    prediction = model.named_steps["classifier"].predict_proba(features)[0]
+    best_classes = sorted(enumerate(prediction), key=lambda v: v[1], reverse=True)
+
+    results: List[DomainPrediction] = []
+    for class_index, probability in best_classes:
+        weights = model.named_steps["classifier"].feature_log_prob_[class_index]
+        domain = model.named_steps["classifier"].classes_[class_index]
+
+        results.append(
+            DomainPrediction(
+                domain=domain,
+                probability=round(probability * 100),
+                explanation=_get_explanation(
+                    feature_names=feature_names,
+                    features=features.A[0],
+                    weights=weights,
+                    token_mapping=token_mapping,
+                ),
+            )
+        )
+
+        if sum(r.probability for r in results) >= cut_off_probability * 100:
+            break
+
+    return results
+
+
+def _get_explanation(
+    feature_names: Iterable[str],
+    features: Iterable[float],
+    weights: Iterable[float],
+    token_mapping: Dict[str, str],
+) -> List[str]:
+    influential = [
+        (weight, name)
+        for weight, value, name in zip(weights, features, feature_names)
+        if value
+    ]
+
+    most_influential = sorted(influential, reverse=True)[:5]
+
+    return [token_mapping[name] for _, name in most_influential]

examples/simple/preprocess.py

@@ -0,0 +1,8 @@
+import re
+
+from great_ai.utilities.lemmatize_text import lemmatize_text
+
+
+def preprocess(text: str) -> str:
+    lemmas = [re.sub(r"\d[\d.,]*", "NUM", lemma) for lemma in lemmatize_text(text)]
+    return " ".join(lemmas)

examples/train.ipynb

@@ -154,7 +154,12 @@
    "outputs": [],
    "source": [
     "classifier = GridSearchCV(\n",
-        "    Pipeline(steps=[(\"vectorizer\", TfidfVectorizer(token_pattern=r\"[^ ]+\")), (\"classifier\", MultinomialNB())]),\n",
+    "    Pipeline(\n",
+    "        steps=[\n",
+    "            (\"vectorizer\", TfidfVectorizer(token_pattern=r\"[^ ]+\")),\n",
+    "            (\"classifier\", MultinomialNB()),\n",
+    "        ]\n",
+    "    ),\n",
     "    {\n",
     "        \"vectorizer__max_df\": [0.05, 0.1, 0.3],\n",
     "        \"vectorizer__min_df\": [5, 10, 30],\n",
@@ -181,7 +186,12 @@
    "source": [
     "classifier = Pipeline(\n",
     "    steps=[\n",
-        "        (\"vectorizer\", TfidfVectorizer(min_df=10, max_df=0.05, sublinear_tf=True, token_pattern=r\"[^ ]+\")),\n",
+    "        (\n",
+    "            \"vectorizer\",\n",
+    "            TfidfVectorizer(\n",
+    "                min_df=10, max_df=0.05, sublinear_tf=True, token_pattern=r\"[^ ]+\"\n",
+    "            ),\n",
+    "        ),\n",
     "        (\"classifier\", MultinomialNB(alpha=0.5, fit_prior=False)),\n",
     "    ]\n",
     ")\n",