Add auto-reload, evaluation endpoint, show docs

examples/README.md

@@ -0,0 +1,12 @@
+# Train Domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)
+
+## Upload the dataset (or a part of it) to shared infrastructure
+
+```sh
+mkdir ss-data && cd ss-data
+wget https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/manifest.txt
+wget -B https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/ -i manifest.txt
+cd -
+python3 -m great_ai.open_s3 --secrets s3.ini --push ss-data
+rm -rf ss-data
+```

examples/main_batch.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+
+import json
+from random import shuffle
+
+from devtools import debug
+from predict_domain import predict_domain
+
+from great_ai import process_batch
+
+if __name__ == "__main__":
+    with open(".cache/data-1/s2-corpus-323.json") as f:
+        raw = json.load(f)
+
+    shuffle(raw)
+    data = {f'{r["title"]} {r["abstract"]}': r["domain"] for r in raw[:10]}
+
+    results = process_batch(predict_domain, data.keys())
+
+    for predicted, actual in zip(results, data.values()):
+        print(", ".join(actual))
+        debug(predicted)
+        print()

examples/main_service.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python3
+
+from great_ai import configure, create_service
+
+configure(development_mode_override=True)
+
+from predict_domain import predict_domain
+
+app = create_service(predict_domain)

examples/predict_domain.py

@@ -0,0 +1,79 @@
+import re
+from typing import Dict, Iterable, List
+
+from preprocess import preprocess
+from pydantic import BaseModel
+from sklearn.pipeline import Pipeline
+
+from great_ai import log_argument, log_metric, use_model
+from great_ai.utilities.clean import clean
+
+
+class DomainPrediction(BaseModel):
+    domain: str
+    probability: float
+    explanation: List[str]
+
+
+@use_model("small-domain-prediction-v2", version="latest")
+@log_argument("text", validator=lambda t: len(t) > 0)
+def predict_domain(
+    text: str, model: Pipeline, cut_off_probability: float = 0.2
+) -> List[DomainPrediction]:
+    """
+    Predict the scientific domain of the input text.
+    Return labels until their sum likelihood is larger than cut_off_probability.
+    """
+    log_metric("text_length", len(text))
+
+    cleaned = clean(text, convert_to_ascii=True)
+    text = re.sub(r"[^a-zA-Z0-9]", " ", cleaned)
+
+    feature_names = model.named_steps["vectorizer"].get_feature_names_out()
+
+    token_mapping = {preprocess(original): original for original in text.split(" ")}
+
+    features = model.named_steps["vectorizer"].transform(
+        [" ".join(token_mapping.keys())]
+    )
+    prediction = model.named_steps["classifier"].predict_proba(features)[0]
+    best_classes = sorted(enumerate(prediction), key=lambda v: v[1], reverse=True)
+
+    results: List[DomainPrediction] = []
+    for class_index, probability in best_classes:
+        weights = model.named_steps["classifier"].feature_log_prob_[class_index]
+
+        results.append(
+            DomainPrediction(
+                domain=model.named_steps["classifier"].classes_[class_index],
+                probability=round(probability * 100),
+                explanation=_get_explanation(
+                    feature_names=feature_names,
+                    features=features.A[0],
+                    weights=weights,
+                    token_mapping=token_mapping,
+                ),
+            )
+        )
+
+        if sum(r.probability for r in results) >= cut_off_probability * 100:
+            break
+
+    return results
+
+
+def _get_explanation(
+    feature_names: Iterable[str],
+    features: Iterable[float],
+    weights: Iterable[float],
+    token_mapping: Dict[str, str],
+) -> List[str]:
+    influential = [
+        (value * weight, name)
+        for name, value, weight in zip(feature_names, features, weights)
+        if value
+    ]
+
+    most_influential = sorted(influential, reverse=True)[:5]
+
+    return [token_mapping[v[1]] for v in most_influential]

examples/preprocess.py

@@ -0,0 +1,8 @@
+import re
+
+from great_ai.utilities.lemmatize_text import lemmatize_text
+
+
+def preprocess(text: str) -> str:
+    lemmas = [re.sub(r"\d[\d.,]*", "NUM", lemma) for lemma in lemmatize_text(text)]
+    return " ".join(lemmas)