Add files

Signed-off-by: András Schmelczer <andras@schmelczer.dev>

example/.gitignore

@@ -0,0 +1,2 @@
+data
+.ipynb_checkpoints

example/config.py

@@ -0,0 +1 @@
+model_key = 

example/helper/__init__.py

@@ -0,0 +1 @@
+from .preprocess import preprocess

example/helper/preprocess.py

@@ -0,0 +1,12 @@
+from sus.clean import clean
+from sus.lemmatize_text import lemmatize_text
+import re
+
+
+def preprocess(text: str) -> str:
+    cleaned = clean(text, convert_to_ascii=True)
+    lemmas = [
+        re.sub(r'\d+', 'NUM', lemma)
+        for lemma in lemmatize_text(cleaned)
+    ]
+    return " ".join(lemmas)

example/models/__init__.py

@@ -0,0 +1 @@
+from .domain_prediction import DomainPrediction

example/models/domain_prediction.py

@@ -0,0 +1,8 @@
+from pydantic import BaseModel
+from typing import List
+
+
+class DomainPrediction(BaseModel):
+    domain: str
+    probability: float
+    explanation: List[str]

example/predict.py

@@ -0,0 +1,56 @@
+
+from models import DomainPrediction
+from typing import Iterable, List, Dict
+from sklearn.pipeline import Pipeline 
+from helper import preprocess
+import re
+# from sus.use_model import use_model
+from config import model_key
+
+
+# @use_model(model_key, version="latest")
+def predict(text: str, model: Pipeline, cut_off_probability: float=0.2) -> List[DomainPrediction]:
+    assert 0 <= cut_off_probability <= 1
+
+    feature_names = model.named_steps['vectorizer'].get_feature_names_out()
+
+    token_mapping = {
+        preprocess(original): original
+        for original in re.sub(r'[^a-zA-Z0-9]', ' ', text).split(' ')
+    }
+    
+    features = model.named_steps['vectorizer'].transform([text])
+    prediction = model.named_steps['classifier'].predict_proba(features)[0]
+    best_classes = sorted(enumerate(prediction), key=lambda v: v[1], reverse=True)
+
+    results: List[DomainPrediction] = []
+    for class_index, probability in best_classes:
+        weights = model.named_steps['classifier'].feature_log_prob_[class_index]
+       
+        results.append(DomainPrediction(
+            domain=model.named_steps['classifier'].classes_[class_index],
+            probability=round(probability * 100),
+            explanation=_get_explanation(
+                feature_names=feature_names,
+                features=features.A[0],
+                weights=weights,
+                token_mapping=token_mapping
+            )
+        ))
+
+        if sum(r.probability for r in results) >= cut_off_probability:
+            break
+    
+    return results
+
+
+def _get_explanation(feature_names: Iterable[str], features: Iterable[float], weights: Iterable[float], token_mapping: Dict[str, str]) -> List[str]:
+    influential = [
+        (value * weight, name)
+        for name, value, weight in zip(feature_names, features, weights)
+        if value
+    ]
+    
+    most_influential = sorted(influential, reverse=True)[:5]
+
+    return [token_mapping[v[1]] for v in most_influential]

example/secrets-backblaze.ini

@@ -0,0 +1,6 @@
+[DEFAULT]
+aws_region_name = us-west-4
+aws_access_key_id = 00411a2454be9f90000000001
+aws_secret_access_key = K004YOSYHePVUHymVzm/vb+AjGahXl8
+large_files_bucket_name = sa-large-files
+endpoint_url = https://s3.us-west-004.backblazeb2.com

example/secrets.ini

@@ -0,0 +1,5 @@
+[DEFAULT]
+aws_region_name = eu-west-2
+aws_access_key_id = AKIAR7XRQBSIKKSKQA6U
+aws_secret_access_key = WPr6mqbtJjmcmibdz12nK6XwQQHsWiXvep7NUexJ
+large_files_bucket_name = sis-large-files