Experiment with framework

Signed-off-by: András Schmelczer <andras@schmelczer.dev>

example/helper/__init__.py

@@ -1 +0,0 @@
-from .preprocess import preprocess

example/main.py

@@ -0,0 +1,16 @@
+import json
+
+from open_s3 import LargeFile
+from predict import predict_domain
+
+from good_ai import process_batch
+
+if __name__ == "__main__":
+    with open("data/s2-corpus-1583.json") as f:
+        raw = json.load(f)
+
+    LargeFile.configure_credentials_from_file("s3.ini")
+
+    data = {f'{r["title"]} {r["abstract"]}': r["domain"] for r in raw[:5]}
+
+    print(process_batch(predict_domain, ["We have found a new type of chemical."]))

example/predict_domain.py

@@ -1,27 +1,30 @@
 import re
 from typing import Dict, Iterable, List
 
-from helper import preprocess
+from config import model_key
+from preprocess import preprocess
 from models import DomainPrediction
 from sklearn.pipeline import Pipeline
 
-# from sus.use_model import use_model
+from good_ai import use_model
+from sus.clean import clean
 
-
-# @use_model(model_key, version="latest")
-def predict(
+@use_model(model_key, version="latest")
+def predict_domain(
     text: str, model: Pipeline, cut_off_probability: float = 0.2
 ) -> List[DomainPrediction]:
     assert 0 <= cut_off_probability <= 1
+    cleaned = clean(text, convert_to_ascii=True)
+    text = re.sub(r"[^a-zA-Z0-9]", " ", cleaned)
 
     feature_names = model.named_steps["vectorizer"].get_feature_names_out()
 
     token_mapping = {
         preprocess(original): original
-        for original in re.sub(r"[^a-zA-Z0-9]", " ", text).split(" ")
+        for original in text.split(" ")
     }
 
-    features = model.named_steps["vectorizer"].transform([text])
+    features = model.named_steps["vectorizer"].transform([" ".join(token_mapping.keys())])
     prediction = model.named_steps["classifier"].predict_proba(features)[0]
     best_classes = sorted(enumerate(prediction), key=lambda v: v[1], reverse=True)
 

example/preprocess.py

@@ -5,6 +5,5 @@ from sus.lemmatize_text import lemmatize_text
 
 
 def preprocess(text: str) -> str:
-    cleaned = clean(text, convert_to_ascii=True)
-    lemmas = [re.sub(r"\d+", "NUM", lemma) for lemma in lemmatize_text(cleaned)]
+    lemmas = [re.sub(r"\d+", "NUM", lemma) for lemma in lemmatize_text(text)]
     return " ".join(lemmas)

example/secrets-backblaze.ini

@@ -1,6 +0,0 @@
-[DEFAULT]
-aws_region_name = us-west-4
-aws_access_key_id = 00411a2454be9f90000000001
-aws_secret_access_key = K004YOSYHePVUHymVzm/vb+AjGahXl8
-large_files_bucket_name = sa-large-files
-endpoint_url = https://s3.us-west-004.backblazeb2.com

example/secrets.ini

@@ -1,5 +0,0 @@
-[DEFAULT]
-aws_region_name = eu-west-2
-aws_access_key_id = AKIAR7XRQBSIKKSKQA6U
-aws_secret_access_key = WPr6mqbtJjmcmibdz12nK6XwQQHsWiXvep7NUexJ
-large_files_bucket_name = sis-large-files