Train a domain classifier on the semantic scholar dataset¶

Part 3: Create production inference function

The blue boxes show the steps implemented in this notebook.

In Part 2, we trained our AI model. Now, it's time to create General Robust End-to-end Automated Trustworthy deployment from it using the GreatAI Python package.

In [1]:

import re
import numpy as np
from sklearn.pipeline import Pipeline
from great_ai.utilities import clean
from great_ai import (
    MultiLabelClassificationOutput,
    ClassificationOutput,
    GreatAI,
    use_model,
    parameter,
)


@GreatAI.create
@use_model("small-domain-prediction", version="latest")
@parameter("target_confidence", validator=lambda c: 0 <= c <= 100)
def predict_domain(
    text: str, model: Pipeline, target_confidence: int = 50
) -> MultiLabelClassificationOutput:
    """
    Predict the scientific domain of the input text.
    Return labels until their sum likelihood is larger than `target_confidence`.
    """

    preprocessed = re.sub(r"[^a-zA-Z\s]", "", clean(text, convert_to_ascii=True))
    features = model.named_steps["vectorizer"].transform([preprocessed])
    prediction = model.named_steps["classifier"].predict_proba(features)[0]

    best_classes = sorted(enumerate(prediction), key=lambda v: v[1], reverse=True)

    results = MultiLabelClassificationOutput()
    for class_index, probability in best_classes:
        results.labels.append(
            get_label(
                model=model,
                features=features,
                class_index=class_index,
                probability=probability,
            )
        )

        if sum(r.confidence for r in results.labels) >= target_confidence:
            break

    return results


def get_label(
    model: Pipeline, features: np.ndarray, class_index: int, probability: float
) -> ClassificationOutput:
    return ClassificationOutput(
        label=model.named_steps["classifier"].classes_[class_index],
        confidence=round(probability * 100),
        explanation=[
            word
            for _, word in sorted(
                (
                    (weight, word)
                    for weight, word, count in zip(
                        model.named_steps["classifier"].feature_log_prob_[class_index],
                        model.named_steps["vectorizer"].get_feature_names_out(),
                        features.A[0],
                    )
                    if count > 0
                ),
                reverse=True,
            )
        ][:5],
    )

import re import numpy as np from sklearn.pipeline import Pipeline from great_ai.utilities import clean from great_ai import ( MultiLabelClassificationOutput, ClassificationOutput, GreatAI, use_model, parameter, ) @GreatAI.create @use_model("small-domain-prediction", version="latest") @parameter("target_confidence", validator=lambda c: 0 <= c <= 100) def predict_domain( text: str, model: Pipeline, target_confidence: int = 50 ) -> MultiLabelClassificationOutput: """ Predict the scientific domain of the input text. Return labels until their sum likelihood is larger than `target_confidence`. """ preprocessed = re.sub(r"[^a-zA-Z\s]", "", clean(text, convert_to_ascii=True)) features = model.named_steps["vectorizer"].transform([preprocessed]) prediction = model.named_steps["classifier"].predict_proba(features)[0] best_classes = sorted(enumerate(prediction), key=lambda v: v[1], reverse=True) results = MultiLabelClassificationOutput() for class_index, probability in best_classes: results.labels.append( get_label( model=model, features=features, class_index=class_index, probability=probability, ) ) if sum(r.confidence for r in results.labels) >= target_confidence: break return results def get_label( model: Pipeline, features: np.ndarray, class_index: int, probability: float ) -> ClassificationOutput: return ClassificationOutput( label=model.named_steps["classifier"].classes_[class_index], confidence=round(probability * 100), explanation=[ word for _, word in sorted( ( (weight, word) for weight, word, count in zip( model.named_steps["classifier"].feature_log_prob_[class_index], model.named_steps["vectorizer"].get_feature_names_out(), features.A[0], ) if count > 0 ), reverse=True, ) ][:5], )

2022-06-25 14:57:20,629 |  WARNING | Environment variable ENVIRONMENT is not set, defaulting to development mode ‼️
2022-06-25 14:57:20,629 |     INFO | Found credentials file (/data/projects/great_ai_example/mongo.ini), initialising MongodbDriver
2022-06-25 14:57:20,630 |     INFO | Found credentials file (/data/projects/great_ai_example/mongo.ini), initialising LargeFileMongo
2022-06-25 14:57:20,631 |     INFO | Settings: configured ✅
2022-06-25 14:57:20,631 |     INFO | 🔩 tracing_database: MongodbDriver
2022-06-25 14:57:20,631 |     INFO | 🔩 large_file_implementation: LargeFileMongo
2022-06-25 14:57:20,632 |     INFO | 🔩 is_production: False
2022-06-25 14:57:20,633 |     INFO | 🔩 should_log_exception_stack: True
2022-06-25 14:57:20,633 |     INFO | 🔩 prediction_cache_size: 512
2022-06-25 14:57:20,634 |     INFO | 🔩 dashboard_table_size: 50
2022-06-25 14:57:20,634 |  WARNING | You still need to check whether you follow all best practices before trusting your deployment.
2022-06-25 14:57:20,635 |  WARNING | > Find out more at https://se-ml.github.io/practices/
2022-06-25 14:57:20,642 |     INFO | Latest version of small-domain-prediction is 0 (from versions: 0)
2022-06-25 14:57:20,643 |     INFO | File small-domain-prediction-0 found in cache

Check accuracy on the test split¶

In [2]:

if __name__ == "__main__":
    from great_ai import query_ground_truth
    from sklearn import metrics

    data = query_ground_truth("test")

    X = [d.input for d in data]
    y_actual = [d.feedback for d in data]

    y_predicted = [
        d.output.labels[0].label
        for d in predict_domain.process_batch(X, do_not_persist_traces=True)
    ]
    y_actual_aligned = [p if p in a else a[0] for p, a in zip(y_predicted, y_actual)]

    import matplotlib.pyplot as plt

    # Configure matplotlib to have nice, high-resolution charts
    %matplotlib inline

    plt.rcParams["figure.figsize"] = (30, 15)
    plt.rcParams["figure.facecolor"] = "white"
    plt.rcParams["font.size"] = 12
    plt.rcParams["axes.xmargin"] = 0

    print(metrics.classification_report(y_actual_aligned, y_predicted))
    metrics.ConfusionMatrixDisplay.from_predictions(
        y_true=y_actual_aligned,
        y_pred=y_predicted,
        xticks_rotation="vertical",
        normalize="pred",
        values_format=".2f",
    )

if __name__ == "__main__": from great_ai import query_ground_truth from sklearn import metrics data = query_ground_truth("test") X = [d.input for d in data] y_actual = [d.feedback for d in data] y_predicted = [ d.output.labels[0].label for d in predict_domain.process_batch(X, do_not_persist_traces=True) ] y_actual_aligned = [p if p in a else a[0] for p, a in zip(y_predicted, y_actual)] import matplotlib.pyplot as plt # Configure matplotlib to have nice, high-resolution charts %matplotlib inline plt.rcParams["figure.figsize"] = (30, 15) plt.rcParams["figure.facecolor"] = "white" plt.rcParams["font.size"] = 12 plt.rcParams["axes.xmargin"] = 0 print(metrics.classification_report(y_actual_aligned, y_predicted)) metrics.ConfusionMatrixDisplay.from_predictions( y_true=y_actual_aligned, y_pred=y_predicted, xticks_rotation="vertical", normalize="pred", values_format=".2f", )

2022-06-25 14:57:21,004 |     INFO | Starting parallel map (concurrency: 12, chunk size: 26)
100%|██████████| 3054/3054 [00:31<00:00, 96.38it/s] 

                       precision    recall  f1-score   support

                  Art       0.35      0.27      0.31        33
              Biology       0.77      0.84      0.80       299
             Business       0.44      0.61      0.51        82
            Chemistry       0.79      0.67      0.72       289
     Computer Science       0.74      0.79      0.77       341
            Economics       0.65      0.56      0.60        61
          Engineering       0.53      0.52      0.53       202
Environmental Science       0.51      0.49      0.50        61
            Geography       0.54      0.26      0.35        58
              Geology       0.69      0.56      0.62        48
              History       0.36      0.17      0.23        29
    Materials Science       0.68      0.83      0.74       248
          Mathematics       0.77      0.57      0.66       122
             Medicine       0.97      0.79      0.87       715
           Philosophy       0.50      0.05      0.10        19
              Physics       0.72      0.75      0.74       162
    Political Science       0.39      0.49      0.43        57
           Psychology       0.49      0.81      0.61       141
            Sociology       0.32      0.53      0.40        87

             accuracy                           0.70      3054
            macro avg       0.59      0.56      0.55      3054
         weighted avg       0.72      0.70      0.70      3054

---

Last update: July 11, 2022