Skip to content

# Train a domain classifier on the semantic scholar dataset¶

> Part 3: Create production inference function

> The blue boxes show the steps implemented in this notebook.

In Part 2, we trained our AI model. Now, it's time to create General Robust End-to-end Automated Trustworthy deployment from it using the `GreatAI` Python package.

In [1]:
Copied!

    import re
    import numpy as np
    from sklearn.pipeline import Pipeline
    from great_ai.utilities import clean
    from great_ai import (
        MultiLabelClassificationOutput,
        ClassificationOutput,
        GreatAI,
        use_model,
        parameter,
    )

    @GreatAI.create
    @use_model("small-domain-prediction", version="latest")
    @parameter("target_confidence", validator=lambda c: 0 <= c <= 100)
    def predict_domain(
        text: str, model: Pipeline, target_confidence: int = 50
    ) -> MultiLabelClassificationOutput:
        """
        Predict the scientific domain of the input text.
        Return labels until their sum likelihood is larger than `target_confidence`.
        """

        preprocessed = re.sub(r"[^a-zA-Z\s]", "", clean(text, convert_to_ascii=True))
        features = model.named_steps["vectorizer"].transform([preprocessed])
        prediction = model.named_steps["classifier"].predict_proba(features)[0]

        best_classes = sorted(enumerate(prediction), key=lambda v: v[1], reverse=True)

        results = MultiLabelClassificationOutput()
        for class_index, probability in best_classes:
            results.labels.append(
                get_label(
                    model=model,
                    features=features,
                    class_index=class_index,
                    probability=probability,
                )
            )

            if sum(r.confidence for r in results.labels) >= target_confidence:
                break

        return results

    def get_label(
        model: Pipeline, features: np.ndarray, class_index: int, probability: float
    ) -> ClassificationOutput:
        return ClassificationOutput(
            label=model.named_steps["classifier"].classes_[class_index],
            confidence=round(probability * 100),
            explanation=[
                word
                for _, word in sorted(
                    (
                        (weight, word)
                        for weight, word, count in zip(
                            model.named_steps["classifier"].feature_log_prob_[class_index],
                            model.named_steps["vectorizer"].get_feature_names_out(),
                            features.A[0],
                        )
                        if count > 0
                    ),
                    reverse=True,
                )
            ][:5],
        )

import re import numpy as np from sklearn.pipeline import Pipeline from great_ai.utilities import clean from great_ai import ( MultiLabelClassificationOutput, ClassificationOutput, GreatAI, use_model, parameter, ) @GreatAI.create @use_model("small-domain-prediction", version="latest") @parameter("target_confidence", validator=lambda c: 0 <= c <= 100) def predict_domain( text: str, model: Pipeline, target_confidence: int = 50 ) -> MultiLabelClassificationOutput: """ Predict the scientific domain of the input text. Return labels until their sum likelihood is larger than `target_confidence`. """ preprocessed = re.sub(r"[^a-zA-Z\s]", "", clean(text, convert_to_ascii=True)) features = model.named_steps["vectorizer"].transform([preprocessed]) prediction = model.named_steps["classifier"].predict_proba(features)[0] best_classes = sorted(enumerate(prediction), key=lambda v: v[1], reverse=True) results = MultiLabelClassificationOutput() for class_index, probability in best_classes: results.labels.append( get_label( model=model, features=features, class_index=class_index, probability=probability, ) ) if sum(r.confidence for r in results.labels) >= target_confidence: break return results def get_label( model: Pipeline, features: np.ndarray, class_index: int, probability: float ) -> ClassificationOutput: return ClassificationOutput( label=model.named_steps["classifier"].classes_[class_index], confidence=round(probability * 100), explanation=[ word for _, word in sorted( ( (weight, word) for weight, word, count in zip( model.named_steps["classifier"].feature_log_prob_[class_index], model.named_steps["vectorizer"].get_feature_names_out(), features.A[0], ) if count > 0 ), reverse=True, ) ][:5], )

    2022-06-25 14:57:20,629 |  WARNING | Environment variable ENVIRONMENT is not set, defaulting to development mode ‼️
    2022-06-25 14:57:20,629 |     INFO | Found credentials file (/data/projects/great_ai_example/mongo.ini), initialising MongodbDriver
    2022-06-25 14:57:20,630 |     INFO | Found credentials file (/data/projects/great_ai_example/mongo.ini), initialising LargeFileMongo
    2022-06-25 14:57:20,631 |     INFO | Settings: configured ✅
    2022-06-25 14:57:20,631 |     INFO | 🔩 tracing_database: MongodbDriver
    2022-06-25 14:57:20,631 |     INFO | 🔩 large_file_implementation: LargeFileMongo
    2022-06-25 14:57:20,632 |     INFO | 🔩 is_production: False
    2022-06-25 14:57:20,633 |     INFO | 🔩 should_log_exception_stack: True
    2022-06-25 14:57:20,633 |     INFO | 🔩 prediction_cache_size: 512
    2022-06-25 14:57:20,634 |     INFO | 🔩 dashboard_table_size: 50
    2022-06-25 14:57:20,634 |  WARNING | You still need to check whether you follow all best practices before trusting your deployment.
    2022-06-25 14:57:20,635 |  WARNING | > Find out more at https://se-ml.github.io/practices/
    2022-06-25 14:57:20,642 |     INFO | Latest version of small-domain-prediction is 0 (from versions: 0)
    2022-06-25 14:57:20,643 |     INFO | File small-domain-prediction-0 found in cache

## Check accuracy on the test split¶

In [2]:
Copied!

    if __name__ == "__main__":
        from great_ai import query_ground_truth
        from sklearn import metrics

        data = query_ground_truth("test")

        X = [d.input for d in data]
        y_actual = [d.feedback for d in data]

        y_predicted = [d.output.labels[0].label for d in predict_domain.process_batch(X, do_not_persist_traces=True)]
        y_actual_aligned = [p if p in a else a[0] for p, a in zip(y_predicted, y_actual)]

        import matplotlib.pyplot as plt

        # Configure matplotlib to have nice, high-resolution charts
        %matplotlib inline

        plt.rcParams["figure.figsize"] = (30, 15)
        plt.rcParams["figure.facecolor"] = "white"
        plt.rcParams["font.size"] = 12
        plt.rcParams["axes.xmargin"] = 0

        print(metrics.classification_report(y_actual_aligned, y_predicted))
        metrics.ConfusionMatrixDisplay.from_predictions(
            y_true=y_actual_aligned,
            y_pred=y_predicted,
            xticks_rotation="vertical",
            normalize="pred",
            values_format=".2f",
        )

if __name__ == "__main__": from great_ai import query_ground_truth from sklearn import metrics data = query_ground_truth("test") X = [d.input for d in data] y_actual = [d.feedback for d in data] y_predicted = [d.output.labels[0].label for d in predict_domain.process_batch(X, do_not_persist_traces=True)] y_actual_aligned = [p if p in a else a[0] for p, a in zip(y_predicted, y_actual)] import matplotlib.pyplot as plt # Configure matplotlib to have nice, high-resolution charts %matplotlib inline plt.rcParams["figure.figsize"] = (30, 15) plt.rcParams["figure.facecolor"] = "white" plt.rcParams["font.size"] = 12 plt.rcParams["axes.xmargin"] = 0 print(metrics.classification_report(y_actual_aligned, y_predicted)) metrics.ConfusionMatrixDisplay.from_predictions( y_true=y_actual_aligned, y_pred=y_predicted, xticks_rotation="vertical", normalize="pred", values_format=".2f", )

    2022-06-25 14:57:21,004 |     INFO | Starting parallel map (concurrency: 12, chunk size: 26)
    100%|██████████| 3054/3054 [00:31<00:00, 96.38it/s]

                           precision    recall  f1-score   support

                      Art       0.35      0.27      0.31        33
                  Biology       0.77      0.84      0.80       299
                 Business       0.44      0.61      0.51        82
                Chemistry       0.79      0.67      0.72       289
         Computer Science       0.74      0.79      0.77       341
                Economics       0.65      0.56      0.60        61
              Engineering       0.53      0.52      0.53       202
    Environmental Science       0.51      0.49      0.50        61
                Geography       0.54      0.26      0.35        58
                  Geology       0.69      0.56      0.62        48
                  History       0.36      0.17      0.23        29
        Materials Science       0.68      0.83      0.74       248
              Mathematics       0.77      0.57      0.66       122
                 Medicine       0.97      0.79      0.87       715
               Philosophy       0.50      0.05      0.10        19
                  Physics       0.72      0.75      0.74       162
        Political Science       0.39      0.49      0.43        57
               Psychology       0.49      0.81      0.61       141
                Sociology       0.32      0.53      0.40        87

                 accuracy                           0.70      3054
                macro avg       0.59      0.56      0.55      3054
             weighted avg       0.72      0.70      0.70      3054

Last update: July 8, 2022