Skip to content

# Train a field of study (domain) classification of sentences on the MAG dataset¶

SciBERT achieves an F1-score of 0.6571 on this dataset. This notebook shows that better results can be achieved without even using transformers.

In [1]:
Copied!

    import json
    from typing import Tuple
    from great_ai.utilities import clean, parallel_map
    from tqdm.cli import tqdm

    def preprocess(line: str) -> Tuple[str, str]:
        data_point = json.loads(line)

        return (clean(data_point["text"], convert_to_ascii=True), data_point["label"])

    with open("mag/train.txt", encoding="utf-8") as f:
        training_data = list(tqdm(parallel_map(preprocess, f.readlines())))

    X_train = [d[0] for d in training_data]
    y_train = [d[1] for d in training_data]

    with open("mag/test.txt", encoding="utf-8") as f:
        test_data = list(tqdm(parallel_map(preprocess, f.readlines())))

    X_test = [d[0] for d in test_data]
    y_test = [d[1] for d in test_data]

import json from typing import Tuple from great_ai.utilities import clean, parallel_map from tqdm.cli import tqdm def preprocess(line: str) -> Tuple[str, str]: data_point = json.loads(line) return (clean(data_point["text"], convert_to_ascii=True), data_point["label"]) with open("mag/train.txt", encoding="utf-8") as f: training_data = list(tqdm(parallel_map(preprocess, f.readlines()))) X_train = [d[0] for d in training_data] y_train = [d[1] for d in training_data] with open("mag/test.txt", encoding="utf-8") as f: test_data = list(tqdm(parallel_map(preprocess, f.readlines()))) X_test = [d[0] for d in test_data] y_test = [d[1] for d in test_data]

    84000it [00:09, 8647.60it/s]
    22399it [00:03, 6368.63it/s]

In [2]:
Copied!

    import pandas as pd
    from collections import Counter
    import plotly.express as px

    df = pd.DataFrame(Counter(y_train).most_common(), columns=["domain", "count"])
    px.bar(df, "domain", "count", width=1200, height=400).show()

import pandas as pd from collections import Counter import plotly.express as px df = pd.DataFrame(Counter(y_train).most_common(), columns=["domain", "count"]) px.bar(df, "domain", "count", width=1200, height=400).show()
In [3]:
Copied!

    from sklearn.naive_bayes import MultinomialNB
    from sklearn.pipeline import Pipeline
    from sklearn.feature_extraction.text import TfidfVectorizer

    def create_pipeline() -> Pipeline:
        return Pipeline(
            steps=[
                ("vectorizer", TfidfVectorizer(sublinear_tf=True)),
                ("classifier", MultinomialNB()),
            ]
        )

from sklearn.naive_bayes import MultinomialNB from sklearn.pipeline import Pipeline from sklearn.feature_extraction.text import TfidfVectorizer def create_pipeline() -> Pipeline: return Pipeline( steps=[ ("vectorizer", TfidfVectorizer(sublinear_tf=True)), ("classifier", MultinomialNB()), ] )
In [4]:
Copied!

    from sklearn.model_selection import GridSearchCV

    optimisation_pipeline = GridSearchCV(
        create_pipeline(),
        {
            "vectorizer__min_df": [5, 20, 100],
            "vectorizer__max_df": [0.05, 0.1],
            "classifier__alpha": [0.5, 1],
            "classifier__fit_prior": [True, False],
        },
        scoring="f1_macro",
        cv=3,
        n_jobs=-1,
        verbose=1,
    )
    optimisation_pipeline.fit(X_train, y_train)

    results = pd.DataFrame(optimisation_pipeline.cv_results_)
    results.sort_values("rank_test_score")

from sklearn.model_selection import GridSearchCV optimisation_pipeline = GridSearchCV( create_pipeline(), { "vectorizer__min_df": [5, 20, 100], "vectorizer__max_df": [0.05, 0.1], "classifier__alpha": [0.5, 1], "classifier__fit_prior": [True, False], }, scoring="f1_macro", cv=3, n_jobs=-1, verbose=1, ) optimisation_pipeline.fit(X_train, y_train) results = pd.DataFrame(optimisation_pipeline.cv_results_) results.sort_values("rank_test_score")

    Fitting 3 folds for each of 24 candidates, totalling 72 fits

Out[4]:

| mean_fit_time | std_fit_time | mean_score_time | std_score_time | param_classifier__alpha | param_classifier__fit_prior | param_vectorizer__max_df | param_vectorizer__min_df | params | split0_test_score | split1_test_score | split2_test_score | mean_test_score | std_test_score | rank_test_score
--- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | ---
12 | 1.227354 | 0.011710 | 0.652844 | 0.040511 | 1 | True | 0.05 | 5 | {'classifier__alpha': 1, 'classifier__fit_prio... | 0.677283 | 0.670398 | 0.665671 | 0.671118 | 0.004768 | 1
15 | 1.279783 | 0.037969 | 0.634761 | 0.068606 | 1 | True | 0.1 | 5 | {'classifier__alpha': 1, 'classifier__fit_prio... | 0.676970 | 0.670244 | 0.666123 | 0.671112 | 0.004470 | 2
18 | 1.332854 | 0.171405 | 0.743611 | 0.109792 | 1 | False | 0.05 | 5 | {'classifier__alpha': 1, 'classifier__fit_prio... | 0.675678 | 0.669321 | 0.665305 | 0.670101 | 0.004271 | 3
21 | 1.253222 | 0.072134 | 0.612344 | 0.016771 | 1 | False | 0.1 | 5 | {'classifier__alpha': 1, 'classifier__fit_prio... | 0.675569 | 0.669482 | 0.665225 | 0.670092 | 0.004245 | 4
3 | 1.472035 | 0.080849 | 0.654935 | 0.044302 | 0.5 | True | 0.1 | 5 | {'classifier__alpha': 0.5, 'classifier__fit_pr... | 0.675225 | 0.668912 | 0.665652 | 0.669930 | 0.003974 | 5
0 | 1.380641 | 0.054306 | 0.739966 | 0.053385 | 0.5 | True | 0.05 | 5 | {'classifier__alpha': 0.5, 'classifier__fit_pr... | 0.675264 | 0.668781 | 0.665394 | 0.669813 | 0.004095 | 6
9 | 1.284987 | 0.113903 | 0.696876 | 0.003757 | 0.5 | False | 0.1 | 5 | {'classifier__alpha': 0.5, 'classifier__fit_pr... | 0.674199 | 0.668005 | 0.664456 | 0.668887 | 0.004026 | 7
6 | 1.291148 | 0.101837 | 0.686561 | 0.083989 | 0.5 | False | 0.05 | 5 | {'classifier__alpha': 0.5, 'classifier__fit_pr... | 0.673996 | 0.668048 | 0.664572 | 0.668872 | 0.003891 | 8
13 | 1.268873 | 0.042412 | 0.645649 | 0.022738 | 1 | True | 0.05 | 20 | {'classifier__alpha': 1, 'classifier__fit_prio... | 0.656459 | 0.647433 | 0.644864 | 0.649585 | 0.004972 | 9
16 | 1.147120 | 0.009335 | 0.599006 | 0.045637 | 1 | True | 0.1 | 20 | {'classifier__alpha': 1, 'classifier__fit_prio... | 0.656196 | 0.646960 | 0.645436 | 0.649530 | 0.004754 | 10
4 | 1.186988 | 0.057330 | 0.642233 | 0.078248 | 0.5 | True | 0.1 | 20 | {'classifier__alpha': 0.5, 'classifier__fit_pr... | 0.654424 | 0.644690 | 0.644991 | 0.648035 | 0.004519 | 11
1 | 1.413231 | 0.153288 | 0.765960 | 0.099535 | 0.5 | True | 0.05 | 20 | {'classifier__alpha': 0.5, 'classifier__fit_pr... | 0.654622 | 0.644901 | 0.644425 | 0.647983 | 0.004699 | 12
19 | 1.193013 | 0.079000 | 0.691768 | 0.027448 | 1 | False | 0.05 | 20 | {'classifier__alpha': 1, 'classifier__fit_prio... | 0.655125 | 0.645586 | 0.643166 | 0.647959 | 0.005163 | 13
22 | 1.043618 | 0.098733 | 0.450375 | 0.061660 | 1 | False | 0.1 | 20 | {'classifier__alpha': 1, 'classifier__fit_prio... | 0.654254 | 0.645409 | 0.643514 | 0.647726 | 0.004680 | 14
7 | 1.301459 | 0.062143 | 0.660748 | 0.056054 | 0.5 | False | 0.05 | 20 | {'classifier__alpha': 0.5, 'classifier__fit_pr... | 0.652727 | 0.643691 | 0.642749 | 0.646389 | 0.004498 | 15
10 | 1.433934 | 0.155240 | 0.636608 | 0.024064 | 0.5 | False | 0.1 | 20 | {'classifier__alpha': 0.5, 'classifier__fit_pr... | 0.652290 | 0.642975 | 0.643421 | 0.646229 | 0.004290 | 16
14 | 1.325535 | 0.073539 | 0.672542 | 0.085835 | 1 | True | 0.05 | 100 | {'classifier__alpha': 1, 'classifier__fit_prio... | 0.587097 | 0.577411 | 0.580170 | 0.581560 | 0.004075 | 17
17 | 1.237677 | 0.070063 | 0.651091 | 0.102538 | 1 | True | 0.1 | 100 | {'classifier__alpha': 1, 'classifier__fit_prio... | 0.585713 | 0.577197 | 0.580763 | 0.581224 | 0.003492 | 18
2 | 1.394873 | 0.105286 | 0.637073 | 0.041708 | 0.5 | True | 0.05 | 100 | {'classifier__alpha': 0.5, 'classifier__fit_pr... | 0.586346 | 0.576987 | 0.579852 | 0.581062 | 0.003915 | 19
5 | 1.270732 | 0.013414 | 0.620984 | 0.025393 | 0.5 | True | 0.1 | 100 | {'classifier__alpha': 0.5, 'classifier__fit_pr... | 0.585634 | 0.576217 | 0.580320 | 0.580723 | 0.003855 | 20
20 | 1.177219 | 0.039633 | 0.586308 | 0.044247 | 1 | False | 0.05 | 100 | {'classifier__alpha': 1, 'classifier__fit_prio... | 0.581096 | 0.572833 | 0.576280 | 0.576736 | 0.003389 | 21
8 | 1.165583 | 0.046999 | 0.603675 | 0.031774 | 0.5 | False | 0.05 | 100 | {'classifier__alpha': 0.5, 'classifier__fit_pr... | 0.580841 | 0.572391 | 0.575731 | 0.576321 | 0.003475 | 22
23 | 0.907388 | 0.121543 | 0.354655 | 0.023775 | 1 | False | 0.1 | 100 | {'classifier__alpha': 1, 'classifier__fit_prio... | 0.579769 | 0.572373 | 0.576699 | 0.576280 | 0.003034 | 23
11 | 1.210845 | 0.041339 | 0.653606 | 0.028117 | 0.5 | False | 0.1 | 100 | {'classifier__alpha': 0.5, 'classifier__fit_pr... | 0.579743 | 0.571607 | 0.576352 | 0.575901 | 0.003337 | 24

In [5]:
Copied!

    from sklearn import set_config

    set_config(display="diagram")

    classifier = create_pipeline()
    classifier.set_params(**optimisation_pipeline.best_params_)
    classifier.fit(X_train, y_train)

from sklearn import set_config set_config(display="diagram") classifier = create_pipeline() classifier.set_params(**optimisation_pipeline.best_params_) classifier.fit(X_train, y_train)
Out[5]:

    Pipeline(steps=[('vectorizer',
                     TfidfVectorizer(max_df=0.05, min_df=5, sublinear_tf=True)),
                    ('classifier', MultinomialNB(alpha=1))])

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
Pipeline

    Pipeline(steps=[('vectorizer',
                     TfidfVectorizer(max_df=0.05, min_df=5, sublinear_tf=True)),
                    ('classifier', MultinomialNB(alpha=1))])

TfidfVectorizer

    TfidfVectorizer(max_df=0.05, min_df=5, sublinear_tf=True)

MultinomialNB

    MultinomialNB(alpha=1)

In [6]:
Copied!

    from sklearn import metrics
    import matplotlib.pyplot as plt

    y_predicted = classifier.predict(X_test)

    %matplotlib inline
    plt.rcParams["figure.figsize"] = (30, 15)
    plt.rcParams["figure.facecolor"] = "white"
    plt.rcParams["font.size"] = 12
    plt.rcParams["axes.xmargin"] = 0

    print(metrics.classification_report(y_test, y_predicted, digits=4))
    metrics.ConfusionMatrixDisplay.from_predictions(
        y_true=y_test,
        y_pred=y_predicted,
        xticks_rotation="vertical",
        normalize="pred",
        values_format=".2f",
    )
    None

from sklearn import metrics import matplotlib.pyplot as plt y_predicted = classifier.predict(X_test) %matplotlib inline plt.rcParams["figure.figsize"] = (30, 15) plt.rcParams["figure.facecolor"] = "white" plt.rcParams["font.size"] = 12 plt.rcParams["axes.xmargin"] = 0 print(metrics.classification_report(y_test, y_predicted, digits=4)) metrics.ConfusionMatrixDisplay.from_predictions( y_true=y_test, y_pred=y_predicted, xticks_rotation="vertical", normalize="pred", values_format=".2f", ) None

                  precision    recall  f1-score   support

        business     0.6412    0.7258    0.6808      3198
       economics     0.6635    0.7146    0.6881      3189
       geography     0.7461    0.6791    0.7111      3207
        medicine     0.8806    0.9021    0.8912      3187
        politics     0.5563    0.5895    0.5724      3169
      psychology     0.7654    0.6811    0.7208      3252
       sociology     0.5165    0.4698    0.4921      3197

        accuracy                         0.6803     22399
       macro avg     0.6814    0.6803    0.6795     22399
    weighted avg     0.6817    0.6803    0.6796     22399

Last update: July 11, 2022