{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Train a field of study (domain) classification of sentences on the [MAG](https://github.com/allenai/scibert/tree/master/data/text_classification/mag) dataset\n",
"\n",
"[SciBERT achieves an F1-score of 0.6571 on this dataset.](https://paperswithcode.com/sota/sentence-classification-on-paper-field) \n",
"This notebook shows that better results can be achieved without even using transformers."
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"84000it [00:09, 8647.60it/s] \n",
"22399it [00:03, 6368.63it/s]\n"
]
}
],
"source": [
"import json\n",
"from typing import Tuple\n",
"from great_ai.utilities import clean, parallel_map\n",
"from tqdm.cli import tqdm\n",
"\n",
"\n",
"def preprocess(line: str) -> Tuple[str, str]:\n",
" data_point = json.loads(line)\n",
"\n",
" return (clean(data_point[\"text\"], convert_to_ascii=True), data_point[\"label\"])\n",
"\n",
"\n",
"with open(\"mag/train.txt\", encoding=\"utf-8\") as f:\n",
" training_data = list(tqdm(parallel_map(preprocess, f.readlines())))\n",
"\n",
"X_train = [d[0] for d in training_data]\n",
"y_train = [d[1] for d in training_data]\n",
"\n",
"\n",
"with open(\"mag/test.txt\", encoding=\"utf-8\") as f:\n",
" test_data = list(tqdm(parallel_map(preprocess, f.readlines())))\n",
"\n",
"X_test = [d[0] for d in test_data]\n",
"y_test = [d[1] for d in test_data]"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.plotly.v1+json": {
"config": {
"plotlyServerURL": "https://plot.ly"
},
"data": [
{
"alignmentgroup": "True",
"hovertemplate": "domain=%{x}
count=%{y}
| \n", " | mean_fit_time | \n", "std_fit_time | \n", "mean_score_time | \n", "std_score_time | \n", "param_classifier__alpha | \n", "param_classifier__fit_prior | \n", "param_vectorizer__max_df | \n", "param_vectorizer__min_df | \n", "params | \n", "split0_test_score | \n", "split1_test_score | \n", "split2_test_score | \n", "mean_test_score | \n", "std_test_score | \n", "rank_test_score | \n", "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 12 | \n", "1.227354 | \n", "0.011710 | \n", "0.652844 | \n", "0.040511 | \n", "1 | \n", "True | \n", "0.05 | \n", "5 | \n", "{'classifier__alpha': 1, 'classifier__fit_prio... | \n", "0.677283 | \n", "0.670398 | \n", "0.665671 | \n", "0.671118 | \n", "0.004768 | \n", "1 | \n", "
| 15 | \n", "1.279783 | \n", "0.037969 | \n", "0.634761 | \n", "0.068606 | \n", "1 | \n", "True | \n", "0.1 | \n", "5 | \n", "{'classifier__alpha': 1, 'classifier__fit_prio... | \n", "0.676970 | \n", "0.670244 | \n", "0.666123 | \n", "0.671112 | \n", "0.004470 | \n", "2 | \n", "
| 18 | \n", "1.332854 | \n", "0.171405 | \n", "0.743611 | \n", "0.109792 | \n", "1 | \n", "False | \n", "0.05 | \n", "5 | \n", "{'classifier__alpha': 1, 'classifier__fit_prio... | \n", "0.675678 | \n", "0.669321 | \n", "0.665305 | \n", "0.670101 | \n", "0.004271 | \n", "3 | \n", "
| 21 | \n", "1.253222 | \n", "0.072134 | \n", "0.612344 | \n", "0.016771 | \n", "1 | \n", "False | \n", "0.1 | \n", "5 | \n", "{'classifier__alpha': 1, 'classifier__fit_prio... | \n", "0.675569 | \n", "0.669482 | \n", "0.665225 | \n", "0.670092 | \n", "0.004245 | \n", "4 | \n", "
| 3 | \n", "1.472035 | \n", "0.080849 | \n", "0.654935 | \n", "0.044302 | \n", "0.5 | \n", "True | \n", "0.1 | \n", "5 | \n", "{'classifier__alpha': 0.5, 'classifier__fit_pr... | \n", "0.675225 | \n", "0.668912 | \n", "0.665652 | \n", "0.669930 | \n", "0.003974 | \n", "5 | \n", "
| 0 | \n", "1.380641 | \n", "0.054306 | \n", "0.739966 | \n", "0.053385 | \n", "0.5 | \n", "True | \n", "0.05 | \n", "5 | \n", "{'classifier__alpha': 0.5, 'classifier__fit_pr... | \n", "0.675264 | \n", "0.668781 | \n", "0.665394 | \n", "0.669813 | \n", "0.004095 | \n", "6 | \n", "
| 9 | \n", "1.284987 | \n", "0.113903 | \n", "0.696876 | \n", "0.003757 | \n", "0.5 | \n", "False | \n", "0.1 | \n", "5 | \n", "{'classifier__alpha': 0.5, 'classifier__fit_pr... | \n", "0.674199 | \n", "0.668005 | \n", "0.664456 | \n", "0.668887 | \n", "0.004026 | \n", "7 | \n", "
| 6 | \n", "1.291148 | \n", "0.101837 | \n", "0.686561 | \n", "0.083989 | \n", "0.5 | \n", "False | \n", "0.05 | \n", "5 | \n", "{'classifier__alpha': 0.5, 'classifier__fit_pr... | \n", "0.673996 | \n", "0.668048 | \n", "0.664572 | \n", "0.668872 | \n", "0.003891 | \n", "8 | \n", "
| 13 | \n", "1.268873 | \n", "0.042412 | \n", "0.645649 | \n", "0.022738 | \n", "1 | \n", "True | \n", "0.05 | \n", "20 | \n", "{'classifier__alpha': 1, 'classifier__fit_prio... | \n", "0.656459 | \n", "0.647433 | \n", "0.644864 | \n", "0.649585 | \n", "0.004972 | \n", "9 | \n", "
| 16 | \n", "1.147120 | \n", "0.009335 | \n", "0.599006 | \n", "0.045637 | \n", "1 | \n", "True | \n", "0.1 | \n", "20 | \n", "{'classifier__alpha': 1, 'classifier__fit_prio... | \n", "0.656196 | \n", "0.646960 | \n", "0.645436 | \n", "0.649530 | \n", "0.004754 | \n", "10 | \n", "
| 4 | \n", "1.186988 | \n", "0.057330 | \n", "0.642233 | \n", "0.078248 | \n", "0.5 | \n", "True | \n", "0.1 | \n", "20 | \n", "{'classifier__alpha': 0.5, 'classifier__fit_pr... | \n", "0.654424 | \n", "0.644690 | \n", "0.644991 | \n", "0.648035 | \n", "0.004519 | \n", "11 | \n", "
| 1 | \n", "1.413231 | \n", "0.153288 | \n", "0.765960 | \n", "0.099535 | \n", "0.5 | \n", "True | \n", "0.05 | \n", "20 | \n", "{'classifier__alpha': 0.5, 'classifier__fit_pr... | \n", "0.654622 | \n", "0.644901 | \n", "0.644425 | \n", "0.647983 | \n", "0.004699 | \n", "12 | \n", "
| 19 | \n", "1.193013 | \n", "0.079000 | \n", "0.691768 | \n", "0.027448 | \n", "1 | \n", "False | \n", "0.05 | \n", "20 | \n", "{'classifier__alpha': 1, 'classifier__fit_prio... | \n", "0.655125 | \n", "0.645586 | \n", "0.643166 | \n", "0.647959 | \n", "0.005163 | \n", "13 | \n", "
| 22 | \n", "1.043618 | \n", "0.098733 | \n", "0.450375 | \n", "0.061660 | \n", "1 | \n", "False | \n", "0.1 | \n", "20 | \n", "{'classifier__alpha': 1, 'classifier__fit_prio... | \n", "0.654254 | \n", "0.645409 | \n", "0.643514 | \n", "0.647726 | \n", "0.004680 | \n", "14 | \n", "
| 7 | \n", "1.301459 | \n", "0.062143 | \n", "0.660748 | \n", "0.056054 | \n", "0.5 | \n", "False | \n", "0.05 | \n", "20 | \n", "{'classifier__alpha': 0.5, 'classifier__fit_pr... | \n", "0.652727 | \n", "0.643691 | \n", "0.642749 | \n", "0.646389 | \n", "0.004498 | \n", "15 | \n", "
| 10 | \n", "1.433934 | \n", "0.155240 | \n", "0.636608 | \n", "0.024064 | \n", "0.5 | \n", "False | \n", "0.1 | \n", "20 | \n", "{'classifier__alpha': 0.5, 'classifier__fit_pr... | \n", "0.652290 | \n", "0.642975 | \n", "0.643421 | \n", "0.646229 | \n", "0.004290 | \n", "16 | \n", "
| 14 | \n", "1.325535 | \n", "0.073539 | \n", "0.672542 | \n", "0.085835 | \n", "1 | \n", "True | \n", "0.05 | \n", "100 | \n", "{'classifier__alpha': 1, 'classifier__fit_prio... | \n", "0.587097 | \n", "0.577411 | \n", "0.580170 | \n", "0.581560 | \n", "0.004075 | \n", "17 | \n", "
| 17 | \n", "1.237677 | \n", "0.070063 | \n", "0.651091 | \n", "0.102538 | \n", "1 | \n", "True | \n", "0.1 | \n", "100 | \n", "{'classifier__alpha': 1, 'classifier__fit_prio... | \n", "0.585713 | \n", "0.577197 | \n", "0.580763 | \n", "0.581224 | \n", "0.003492 | \n", "18 | \n", "
| 2 | \n", "1.394873 | \n", "0.105286 | \n", "0.637073 | \n", "0.041708 | \n", "0.5 | \n", "True | \n", "0.05 | \n", "100 | \n", "{'classifier__alpha': 0.5, 'classifier__fit_pr... | \n", "0.586346 | \n", "0.576987 | \n", "0.579852 | \n", "0.581062 | \n", "0.003915 | \n", "19 | \n", "
| 5 | \n", "1.270732 | \n", "0.013414 | \n", "0.620984 | \n", "0.025393 | \n", "0.5 | \n", "True | \n", "0.1 | \n", "100 | \n", "{'classifier__alpha': 0.5, 'classifier__fit_pr... | \n", "0.585634 | \n", "0.576217 | \n", "0.580320 | \n", "0.580723 | \n", "0.003855 | \n", "20 | \n", "
| 20 | \n", "1.177219 | \n", "0.039633 | \n", "0.586308 | \n", "0.044247 | \n", "1 | \n", "False | \n", "0.05 | \n", "100 | \n", "{'classifier__alpha': 1, 'classifier__fit_prio... | \n", "0.581096 | \n", "0.572833 | \n", "0.576280 | \n", "0.576736 | \n", "0.003389 | \n", "21 | \n", "
| 8 | \n", "1.165583 | \n", "0.046999 | \n", "0.603675 | \n", "0.031774 | \n", "0.5 | \n", "False | \n", "0.05 | \n", "100 | \n", "{'classifier__alpha': 0.5, 'classifier__fit_pr... | \n", "0.580841 | \n", "0.572391 | \n", "0.575731 | \n", "0.576321 | \n", "0.003475 | \n", "22 | \n", "
| 23 | \n", "0.907388 | \n", "0.121543 | \n", "0.354655 | \n", "0.023775 | \n", "1 | \n", "False | \n", "0.1 | \n", "100 | \n", "{'classifier__alpha': 1, 'classifier__fit_prio... | \n", "0.579769 | \n", "0.572373 | \n", "0.576699 | \n", "0.576280 | \n", "0.003034 | \n", "23 | \n", "
| 11 | \n", "1.210845 | \n", "0.041339 | \n", "0.653606 | \n", "0.028117 | \n", "0.5 | \n", "False | \n", "0.1 | \n", "100 | \n", "{'classifier__alpha': 0.5, 'classifier__fit_pr... | \n", "0.579743 | \n", "0.571607 | \n", "0.576352 | \n", "0.575901 | \n", "0.003337 | \n", "24 | \n", "
Pipeline(steps=[('vectorizer',\n",
" TfidfVectorizer(max_df=0.05, min_df=5, sublinear_tf=True)),\n",
" ('classifier', MultinomialNB(alpha=1))])In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. Pipeline(steps=[('vectorizer',\n",
" TfidfVectorizer(max_df=0.05, min_df=5, sublinear_tf=True)),\n",
" ('classifier', MultinomialNB(alpha=1))])TfidfVectorizer(max_df=0.05, min_df=5, sublinear_tf=True)
MultinomialNB(alpha=1)