{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Train a field of study (domain) classification of sentences on the [MAG](https://github.com/allenai/scibert/tree/master/data/text_classification/mag) dataset\n",
"\n",
"[SciBERT achieves an F1-score of 0.6571 on this dataset.](https://paperswithcode.com/sota/sentence-classification-on-paper-field) \n",
"This notebook shows that better results can be achieved without even using transformers."
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[38;5;39m2022-06-20 14:33:17,570 | INFO | Starting parallel map (concurrency: 12, chunk size: 700)\u001b[0m\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "e6f76bf83615422cb4352dcb3af48a26",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/84000 [00:00, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[38;5;39m2022-06-20 14:33:24,846 | INFO | Starting parallel map (concurrency: 12, chunk size: 187)\u001b[0m\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "ab012f9276d74bdda15b00cfec6c8f56",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/22399 [00:00, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"import json\n",
"from typing import Tuple\n",
"from great_ai.utilities import clean, parallel_map\n",
"\n",
"\n",
"def preprocess(line: str) -> Tuple[str, str]:\n",
" data_point = json.loads(line)\n",
"\n",
" return (clean(data_point[\"text\"], convert_to_ascii=True), data_point[\"label\"])\n",
"\n",
"\n",
"with open(\"mag/train.txt\", encoding=\"utf-8\") as f:\n",
" training_data = parallel_map(preprocess, f.readlines())\n",
"\n",
"X_train = [d[0] for d in training_data]\n",
"y_train = [d[1] for d in training_data]\n",
"\n",
"\n",
"with open(\"mag/test.txt\", encoding=\"utf-8\") as f:\n",
" test_data = parallel_map(preprocess, f.readlines())\n",
"\n",
"X_test = [d[0] for d in test_data]\n",
"y_test = [d[1] for d in test_data]"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.plotly.v1+json": {
"config": {
"plotlyServerURL": "https://plot.ly"
},
"data": [
{
"alignmentgroup": "True",
"hovertemplate": "domain=%{x}
count=%{y}
| \n", " | mean_fit_time | \n", "std_fit_time | \n", "mean_score_time | \n", "std_score_time | \n", "param_classifier__alpha | \n", "param_classifier__fit_prior | \n", "param_vectorizer__max_df | \n", "param_vectorizer__min_df | \n", "params | \n", "split0_test_score | \n", "split1_test_score | \n", "split2_test_score | \n", "mean_test_score | \n", "std_test_score | \n", "rank_test_score | \n", "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 12 | \n", "1.172014 | \n", "0.033318 | \n", "0.625773 | \n", "0.013720 | \n", "1 | \n", "True | \n", "0.05 | \n", "5 | \n", "{'classifier__alpha': 1, 'classifier__fit_prio... | \n", "0.677283 | \n", "0.670398 | \n", "0.665671 | \n", "0.671118 | \n", "0.004768 | \n", "1 | \n", "
| 15 | \n", "1.201231 | \n", "0.016728 | \n", "0.613568 | \n", "0.013720 | \n", "1 | \n", "True | \n", "0.1 | \n", "5 | \n", "{'classifier__alpha': 1, 'classifier__fit_prio... | \n", "0.676970 | \n", "0.670244 | \n", "0.666123 | \n", "0.671112 | \n", "0.004470 | \n", "2 | \n", "
| 18 | \n", "1.211270 | \n", "0.036636 | \n", "0.670754 | \n", "0.076857 | \n", "1 | \n", "False | \n", "0.05 | \n", "5 | \n", "{'classifier__alpha': 1, 'classifier__fit_prio... | \n", "0.675678 | \n", "0.669321 | \n", "0.665305 | \n", "0.670101 | \n", "0.004271 | \n", "3 | \n", "
| 21 | \n", "1.165763 | \n", "0.061206 | \n", "0.549646 | \n", "0.046776 | \n", "1 | \n", "False | \n", "0.1 | \n", "5 | \n", "{'classifier__alpha': 1, 'classifier__fit_prio... | \n", "0.675569 | \n", "0.669482 | \n", "0.665225 | \n", "0.670092 | \n", "0.004245 | \n", "4 | \n", "
| 3 | \n", "1.277603 | \n", "0.127785 | \n", "0.650135 | \n", "0.023038 | \n", "0.5 | \n", "True | \n", "0.1 | \n", "5 | \n", "{'classifier__alpha': 0.5, 'classifier__fit_pr... | \n", "0.675225 | \n", "0.668912 | \n", "0.665652 | \n", "0.669930 | \n", "0.003974 | \n", "5 | \n", "
| 0 | \n", "1.118055 | \n", "0.058978 | \n", "0.622284 | \n", "0.014230 | \n", "0.5 | \n", "True | \n", "0.05 | \n", "5 | \n", "{'classifier__alpha': 0.5, 'classifier__fit_pr... | \n", "0.675264 | \n", "0.668781 | \n", "0.665394 | \n", "0.669813 | \n", "0.004095 | \n", "6 | \n", "
| 9 | \n", "1.166192 | \n", "0.044232 | \n", "0.619351 | \n", "0.024824 | \n", "0.5 | \n", "False | \n", "0.1 | \n", "5 | \n", "{'classifier__alpha': 0.5, 'classifier__fit_pr... | \n", "0.674199 | \n", "0.668005 | \n", "0.664456 | \n", "0.668887 | \n", "0.004026 | \n", "7 | \n", "
| 6 | \n", "1.233377 | \n", "0.063715 | \n", "0.666418 | \n", "0.061664 | \n", "0.5 | \n", "False | \n", "0.05 | \n", "5 | \n", "{'classifier__alpha': 0.5, 'classifier__fit_pr... | \n", "0.673996 | \n", "0.668048 | \n", "0.664572 | \n", "0.668872 | \n", "0.003891 | \n", "8 | \n", "
| 13 | \n", "1.245147 | \n", "0.089528 | \n", "0.645836 | \n", "0.006279 | \n", "1 | \n", "True | \n", "0.05 | \n", "20 | \n", "{'classifier__alpha': 1, 'classifier__fit_prio... | \n", "0.656459 | \n", "0.647433 | \n", "0.644864 | \n", "0.649585 | \n", "0.004972 | \n", "9 | \n", "
| 16 | \n", "1.145362 | \n", "0.071149 | \n", "0.637540 | \n", "0.028854 | \n", "1 | \n", "True | \n", "0.1 | \n", "20 | \n", "{'classifier__alpha': 1, 'classifier__fit_prio... | \n", "0.656196 | \n", "0.646960 | \n", "0.645436 | \n", "0.649530 | \n", "0.004754 | \n", "10 | \n", "
| 4 | \n", "1.179663 | \n", "0.046808 | \n", "0.598699 | \n", "0.035973 | \n", "0.5 | \n", "True | \n", "0.1 | \n", "20 | \n", "{'classifier__alpha': 0.5, 'classifier__fit_pr... | \n", "0.654424 | \n", "0.644690 | \n", "0.644991 | \n", "0.648035 | \n", "0.004519 | \n", "11 | \n", "
| 1 | \n", "1.097773 | \n", "0.005871 | \n", "0.596514 | \n", "0.025899 | \n", "0.5 | \n", "True | \n", "0.05 | \n", "20 | \n", "{'classifier__alpha': 0.5, 'classifier__fit_pr... | \n", "0.654622 | \n", "0.644901 | \n", "0.644425 | \n", "0.647983 | \n", "0.004699 | \n", "12 | \n", "
| 19 | \n", "1.161582 | \n", "0.040011 | \n", "0.586831 | \n", "0.006144 | \n", "1 | \n", "False | \n", "0.05 | \n", "20 | \n", "{'classifier__alpha': 1, 'classifier__fit_prio... | \n", "0.655125 | \n", "0.645586 | \n", "0.643166 | \n", "0.647959 | \n", "0.005163 | \n", "13 | \n", "
| 22 | \n", "1.106277 | \n", "0.060809 | \n", "0.371986 | \n", "0.014114 | \n", "1 | \n", "False | \n", "0.1 | \n", "20 | \n", "{'classifier__alpha': 1, 'classifier__fit_prio... | \n", "0.654254 | \n", "0.645409 | \n", "0.643514 | \n", "0.647726 | \n", "0.004680 | \n", "14 | \n", "
| 7 | \n", "1.217713 | \n", "0.044787 | \n", "0.587896 | \n", "0.024613 | \n", "0.5 | \n", "False | \n", "0.05 | \n", "20 | \n", "{'classifier__alpha': 0.5, 'classifier__fit_pr... | \n", "0.652727 | \n", "0.643691 | \n", "0.642749 | \n", "0.646389 | \n", "0.004498 | \n", "15 | \n", "
| 10 | \n", "1.176654 | \n", "0.045971 | \n", "0.628005 | \n", "0.082375 | \n", "0.5 | \n", "False | \n", "0.1 | \n", "20 | \n", "{'classifier__alpha': 0.5, 'classifier__fit_pr... | \n", "0.652290 | \n", "0.642975 | \n", "0.643421 | \n", "0.646229 | \n", "0.004290 | \n", "16 | \n", "
| 14 | \n", "1.207308 | \n", "0.080923 | \n", "0.610284 | \n", "0.039157 | \n", "1 | \n", "True | \n", "0.05 | \n", "100 | \n", "{'classifier__alpha': 1, 'classifier__fit_prio... | \n", "0.587097 | \n", "0.577411 | \n", "0.580170 | \n", "0.581560 | \n", "0.004075 | \n", "17 | \n", "
| 17 | \n", "1.201498 | \n", "0.035161 | \n", "0.574234 | \n", "0.019247 | \n", "1 | \n", "True | \n", "0.1 | \n", "100 | \n", "{'classifier__alpha': 1, 'classifier__fit_prio... | \n", "0.585713 | \n", "0.577197 | \n", "0.580763 | \n", "0.581224 | \n", "0.003492 | \n", "18 | \n", "
| 2 | \n", "1.157947 | \n", "0.096841 | \n", "0.586462 | \n", "0.010952 | \n", "0.5 | \n", "True | \n", "0.05 | \n", "100 | \n", "{'classifier__alpha': 0.5, 'classifier__fit_pr... | \n", "0.586346 | \n", "0.576987 | \n", "0.579852 | \n", "0.581062 | \n", "0.003915 | \n", "19 | \n", "
| 5 | \n", "1.150511 | \n", "0.027807 | \n", "0.562870 | \n", "0.013724 | \n", "0.5 | \n", "True | \n", "0.1 | \n", "100 | \n", "{'classifier__alpha': 0.5, 'classifier__fit_pr... | \n", "0.585634 | \n", "0.576217 | \n", "0.580320 | \n", "0.580723 | \n", "0.003855 | \n", "20 | \n", "
| 20 | \n", "1.237030 | \n", "0.006805 | \n", "0.529591 | \n", "0.064449 | \n", "1 | \n", "False | \n", "0.05 | \n", "100 | \n", "{'classifier__alpha': 1, 'classifier__fit_prio... | \n", "0.581096 | \n", "0.572833 | \n", "0.576280 | \n", "0.576736 | \n", "0.003389 | \n", "21 | \n", "
| 8 | \n", "1.103937 | \n", "0.046513 | \n", "0.569367 | \n", "0.011001 | \n", "0.5 | \n", "False | \n", "0.05 | \n", "100 | \n", "{'classifier__alpha': 0.5, 'classifier__fit_pr... | \n", "0.580841 | \n", "0.572391 | \n", "0.575731 | \n", "0.576321 | \n", "0.003475 | \n", "22 | \n", "
| 23 | \n", "1.080252 | \n", "0.040597 | \n", "0.326627 | \n", "0.002905 | \n", "1 | \n", "False | \n", "0.1 | \n", "100 | \n", "{'classifier__alpha': 1, 'classifier__fit_prio... | \n", "0.579769 | \n", "0.572373 | \n", "0.576699 | \n", "0.576280 | \n", "0.003034 | \n", "23 | \n", "
| 11 | \n", "1.275601 | \n", "0.087971 | \n", "0.580025 | \n", "0.034914 | \n", "0.5 | \n", "False | \n", "0.1 | \n", "100 | \n", "{'classifier__alpha': 0.5, 'classifier__fit_pr... | \n", "0.579743 | \n", "0.571607 | \n", "0.576352 | \n", "0.575901 | \n", "0.003337 | \n", "24 | \n", "
Pipeline(steps=[('vectorizer',\n",
" TfidfVectorizer(max_df=0.05, min_df=5, sublinear_tf=True)),\n",
" ('classifier', MultinomialNB(alpha=1))])In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. Pipeline(steps=[('vectorizer',\n",
" TfidfVectorizer(max_df=0.05, min_df=5, sublinear_tf=True)),\n",
" ('classifier', MultinomialNB(alpha=1))])TfidfVectorizer(max_df=0.05, min_df=5, sublinear_tf=True)
MultinomialNB(alpha=1)