{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Train Domain classifier from the [semantic scholar dataset](https://api.semanticscholar.org/corpus)" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import json\n", "import matplotlib.pyplot as plt\n", "from pathlib import Path\n", "import pandas as pd\n", "from sklearn import metrics, set_config\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.naive_bayes import ComplementNB\n", "from sklearn.pipeline import Pipeline\n", "from sklearn.feature_extraction.text import TfidfVectorizer\n", "from sklearn.model_selection import GridSearchCV\n", "\n", "from good_ai.utilities.clean import clean\n", "from good_ai.utilities.parallel_map import parallel_map\n", "from good_ai.utilities.language import is_english, predict_language\n", "from good_ai import save_model, set_default_config, LargeFile\n", "\n", "from preprocess import preprocess\n", "from config import model_key" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Configuration" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "PREFIX = \"domain-\"\n", "DATASET_KEY = \"data\"\n", "MAX_FILE_COUNT = 5" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "\u001b[38;5;39m2022-04-10 09:40:33,153 | INFO | Running in production mode ✅\u001b[0m\n", "\u001b[38;5;226m2022-04-10 09:40:33,156 | WARNING | The selected persistence driver (TinyDbDriver) is not threadsafe\u001b[0m\n", "\u001b[38;5;39m2022-04-10 09:40:33,157 | INFO | Defaults: configured ✅\u001b[0m\n", "\u001b[38;5;39m2022-04-10 09:40:33,348 | INFO | Fetching online versions of data\u001b[0m\n", "\u001b[38;5;39m2022-04-10 09:40:33,743 | INFO | Found versions: [0, 1]\u001b[0m\n", "\u001b[38;5;39m2022-04-10 09:40:33,744 | INFO | Lastest version of data-1 is 1\u001b[0m\n", "\u001b[38;5;39m2022-04-10 09:40:33,745 | INFO | File data-1 found in cache\u001b[0m\n" ] } ], "source": [ "set_default_config()\n", "corpus_path = LargeFile(DATASET_KEY).get()\n", "\n", "set_config(display=\"diagram\")\n", "plt.rcParams[\"figure.figsize\"] = (30, 15)\n", "plt.rcParams[\"figure.facecolor\"] = \"white\"\n", "plt.rcParams[\"font.size\"] = 12\n", "plt.rcParams[\"axes.xmargin\"] = 0" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Preprocessing" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ " 0%| | 0/5 [00:00, ?it/s]INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,7093)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,811)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,196)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,561)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1370)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,215)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,330)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,603)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected closing environment: 'document' @(26,0)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected closing math mode: '\\]' @(1,533)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,713)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,6244)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,6549)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,997)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,549)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected closing math mode: '\\)' @(1,674)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,748)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,70)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,128)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,112)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,175)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,321)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected closing math mode: '\\]' @(1,700)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1180)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,754)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,487)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,390)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,942)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1471)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,2235)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,27)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1176)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,2502)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,2545)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,2856)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,84)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,257)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,391)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,648)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,346)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,716)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,782)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,958)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,964)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1017)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1287)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,104)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,119)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1139)\n", "INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,366)\n", "100%|██████████| 5/5 [34:00<00:00, 408.07s/it] \n" ] } ], "source": [ "def clean_file(p: Path) -> None:\n", " try:\n", " processed_path = p.with_name(f\"{PREFIX}{p.stem}{p.suffix}\")\n", "\n", " if processed_path.exists():\n", " return\n", "\n", " with open(p) as f:\n", " content = json.load(f)\n", "\n", " result = {\n", " preprocess(\n", " clean(f'{c[\"title\"]} {c[\"abstract\"]}', convert_to_ascii=True)\n", " ): c[\"domain\"]\n", " for c in content\n", " if (\n", " c[\"domain\"]\n", " and c[\"abstract\"]\n", " and is_english(predict_language(c[\"abstract\"]))\n", " )\n", " }\n", "\n", " with open(processed_path, \"w\") as f:\n", " json.dump(result, f)\n", " except Exception as e:\n", " print(f\"Error ({e}) processing {p}\")\n", "\n", "\n", "parallel_map(\n", " clean_file,\n", " list(corpus_path.glob(\"s2-corpus-*.json\"))[:MAX_FILE_COUNT],\n", " chunk_size=1,\n", ")\n", "None" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Found 5 files\n", "Found 76862 documents\n" ] } ], "source": [ "corpora = list(corpus_path.glob(f\"{PREFIX}*.json\"))[:MAX_FILE_COUNT]\n", "print(f\"Found {len(corpora)} files\")\n", "\n", "data = []\n", "for p in corpora:\n", " with open(p) as f:\n", " data.extend(json.load(f).items())\n", "\n", "print(f\"Found {len(data)} documents\")\n", "\n", "X_train, X_test, y_train, y_test = train_test_split(\n", " [d[0] for d in data], [d[1] for d in data], test_size=0.1, random_state=1\n", ")\n", "\n", "X_train = [x for x, y in zip(X_train, y_train) for domain in y]\n", "y_train = [domain for x, y in zip(X_train, y_train) for domain in y]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Naive Bayes" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Fitting 3 folds for each of 72 candidates, totalling 216 fits\n" ] }, { "data": { "text/html": [ "
| \n", " | mean_fit_time | \n", "std_fit_time | \n", "mean_score_time | \n", "std_score_time | \n", "param_classifier__alpha | \n", "param_classifier__fit_prior | \n", "param_vectorizer__max_df | \n", "param_vectorizer__min_df | \n", "param_vectorizer__sublinear_tf | \n", "params | \n", "split0_test_score | \n", "split1_test_score | \n", "split2_test_score | \n", "mean_test_score | \n", "std_test_score | \n", "rank_test_score | \n", "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 37 | \n", "353.990156 | \n", "490.126502 | \n", "3.731860 | \n", "0.097140 | \n", "0.5 | \n", "False | \n", "0.1 | \n", "5 | \n", "False | \n", "{'classifier__alpha': 0.5, 'classifier__fit_pr... | \n", "0.495838 | \n", "0.499754 | \n", "0.501146 | \n", "0.498912 | \n", "0.002247 | \n", "1 | \n", "
| 25 | \n", "671.987925 | \n", "469.392562 | \n", "335.414978 | \n", "468.916068 | \n", "0.5 | \n", "True | \n", "0.1 | \n", "5 | \n", "False | \n", "{'classifier__alpha': 0.5, 'classifier__fit_pr... | \n", "0.495838 | \n", "0.499754 | \n", "0.501146 | \n", "0.498912 | \n", "0.002247 | \n", "1 | \n", "
| 1 | \n", "334.102561 | \n", "460.200041 | \n", "4.549770 | \n", "0.351197 | \n", "0.1 | \n", "True | \n", "0.1 | \n", "5 | \n", "False | \n", "{'classifier__alpha': 0.1, 'classifier__fit_pr... | \n", "0.495637 | \n", "0.499200 | \n", "0.501269 | \n", "0.498702 | \n", "0.002326 | \n", "3 | \n", "
| 13 | \n", "6.975998 | \n", "0.301279 | \n", "3.720828 | \n", "0.145939 | \n", "0.1 | \n", "False | \n", "0.1 | \n", "5 | \n", "False | \n", "{'classifier__alpha': 0.1, 'classifier__fit_pr... | \n", "0.495637 | \n", "0.499200 | \n", "0.501269 | \n", "0.498702 | \n", "0.002326 | \n", "3 | \n", "
| 7 | \n", "8.587186 | \n", "0.258101 | \n", "3.718949 | \n", "0.054138 | \n", "0.1 | \n", "True | \n", "0.3 | \n", "5 | \n", "False | \n", "{'classifier__alpha': 0.1, 'classifier__fit_pr... | \n", "0.495246 | \n", "0.498718 | \n", "0.500819 | \n", "0.498261 | \n", "0.002298 | \n", "5 | \n", "
| ... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "
| 34 | \n", "7.424036 | \n", "0.104495 | \n", "3.518187 | \n", "0.103824 | \n", "0.5 | \n", "True | \n", "0.3 | \n", "30 | \n", "True | \n", "{'classifier__alpha': 0.5, 'classifier__fit_pr... | \n", "0.471803 | \n", "0.470317 | \n", "0.465951 | \n", "0.469357 | \n", "0.002484 | \n", "67 | \n", "
| 22 | \n", "7.288773 | \n", "0.153564 | \n", "3.557420 | \n", "0.152864 | \n", "0.1 | \n", "False | \n", "0.3 | \n", "30 | \n", "True | \n", "{'classifier__alpha': 0.1, 'classifier__fit_pr... | \n", "0.470711 | \n", "0.469985 | \n", "0.467028 | \n", "0.469241 | \n", "0.001593 | \n", "69 | \n", "
| 10 | \n", "627.484071 | \n", "438.167044 | \n", "3.580379 | \n", "0.105862 | \n", "0.1 | \n", "True | \n", "0.3 | \n", "30 | \n", "True | \n", "{'classifier__alpha': 0.1, 'classifier__fit_pr... | \n", "0.470711 | \n", "0.469985 | \n", "0.467028 | \n", "0.469241 | \n", "0.001593 | \n", "69 | \n", "
| 58 | \n", "341.006728 | \n", "471.608243 | \n", "3.780825 | \n", "0.007465 | \n", "1 | \n", "True | \n", "0.3 | \n", "30 | \n", "True | \n", "{'classifier__alpha': 1, 'classifier__fit_prio... | \n", "0.470638 | \n", "0.469516 | \n", "0.464513 | \n", "0.468223 | \n", "0.002663 | \n", "71 | \n", "
| 70 | \n", "677.439482 | \n", "473.755547 | \n", "4.152508 | \n", "0.296995 | \n", "1 | \n", "False | \n", "0.3 | \n", "30 | \n", "True | \n", "{'classifier__alpha': 1, 'classifier__fit_prio... | \n", "0.470638 | \n", "0.469516 | \n", "0.464513 | \n", "0.468223 | \n", "0.002663 | \n", "71 | \n", "
72 rows × 16 columns
\n", "Pipeline(steps=[('vectorizer',\n",
" TfidfVectorizer(max_df=0.1, min_df=10, token_pattern='[^ ]+')),\n",
" ('classifier', ComplementNB(alpha=0.5, fit_prior=False))])Please rerun this cell to show the HTML repr or trust the notebook.Pipeline(steps=[('vectorizer',\n",
" TfidfVectorizer(max_df=0.1, min_df=10, token_pattern='[^ ]+')),\n",
" ('classifier', ComplementNB(alpha=0.5, fit_prior=False))])TfidfVectorizer(max_df=0.1, min_df=10, token_pattern='[^ ]+')
ComplementNB(alpha=0.5, fit_prior=False)