From 911c4f64f796c75740c538c83138bed4ab887f9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A1s=20Schmelczer?= Date: Thu, 26 May 2022 13:33:27 +0200 Subject: [PATCH] Imrpove examples --- examples/complex/main_batch.py | 4 +- examples/complex/predict_domain.py | 6 +- examples/simple/README.md | 12 ++ examples/simple/helpers.py | 16 ++ examples/simple/predict_domain.py | 21 +-- examples/simple/preprocess.py | 8 - examples/simple/train.ipynb | 269 +++++++++++++++++++++++++++++ 7 files changed, 308 insertions(+), 28 deletions(-) create mode 100644 examples/simple/README.md create mode 100644 examples/simple/helpers.py delete mode 100644 examples/simple/preprocess.py create mode 100644 examples/simple/train.ipynb diff --git a/examples/complex/main_batch.py b/examples/complex/main_batch.py index 735c875..9d33222 100755 --- a/examples/complex/main_batch.py +++ b/examples/complex/main_batch.py @@ -4,10 +4,10 @@ import json from random import shuffle from devtools import debug -from predict_domain import predict_domain - from great_ai import process_batch +from predict_domain import predict_domain + if __name__ == "__main__": with open(".cache/data-1/s2-corpus-323.json") as f: raw = json.load(f) diff --git a/examples/complex/predict_domain.py b/examples/complex/predict_domain.py index 04c0260..8af032d 100644 --- a/examples/complex/predict_domain.py +++ b/examples/complex/predict_domain.py @@ -1,12 +1,12 @@ import re from typing import Dict, Iterable, List -from preprocess import preprocess +from great_ai import log_argument, log_metric, use_model +from great_ai.utilities.clean import clean from pydantic import BaseModel from sklearn.pipeline import Pipeline -from great_ai import log_argument, log_metric, use_model -from great_ai.utilities.clean import clean +from preprocess import preprocess class DomainPrediction(BaseModel): diff --git a/examples/simple/README.md b/examples/simple/README.md new file mode 100644 index 0000000..627f957 --- /dev/null +++ b/examples/simple/README.md @@ -0,0 +1,12 @@ +# Train Domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus) + +## Upload the dataset (or a part of it) to shared infrastructure + +```sh +mkdir ss-data && cd ss-data +wget https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/manifest.txt +wget -B https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/ -i manifest.txt +cd - +python3 -m great_ai.open_s3 --secrets s3.ini --push ss-data +rm -rf ss-data +``` diff --git a/examples/simple/helpers.py b/examples/simple/helpers.py new file mode 100644 index 0000000..af2c51e --- /dev/null +++ b/examples/simple/helpers.py @@ -0,0 +1,16 @@ +import re + +from great_ai.utilities.clean import clean +from great_ai.utilities.lemmatize_text import lemmatize_text + + +def preprocess(text: str) -> str: + text = clean(text, convert_to_ascii=True) + text = re.sub(r"[^a-zA-Z0-9]", " ", text) + return text + + +def lemmatize(text: str) -> str: + lemmatized = lemmatize_text(text) + clean_lemmas = [re.sub(r"\d[\d.,]*", "NUM", lemma) for lemma in lemmatized] + return " ".join(clean_lemmas) diff --git a/examples/simple/predict_domain.py b/examples/simple/predict_domain.py index f48170b..f35bdc3 100644 --- a/examples/simple/predict_domain.py +++ b/examples/simple/predict_domain.py @@ -1,15 +1,10 @@ -from great_ai import configure, create_service, log_argument, log_metric, use_model - -configure(development_mode_override=True) - -import re from typing import Dict, Iterable, List -from preprocess import preprocess +from great_ai import GreatAI, use_model from pydantic import BaseModel from sklearn.pipeline import Pipeline -from great_ai.utilities.clean import clean +from helpers import lemmatize, preprocess class DomainPrediction(BaseModel): @@ -18,26 +13,22 @@ class DomainPrediction(BaseModel): explanation: List[str] -@create_service +@GreatAI.deploy() @use_model("small-domain-prediction-v2", version="latest") -@log_argument("text", validator=lambda t: len(t) > 0) def predict_domain( text: str, model: Pipeline, cut_off_probability: float = 0.2 ) -> List[DomainPrediction]: - assert 0 <= cut_off_probability <= 1 - """ Predict the scientific domain of the input text. Return labels until their sum likelihood is larger than cut_off_probability. """ - log_metric("text_length", len(text)) + assert 0 <= cut_off_probability <= 1 - cleaned = clean(text, convert_to_ascii=True) - text = re.sub(r"[^a-zA-Z0-9]", " ", cleaned) + text = preprocess(text) feature_names = model.named_steps["vectorizer"].get_feature_names_out() - token_mapping = {preprocess(original): original for original in text.split(" ")} + token_mapping = {lemmatize(original): original for original in text.split(" ")} features = model.named_steps["vectorizer"].transform( [" ".join(token_mapping.keys())] diff --git a/examples/simple/preprocess.py b/examples/simple/preprocess.py deleted file mode 100644 index 6440b14..0000000 --- a/examples/simple/preprocess.py +++ /dev/null @@ -1,8 +0,0 @@ -import re - -from great_ai.utilities.lemmatize_text import lemmatize_text - - -def preprocess(text: str) -> str: - lemmas = [re.sub(r"\d[\d.,]*", "NUM", lemma) for lemma in lemmatize_text(text)] - return " ".join(lemmas) diff --git a/examples/simple/train.ipynb b/examples/simple/train.ipynb new file mode 100644 index 0000000..65a0073 --- /dev/null +++ b/examples/simple/train.ipynb @@ -0,0 +1,269 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Train domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import matplotlib.pyplot as plt\n", + "from pathlib import Path\n", + "import pandas as pd\n", + "from sklearn import metrics, set_config\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.naive_bayes import MultinomialNB\n", + "from sklearn.pipeline import Pipeline\n", + "from sklearn.feature_extraction.text import TfidfVectorizer\n", + "from sklearn.model_selection import GridSearchCV\n", + "\n", + "from great_ai.utilities.parallel_map import parallel_map\n", + "from great_ai.utilities.language import is_english, predict_language\n", + "from great_ai import save_model, configure, LargeFile\n", + "\n", + "from helpers import preprocess, lemmatize" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Configuration" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "PREFIX = \"domain-\"\n", + "DATASET_KEY = \"data\"\n", + "MAX_FILE_COUNT = 5\n", + "MODEL_KEY = \"small-domain-prediction-v2\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "configure()\n", + "corpus_path = LargeFile(DATASET_KEY).get()\n", + "\n", + "set_config(display=\"diagram\")\n", + "plt.rcParams[\"figure.figsize\"] = (30, 15)\n", + "plt.rcParams[\"figure.facecolor\"] = \"white\"\n", + "plt.rcParams[\"font.size\"] = 12\n", + "plt.rcParams[\"axes.xmargin\"] = 0" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Preprocessing" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def clean_file(p: Path) -> None:\n", + " try:\n", + " processed_path = p.with_name(f\"{PREFIX}{p.stem}{p.suffix}\")\n", + "\n", + " if processed_path.exists():\n", + " return\n", + "\n", + " with open(p) as f:\n", + " content = json.load(f)\n", + "\n", + " result = {\n", + " lemmatize(preprocess(f'{c[\"title\"]} {c[\"abstract\"]}')): c[\"domain\"]\n", + " for c in content\n", + " if (\n", + " c[\"domain\"]\n", + " and c[\"abstract\"]\n", + " and is_english(predict_language(c[\"abstract\"]))\n", + " )\n", + " }\n", + "\n", + " with open(processed_path, \"w\") as f:\n", + " json.dump(result, f)\n", + " except Exception as e:\n", + " print(f\"Error ({e}) processing {p}\")\n", + "\n", + "\n", + "parallel_map(\n", + " clean_file,\n", + " list(corpus_path.glob(\"s2-corpus-*.json\"))[:MAX_FILE_COUNT],\n", + " chunk_size=1,\n", + ")\n", + "None" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "corpora = list(corpus_path.glob(f\"{PREFIX}*.json\"))[:MAX_FILE_COUNT]\n", + "print(f\"Found {len(corpora)} files\")\n", + "\n", + "data = []\n", + "for p in corpora:\n", + " with open(p) as f:\n", + " data.extend(json.load(f).items())\n", + "\n", + "print(f\"Found {len(data)} documents\")\n", + "\n", + "X_train, X_test, y_train, y_test = train_test_split(\n", + " [d[0] for d in data], [d[1] for d in data], test_size=0.1, random_state=1\n", + ")\n", + "\n", + "X_train = [x for x, y in zip(X_train, y_train) for domain in y]\n", + "y_train = [domain for x, y in zip(X_train, y_train) for domain in y]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Optimise and train Multinomial Naive Bayes classifier" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def create_pipeline() -> Pipeline:\n", + " return Pipeline(\n", + " steps=[\n", + " (\"vectorizer\", TfidfVectorizer()),\n", + " (\"classifier\", MultinomialNB()),\n", + " ]\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "optimisation_pipeline = GridSearchCV(\n", + " create_pipeline(),\n", + " {\n", + " \"vectorizer__max_df\": [0.05, 0.1],\n", + " \"vectorizer__min_df\": [5, 20],\n", + " \"vectorizer__sublinear_tf\": [True, False],\n", + " \"classifier__alpha\": [0.5, 1],\n", + " \"classifier__fit_prior\": [True, False],\n", + " },\n", + " scoring=\"f1_macro\",\n", + " cv=3,\n", + " n_jobs=8,\n", + " verbose=1,\n", + ")\n", + "optimisation_pipeline.fit(X_train, y_train)\n", + "\n", + "results = pd.DataFrame(optimisation_pipeline.cv_results_)\n", + "results.sort_values(\"rank_test_score\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "classifier = create_pipeline()\n", + "classifier.set_params(**optimisation_pipeline.best_params_)\n", + "classifier.fit(X_train, y_train)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Check accuracy on the test split" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "predicted = classifier.predict(X_test)\n", + "\n", + "y_test_aligned = [p if p in y else y[0] for p, y in zip(predicted, y_test)]\n", + "\n", + "print(metrics.classification_report(y_test_aligned, predicted))\n", + "metrics.ConfusionMatrixDisplay.from_predictions(\n", + " y_true=y_test_aligned,\n", + " y_pred=predicted,\n", + " xticks_rotation=\"vertical\",\n", + " normalize=\"pred\",\n", + " values_format=\".2f\",\n", + ")\n", + "None" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Export the model using GreatAI" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "save_model(classifier, key=MODEL_KEY, keep_last_n=5)" + ] + } + ], + "metadata": { + "interpreter": { + "hash": "acc70e949538f42041ccc57bc4df2261507e3fd7d6b9ce5dcc28e3bcf9d48274" + }, + "kernelspec": { + "display_name": "Python 3.8.5 ('.env': venv)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +}