{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Train a domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)\n", "\n", "> Part 3: Create production inference function\n", "\n", "![position of this step in the lifecycle](diagrams/scope-deploy.svg)\n", "> The blue boxes show the steps implemented in this notebook.\n", "\n", "In [Part 2](train.ipynb), we trained our AI model. Now, it's time to create **G**eneral **R**obust **E**nd-to-end **A**utomated **T**rustworthy deployment from it using the `GreatAI` Python package." ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "\u001b[38;5;226m2022-06-19 15:16:23,856 | WARNING | Environment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n", "\u001b[38;5;226m2022-06-19 15:16:23,857 | WARNING | The selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n", "\u001b[38;5;39m2022-06-19 15:16:23,858 | INFO | Options: configured ✅\u001b[0m\n", "\u001b[38;5;39m2022-06-19 15:16:23,858 | INFO | Fetching cached versions of small-domain-prediction\u001b[0m\n", "\u001b[38;5;39m2022-06-19 15:16:23,859 | INFO | Latest version of small-domain-prediction is 12 (from versions: 9, 10, 11, 12)\u001b[0m\n", "\u001b[38;5;39m2022-06-19 15:16:23,860 | INFO | File small-domain-prediction-12 found in cache\u001b[0m\n" ] } ], "source": [ "import re\n", "from sklearn.pipeline import Pipeline\n", "from great_ai.utilities import clean\n", "from great_ai import (\n", " MultiLabelClassificationOutput,\n", " ClassificationOutput,\n", " GreatAI,\n", " use_model,\n", " parameter,\n", ")\n", "\n", "\n", "@GreatAI.deploy\n", "@use_model(\"small-domain-prediction\", version=\"latest\")\n", "@parameter(\"explanation_length\", disable_logging=True)\n", "def predict_domain(\n", " text: str, model: Pipeline, target_confidence: int = 50, explanation_length: int = 5\n", ") -> MultiLabelClassificationOutput:\n", " \"\"\"\n", " Predict the scientific domain of the input text.\n", " Return labels until their sum likelihood is larger than `target_confidence`.\n", " \"\"\"\n", " assert 0 <= target_confidence <= 100, \"invalid argument\"\n", "\n", " preprocessed = re.sub(r\"[^a-zA-Z\\s]\", \"\", clean(text, convert_to_ascii=True))\n", " features = model.named_steps[\"vectorizer\"].transform([preprocessed])\n", " prediction = model.named_steps[\"classifier\"].predict_proba(features)[0]\n", "\n", " best_classes = sorted(enumerate(prediction), key=lambda v: v[1], reverse=True)\n", "\n", " results = MultiLabelClassificationOutput()\n", " for class_index, probability in best_classes:\n", " results.labels.append(\n", " ClassificationOutput(\n", " label=model.named_steps[\"classifier\"].classes_[class_index],\n", " confidence=round(probability * 100),\n", " explanation=[\n", " word\n", " for _, word in sorted(\n", " (\n", " (weight, word)\n", " for weight, word, count in zip(\n", " model.named_steps[\"classifier\"].feature_log_prob_[\n", " class_index\n", " ],\n", " model.named_steps[\"vectorizer\"].get_feature_names_out(),\n", " features.A[0],\n", " )\n", " if count > 0\n", " ),\n", " reverse=True,\n", " )\n", " ][:explanation_length],\n", " )\n", " )\n", "\n", " if sum(r.confidence for r in results.labels) >= target_confidence:\n", " break\n", "\n", " return results" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Check accuracy on the test split" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "\u001b[38;5;226m2022-06-19 15:16:29,300 | WARNING | Limiting concurrency to 10 because there are only 10 chunks\u001b[0m\n", "\u001b[38;5;39m2022-06-19 15:16:29,301 | INFO | Starting parallel map (concurrency: 10, chunk size: 1)\u001b[0m\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "34e54eabd5f945b8aa2809907964eec8", "version_major": 2, "version_minor": 0 }, "text/plain": [ " 0%| | 0/10 [00:00" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "if __name__ == \"__main__\":\n", " from great_ai import query_ground_truth\n", " from sklearn import metrics\n", "\n", " data = query_ground_truth(\"test\", return_max_count=10)\n", "\n", " X = [d.input for d in data]\n", " y_actual = [d.feedback for d in data]\n", "\n", " y_predicted = [d.output.labels[0].label for d in predict_domain.process_batch(X)]\n", " y_actual_aligned = [p if p in a else a[0] for p, a in zip(y_predicted, y_actual)]\n", "\n", " import matplotlib.pyplot as plt\n", "\n", " # Configure matplotlib to have nice, high-resolution charts.\n", "\n", " %matplotlib inline\n", "\n", " plt.rcParams[\"figure.figsize\"] = (30, 15)\n", " plt.rcParams[\"figure.facecolor\"] = \"white\"\n", " plt.rcParams[\"font.size\"] = 12\n", " plt.rcParams[\"axes.xmargin\"] = 0\n", "\n", " print(metrics.classification_report(y_actual_aligned, y_predicted))\n", " metrics.ConfusionMatrixDisplay.from_predictions(\n", " y_true=y_actual_aligned,\n", " y_pred=y_predicted,\n", " xticks_rotation=\"vertical\",\n", " normalize=\"pred\",\n", " values_format=\".2f\",\n", " )" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3.10.4 ('.env': venv)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.4" }, "orig_nbformat": 4, "vscode": { "interpreter": { "hash": "02dd6d3afbfa9fbbe1037d64ad9014965528a1ccad21929d6e72f466389a68ad" } } }, "nbformat": 4, "nbformat_minor": 2 }