Update notebooks
This commit is contained in:
parent
0f9040b405
commit
266e76d4f4
4 changed files with 1679919 additions and 347 deletions
|
|
@ -46,7 +46,6 @@
|
|||
"\n",
|
||||
"DATASET_KEY = \"semantic-scholar-dataset-small\"\n",
|
||||
"MAX_FILE_COUNT = 12\n",
|
||||
"\n",
|
||||
"configure()"
|
||||
]
|
||||
},
|
||||
|
|
|
|||
170
examples/simple/deploy.ipynb
Normal file
170
examples/simple/deploy.ipynb
Normal file
|
|
@ -0,0 +1,170 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Train a domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)\n",
|
||||
"\n",
|
||||
"## Part 3: Create production inference function\n",
|
||||
"\n",
|
||||
"In the [previous notebook](train.ipynb), we trained our AI model. Now, it's time to create **G**eneral **R**obust **E**nd-to-end **A**utomated **T**rustworthy deployment from it using the `GreatAI` Python package."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from typing import List\n",
|
||||
"import re\n",
|
||||
"from great_ai import ClassificationOutput, GreatAI, use_model\n",
|
||||
"from sklearn.pipeline import Pipeline\n",
|
||||
"from great_ai.utilities.clean import clean"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\u001b[38;5;226m2022-05-28 15:02:20,852 | WARNING | Environment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-28 15:02:20,853 | INFO | Options: configured ✅\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-28 15:02:21,168 | INFO | Latest version of small-domain-prediction-v2 is 8 (from versions: 3, 4, 5, 6, 7, 8)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-28 15:02:21,169 | INFO | File small-domain-prediction-v2-8 found in cache\u001b[0m\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"@GreatAI.deploy\n",
|
||||
"@use_model(\"small-domain-prediction-v2\", version=\"latest\")\n",
|
||||
"def predict_domain(\n",
|
||||
" text: str, model: Pipeline, target_confidence: int = 20\n",
|
||||
") -> List[ClassificationOutput]:\n",
|
||||
" \"\"\"\n",
|
||||
" Predict the scientific domain of the input text.\n",
|
||||
" Return labels until their sum likelihood is larger than target_confidence.\n",
|
||||
" \"\"\"\n",
|
||||
" assert 0 <= target_confidence <= 100, \"invalid argument\"\n",
|
||||
"\n",
|
||||
" preprocessed = re.sub(r\"[^a-zA-Z ]\", \"\", clean(text, convert_to_ascii=True))\n",
|
||||
" features = model.named_steps[\"vectorizer\"].transform([preprocessed])\n",
|
||||
" prediction = model.named_steps[\"classifier\"].predict_proba(features)[0]\n",
|
||||
"\n",
|
||||
" best_classes = sorted(enumerate(prediction), key=lambda v: v[1], reverse=True)\n",
|
||||
"\n",
|
||||
" results: List[ClassificationOutput] = []\n",
|
||||
" for class_index, probability in best_classes:\n",
|
||||
" results.append(\n",
|
||||
" ClassificationOutput(\n",
|
||||
" label=model.named_steps[\"classifier\"].classes_[class_index],\n",
|
||||
" confidence=round(probability * 100),\n",
|
||||
" explanation=[\n",
|
||||
" word\n",
|
||||
" for _, word in sorted(\n",
|
||||
" (\n",
|
||||
" (weight, word)\n",
|
||||
" for weight, word, count in zip(\n",
|
||||
" model.named_steps[\"classifier\"].feature_log_prob_[\n",
|
||||
" class_index\n",
|
||||
" ],\n",
|
||||
" model.named_steps[\"vectorizer\"].get_feature_names_out(),\n",
|
||||
" features.A[0],\n",
|
||||
" )\n",
|
||||
" if count > 0\n",
|
||||
" ),\n",
|
||||
" reverse=True,\n",
|
||||
" )\n",
|
||||
" ][:5],\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" if sum(r.confidence for r in results) >= target_confidence:\n",
|
||||
" break\n",
|
||||
"\n",
|
||||
" return results"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"{'created': '2022-05-28T13:02:22.035886',\n",
|
||||
" 'evaluation': None,\n",
|
||||
" 'evaluation_id': 'c035d78e-dea9-415f-8f93-0dc4cd8dd7b5',\n",
|
||||
" 'exception': None,\n",
|
||||
" 'execution_time_ms': 94.814,\n",
|
||||
" 'logged_values': {'arg:predict_domain:target_confidence': 20,\n",
|
||||
" 'arg:predict_domain:text': '\\n'\n",
|
||||
" ' State-of-the-art methods for zero-shot visual recognition formulate '\n",
|
||||
" 'learning as a joint embedding problem of images and side information. '\n",
|
||||
" 'In these formulations the current best complement to visual features '\n",
|
||||
" 'are attributes: manually encoded vectors describing shared '\n",
|
||||
" 'characteristics among categories. Despite good performance, attributes '\n",
|
||||
" 'have limitations: (1) finer-grained recognition requires commensurately '\n",
|
||||
" 'more, and (2) attributes do not provide a natural language interface. '\n",
|
||||
" 'We propose to overcome these limitations by training neural language '\n",
|
||||
" 'models from scratch; i.e. without pre-training and only consuming words '\n",
|
||||
" 'and characters. Our proposed models train end-to-end to align with the '\n",
|
||||
" 'fine-grained and category-specific content of images. Natural language '\n",
|
||||
" 'provides a flexible and compact way of encoding only the salient visual '\n",
|
||||
" 'aspects for distinguishing categories. By training on raw text, our '\n",
|
||||
" 'model can do inference on raw text as well, providing humans a familiar '\n",
|
||||
" 'mode both for annotation and retrieval. Our model achieves strong '\n",
|
||||
" 'performance on zero-shot text-based image retrieval and significantly '\n",
|
||||
" 'outperforms the attribute-based state-of-the-art for zero-shot '\n",
|
||||
" 'classification on the CaltechUCSD Birds 200-2011 dataset. ',\n",
|
||||
" 'arg:predict_domain:text:length': 1236},\n",
|
||||
" 'models': [{'key': 'small-domain-prediction-v2', 'version': 8}],\n",
|
||||
" 'output': [{'confidence': 99.0,\n",
|
||||
" 'explanation': ['information', 'model', 'learning', 'proposed', 'image'],\n",
|
||||
" 'label': 'Computer Science'}]}\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"result = predict_domain(\"\"\"\n",
|
||||
" State-of-the-art methods for zero-shot visual recognition formulate learning as a joint embedding problem of images and side information. In these formulations the current best complement to visual features are attributes: manually encoded vectors describing shared characteristics among categories. Despite good performance, attributes have limitations: (1) finer-grained recognition requires commensurately more, and (2) attributes do not provide a natural language interface. We propose to overcome these limitations by training neural language models from scratch; i.e. without pre-training and only consuming words and characters. Our proposed models train end-to-end to align with the fine-grained and category-specific content of images. Natural language provides a flexible and compact way of encoding only the salient visual aspects for distinguishing categories. By training on raw text, our model can do inference on raw text as well, providing humans a familiar mode both for annotation and retrieval. Our model achieves strong performance on zero-shot text-based image retrieval and significantly outperforms the attribute-based state-of-the-art for zero-shot classification on the CaltechUCSD Birds 200-2011 dataset. \"\"\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"from pprint import pprint\n",
|
||||
"pprint(result.dict(), width=120)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"interpreter": {
|
||||
"hash": "9583b8c15346c7ad861da481b60c8e8605a71a48eda767f1640baa8f25efebcb"
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3.9.2 ('.venv': venv)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.9.2"
|
||||
},
|
||||
"orig_nbformat": 4
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
1679835
examples/simple/train.ipynb
1679835
examples/simple/train.ipynb
File diff suppressed because one or more lines are too long
|
|
@ -1,258 +0,0 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Train Domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import json\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"from pathlib import Path\n",
|
||||
"import pandas as pd\n",
|
||||
"from sklearn import metrics, set_config\n",
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"from sklearn.naive_bayes import MultinomialNB\n",
|
||||
"from sklearn.pipeline import Pipeline\n",
|
||||
"from sklearn.feature_extraction.text import TfidfVectorizer\n",
|
||||
"from sklearn.model_selection import GridSearchCV\n",
|
||||
"\n",
|
||||
"from great_ai.utilities.clean import clean\n",
|
||||
"from great_ai.utilities.parallel_map import parallel_map\n",
|
||||
"from great_ai.utilities.language import is_english, predict_language\n",
|
||||
"from great_ai import save_model, configure, LargeFile\n",
|
||||
"\n",
|
||||
"from preprocess import preprocess"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Configuration"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"PREFIX = \"domain-\"\n",
|
||||
"DATASET_KEY = \"data\"\n",
|
||||
"MAX_FILE_COUNT = 5\n",
|
||||
"MODEL_KEY = \"small-domain-prediction-v2\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"configure()\n",
|
||||
"corpus_path = LargeFile(DATASET_KEY).get()\n",
|
||||
"\n",
|
||||
"set_config(display=\"diagram\")\n",
|
||||
"plt.rcParams[\"figure.figsize\"] = (30, 15)\n",
|
||||
"plt.rcParams[\"figure.facecolor\"] = \"white\"\n",
|
||||
"plt.rcParams[\"font.size\"] = 12\n",
|
||||
"plt.rcParams[\"axes.xmargin\"] = 0"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Preprocessing"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def clean_file(p: Path) -> None:\n",
|
||||
" try:\n",
|
||||
" processed_path = p.with_name(f\"{PREFIX}{p.stem}{p.suffix}\")\n",
|
||||
"\n",
|
||||
" if processed_path.exists():\n",
|
||||
" return\n",
|
||||
"\n",
|
||||
" with open(p) as f:\n",
|
||||
" content = json.load(f)\n",
|
||||
"\n",
|
||||
" result = {\n",
|
||||
" preprocess(\n",
|
||||
" clean(f'{c[\"title\"]} {c[\"abstract\"]}', convert_to_ascii=True)\n",
|
||||
" ): c[\"domain\"]\n",
|
||||
" for c in content\n",
|
||||
" if (\n",
|
||||
" c[\"domain\"]\n",
|
||||
" and c[\"abstract\"]\n",
|
||||
" and is_english(predict_language(c[\"abstract\"]))\n",
|
||||
" )\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" with open(processed_path, \"w\") as f:\n",
|
||||
" json.dump(result, f)\n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"Error ({e}) processing {p}\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"parallel_map(\n",
|
||||
" clean_file,\n",
|
||||
" list(corpus_path.glob(\"s2-corpus-*.json\"))[:MAX_FILE_COUNT],\n",
|
||||
" chunk_size=1,\n",
|
||||
")\n",
|
||||
"None"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"corpora = list(corpus_path.glob(f\"{PREFIX}*.json\"))[:MAX_FILE_COUNT]\n",
|
||||
"print(f\"Found {len(corpora)} files\")\n",
|
||||
"\n",
|
||||
"data = []\n",
|
||||
"for p in corpora:\n",
|
||||
" with open(p) as f:\n",
|
||||
" data.extend(json.load(f).items())\n",
|
||||
"\n",
|
||||
"print(f\"Found {len(data)} documents\")\n",
|
||||
"\n",
|
||||
"X_train, X_test, y_train, y_test = train_test_split(\n",
|
||||
" [d[0] for d in data], [d[1] for d in data], test_size=0.1, random_state=1\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"X_train = [x for x, y in zip(X_train, y_train) for domain in y]\n",
|
||||
"y_train = [domain for x, y in zip(X_train, y_train) for domain in y]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Naive Bayes"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"classifier = GridSearchCV(\n",
|
||||
" Pipeline(\n",
|
||||
" steps=[\n",
|
||||
" (\"vectorizer\", TfidfVectorizer(token_pattern=r\"[^ ]+\")),\n",
|
||||
" (\"classifier\", MultinomialNB()),\n",
|
||||
" ]\n",
|
||||
" ),\n",
|
||||
" {\n",
|
||||
" \"vectorizer__max_df\": [0.05, 0.1, 0.3],\n",
|
||||
" \"vectorizer__min_df\": [5, 10, 30],\n",
|
||||
" \"vectorizer__sublinear_tf\": [True, False],\n",
|
||||
" \"classifier__alpha\": [0.1, 0.25, 0.5, 0.75, 1],\n",
|
||||
" \"classifier__fit_prior\": [True, False],\n",
|
||||
" },\n",
|
||||
" scoring=\"f1_macro\",\n",
|
||||
" cv=3,\n",
|
||||
" n_jobs=4,\n",
|
||||
" verbose=1,\n",
|
||||
")\n",
|
||||
"classifier.fit(X_train, y_train)\n",
|
||||
"\n",
|
||||
"results = pd.DataFrame(classifier.cv_results_)\n",
|
||||
"results.sort_values(\"rank_test_score\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"classifier = Pipeline(\n",
|
||||
" steps=[\n",
|
||||
" (\n",
|
||||
" \"vectorizer\",\n",
|
||||
" TfidfVectorizer(\n",
|
||||
" min_df=10, max_df=0.05, sublinear_tf=True, token_pattern=r\"[^ ]+\"\n",
|
||||
" ),\n",
|
||||
" ),\n",
|
||||
" (\"classifier\", MultinomialNB(alpha=0.5, fit_prior=False)),\n",
|
||||
" ]\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"classifier.fit(X_train, y_train)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"predicted = classifier.predict(X_test)\n",
|
||||
"\n",
|
||||
"y_test_aligned = [p if p in y else y[0] for p, y in zip(predicted, y_test)]\n",
|
||||
"\n",
|
||||
"print(metrics.classification_report(y_test_aligned, predicted))\n",
|
||||
"metrics.ConfusionMatrixDisplay.from_predictions(\n",
|
||||
" y_true=y_test_aligned,\n",
|
||||
" y_pred=predicted,\n",
|
||||
" xticks_rotation=\"vertical\",\n",
|
||||
" normalize=\"pred\",\n",
|
||||
" values_format=\".2f\",\n",
|
||||
")\n",
|
||||
"None"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"save_model(classifier, key=MODEL_KEY, keep_last_n=1)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"interpreter": {
|
||||
"hash": "acc70e949538f42041ccc57bc4df2261507e3fd7d6b9ce5dcc28e3bcf9d48274"
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3.8.5 ('.env': venv)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.8.5"
|
||||
},
|
||||
"orig_nbformat": 4
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue