diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 6e1f719..e458c9c 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -4,9 +4,9 @@ { "label": "Format and lint Python", "type": "shell", - "command": "source .env/bin/activate && scripts/format-python.sh great_ai && scripts/format-python.sh examples", + "command": "source .env/bin/activate && scripts/format-python.sh great_ai && scripts/format-python.sh examples/simple && scripts/format-python.sh examples/complex", "windows": { - "command": ".env/bin/activate.bat; scripts/format-python.sh great_ai; scripts\\format-python.sh examples" + "command": ".env\\bin\\activate.bat; scripts\\format-python.sh great_ai; scripts\\format-python.sh examples\\simple; scripts\\format-python.sh examples\\complex" }, "group": "test", "presentation": { @@ -22,7 +22,7 @@ "type": "shell", "command": "source .env/bin/activate && python3 -m pytest great_ai", "windows": { - "command": ".env/bin/activate.bat; python3 -m pytest great_ai" + "command": ".env\\bin\\activate.bat; python3 -m pytest great_ai" }, "group": "test", "presentation": { diff --git a/examples/complex/Dockerfile b/examples/complex/Dockerfile new file mode 100644 index 0000000..21aa538 --- /dev/null +++ b/examples/complex/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.8.12-slim-bullseye + +ENV ENVIRONMENT production + +WORKDIR /app + +RUN apt-get update &&\ + apt-get install curl -y &&\ + rm -rf /var/lib/apt/lists/* + +RUN pip install --no-cache-dir en-core-web-sm@https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl +COPY sus sus +RUN pip install --no-cache-dir --use-feature=in-tree-build ./sus + +COPY requirements.txt ./ +RUN pip install --no-cache-dir --requirement requirements.txt + +COPY . . +RUN rm -rf sus secrets + +EXPOSE 5000 + +HEALTHCHECK --interval=60s --timeout=60s --start-period=90s --retries=5 CMD [ "curl", "--fail", "http://localhost:5000/health" ] + +CMD ["uvicorn", "main:app", "--proxy-headers", "--backlog", "8196", "--timeout-keep-alive", "900", "--host", "0.0.0.0", "--port", "5000"] diff --git a/examples/complex/Dockerfile.development b/examples/complex/Dockerfile.development new file mode 100644 index 0000000..ff9cdff --- /dev/null +++ b/examples/complex/Dockerfile.development @@ -0,0 +1,22 @@ +FROM python:3.8.12-slim-bullseye + +ENV ENVIRONMENT development + +VOLUME /app +WORKDIR /app + +RUN apt-get update &&\ + apt-get install build-essential -y &&\ + rm -rf /var/lib/apt/lists/* + +RUN python3 -m pip install --no-cache-dir --upgrade pip &&\ + python3 -m pip install --no-cache-dir en-core-web-sm@https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl + +COPY requirements.txt ./ +RUN python3 -m pip install --no-cache-dir -r requirements.txt + +EXPOSE 5000 + +VOLUME /dependencies + +CMD ["/bin/bash", "-c", "python3 -m pip install -e /dependencies/sus && python3 -m uvicorn --reload --host 0.0.0.0 --port 5000 main:app"] diff --git a/examples/README.md b/examples/complex/README.md similarity index 100% rename from examples/README.md rename to examples/complex/README.md diff --git a/examples/main_batch.py b/examples/complex/main_batch.py similarity index 100% rename from examples/main_batch.py rename to examples/complex/main_batch.py diff --git a/examples/main.py b/examples/complex/main_service.py similarity index 65% rename from examples/main.py rename to examples/complex/main_service.py index 20d5537..3dcc030 100755 --- a/examples/main.py +++ b/examples/complex/main_service.py @@ -1,8 +1,6 @@ -#!/usr/bin/env python3 - from great_ai import configure, create_service -# configure(development_mode_override=True) +configure(development_mode_override=True) from predict_domain import predict_domain diff --git a/examples/predict_domain.py b/examples/complex/predict_domain.py similarity index 98% rename from examples/predict_domain.py rename to examples/complex/predict_domain.py index 7c29a37..04c0260 100644 --- a/examples/predict_domain.py +++ b/examples/complex/predict_domain.py @@ -1,5 +1,5 @@ import re -from typing import Dict, Iterable, List +from typing import Dict, Iterable, List from preprocess import preprocess from pydantic import BaseModel diff --git a/examples/preprocess.py b/examples/complex/preprocess.py similarity index 100% rename from examples/preprocess.py rename to examples/complex/preprocess.py diff --git a/examples/simple/predict_domain.py b/examples/simple/predict_domain.py new file mode 100644 index 0000000..f48170b --- /dev/null +++ b/examples/simple/predict_domain.py @@ -0,0 +1,86 @@ +from great_ai import configure, create_service, log_argument, log_metric, use_model + +configure(development_mode_override=True) + +import re +from typing import Dict, Iterable, List + +from preprocess import preprocess +from pydantic import BaseModel +from sklearn.pipeline import Pipeline + +from great_ai.utilities.clean import clean + + +class DomainPrediction(BaseModel): + domain: str + probability: float + explanation: List[str] + + +@create_service +@use_model("small-domain-prediction-v2", version="latest") +@log_argument("text", validator=lambda t: len(t) > 0) +def predict_domain( + text: str, model: Pipeline, cut_off_probability: float = 0.2 +) -> List[DomainPrediction]: + assert 0 <= cut_off_probability <= 1 + + """ + Predict the scientific domain of the input text. + Return labels until their sum likelihood is larger than cut_off_probability. + """ + log_metric("text_length", len(text)) + + cleaned = clean(text, convert_to_ascii=True) + text = re.sub(r"[^a-zA-Z0-9]", " ", cleaned) + + feature_names = model.named_steps["vectorizer"].get_feature_names_out() + + token_mapping = {preprocess(original): original for original in text.split(" ")} + + features = model.named_steps["vectorizer"].transform( + [" ".join(token_mapping.keys())] + ) + prediction = model.named_steps["classifier"].predict_proba(features)[0] + best_classes = sorted(enumerate(prediction), key=lambda v: v[1], reverse=True) + + results: List[DomainPrediction] = [] + for class_index, probability in best_classes: + weights = model.named_steps["classifier"].feature_log_prob_[class_index] + domain = model.named_steps["classifier"].classes_[class_index] + + results.append( + DomainPrediction( + domain=domain, + probability=round(probability * 100), + explanation=_get_explanation( + feature_names=feature_names, + features=features.A[0], + weights=weights, + token_mapping=token_mapping, + ), + ) + ) + + if sum(r.probability for r in results) >= cut_off_probability * 100: + break + + return results + + +def _get_explanation( + feature_names: Iterable[str], + features: Iterable[float], + weights: Iterable[float], + token_mapping: Dict[str, str], +) -> List[str]: + influential = [ + (weight, name) + for weight, value, name in zip(weights, features, feature_names) + if value + ] + + most_influential = sorted(influential, reverse=True)[:5] + + return [token_mapping[name] for _, name in most_influential] diff --git a/examples/simple/preprocess.py b/examples/simple/preprocess.py new file mode 100644 index 0000000..6440b14 --- /dev/null +++ b/examples/simple/preprocess.py @@ -0,0 +1,8 @@ +import re + +from great_ai.utilities.lemmatize_text import lemmatize_text + + +def preprocess(text: str) -> str: + lemmas = [re.sub(r"\d[\d.,]*", "NUM", lemma) for lemma in lemmatize_text(text)] + return " ".join(lemmas) diff --git a/examples/train.ipynb b/examples/train.ipynb index 77a093f..da036f1 100644 --- a/examples/train.ipynb +++ b/examples/train.ipynb @@ -1,248 +1,258 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Train Domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "import matplotlib.pyplot as plt\n", - "from pathlib import Path\n", - "import pandas as pd\n", - "from sklearn import metrics, set_config\n", - "from sklearn.model_selection import train_test_split\n", - "from sklearn.naive_bayes import MultinomialNB\n", - "from sklearn.pipeline import Pipeline\n", - "from sklearn.feature_extraction.text import TfidfVectorizer\n", - "from sklearn.model_selection import GridSearchCV\n", - "\n", - "from great_ai.utilities.clean import clean\n", - "from great_ai.utilities.parallel_map import parallel_map\n", - "from great_ai.utilities.language import is_english, predict_language\n", - "from great_ai import save_model, configure, LargeFile\n", - "\n", - "from preprocess import preprocess" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Configuration" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "PREFIX = \"domain-\"\n", - "DATASET_KEY = \"data\"\n", - "MAX_FILE_COUNT = 5\n", - "MODEL_KEY = \"small-domain-prediction-v2\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "configure()\n", - "corpus_path = LargeFile(DATASET_KEY).get()\n", - "\n", - "set_config(display=\"diagram\")\n", - "plt.rcParams[\"figure.figsize\"] = (30, 15)\n", - "plt.rcParams[\"figure.facecolor\"] = \"white\"\n", - "plt.rcParams[\"font.size\"] = 12\n", - "plt.rcParams[\"axes.xmargin\"] = 0" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Preprocessing" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def clean_file(p: Path) -> None:\n", - " try:\n", - " processed_path = p.with_name(f\"{PREFIX}{p.stem}{p.suffix}\")\n", - "\n", - " if processed_path.exists():\n", - " return\n", - "\n", - " with open(p) as f:\n", - " content = json.load(f)\n", - "\n", - " result = {\n", - " preprocess(\n", - " clean(f'{c[\"title\"]} {c[\"abstract\"]}', convert_to_ascii=True)\n", - " ): c[\"domain\"]\n", - " for c in content\n", - " if (\n", - " c[\"domain\"]\n", - " and c[\"abstract\"]\n", - " and is_english(predict_language(c[\"abstract\"]))\n", - " )\n", - " }\n", - "\n", - " with open(processed_path, \"w\") as f:\n", - " json.dump(result, f)\n", - " except Exception as e:\n", - " print(f\"Error ({e}) processing {p}\")\n", - "\n", - "\n", - "parallel_map(\n", - " clean_file,\n", - " list(corpus_path.glob(\"s2-corpus-*.json\"))[:MAX_FILE_COUNT],\n", - " chunk_size=1,\n", - ")\n", - "None" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "corpora = list(corpus_path.glob(f\"{PREFIX}*.json\"))[:MAX_FILE_COUNT]\n", - "print(f\"Found {len(corpora)} files\")\n", - "\n", - "data = []\n", - "for p in corpora:\n", - " with open(p) as f:\n", - " data.extend(json.load(f).items())\n", - "\n", - "print(f\"Found {len(data)} documents\")\n", - "\n", - "X_train, X_test, y_train, y_test = train_test_split(\n", - " [d[0] for d in data], [d[1] for d in data], test_size=0.1, random_state=1\n", - ")\n", - "\n", - "X_train = [x for x, y in zip(X_train, y_train) for domain in y]\n", - "y_train = [domain for x, y in zip(X_train, y_train) for domain in y]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Naive Bayes" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "classifier = GridSearchCV(\n", - " Pipeline(steps=[(\"vectorizer\", TfidfVectorizer(token_pattern=r\"[^ ]+\")), (\"classifier\", MultinomialNB())]),\n", - " {\n", - " \"vectorizer__max_df\": [0.05, 0.1, 0.3],\n", - " \"vectorizer__min_df\": [5, 10, 30],\n", - " \"vectorizer__sublinear_tf\": [True, False],\n", - " \"classifier__alpha\": [0.1, 0.25, 0.5, 0.75, 1],\n", - " \"classifier__fit_prior\": [True, False],\n", - " },\n", - " scoring=\"f1_macro\",\n", - " cv=3,\n", - " n_jobs=4,\n", - " verbose=1,\n", - ")\n", - "classifier.fit(X_train, y_train)\n", - "\n", - "results = pd.DataFrame(classifier.cv_results_)\n", - "results.sort_values(\"rank_test_score\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "classifier = Pipeline(\n", - " steps=[\n", - " (\"vectorizer\", TfidfVectorizer(min_df=10, max_df=0.05, sublinear_tf=True, token_pattern=r\"[^ ]+\")),\n", - " (\"classifier\", MultinomialNB(alpha=0.5, fit_prior=False)),\n", - " ]\n", - ")\n", - "\n", - "classifier.fit(X_train, y_train)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "predicted = classifier.predict(X_test)\n", - "\n", - "y_test_aligned = [p if p in y else y[0] for p, y in zip(predicted, y_test)]\n", - "\n", - "print(metrics.classification_report(y_test_aligned, predicted))\n", - "metrics.ConfusionMatrixDisplay.from_predictions(\n", - " y_true=y_test_aligned,\n", - " y_pred=predicted,\n", - " xticks_rotation=\"vertical\",\n", - " normalize=\"pred\",\n", - " values_format=\".2f\",\n", - ")\n", - "None" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "save_model(classifier, key=MODEL_KEY, keep_last_n=1)" - ] - } - ], - "metadata": { - "interpreter": { - "hash": "acc70e949538f42041ccc57bc4df2261507e3fd7d6b9ce5dcc28e3bcf9d48274" - }, - "kernelspec": { - "display_name": "Python 3.8.5 ('.env': venv)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.5" - }, - "orig_nbformat": 4 + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Train Domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)" + ] }, - "nbformat": 4, - "nbformat_minor": 2 + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import matplotlib.pyplot as plt\n", + "from pathlib import Path\n", + "import pandas as pd\n", + "from sklearn import metrics, set_config\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.naive_bayes import MultinomialNB\n", + "from sklearn.pipeline import Pipeline\n", + "from sklearn.feature_extraction.text import TfidfVectorizer\n", + "from sklearn.model_selection import GridSearchCV\n", + "\n", + "from great_ai.utilities.clean import clean\n", + "from great_ai.utilities.parallel_map import parallel_map\n", + "from great_ai.utilities.language import is_english, predict_language\n", + "from great_ai import save_model, configure, LargeFile\n", + "\n", + "from preprocess import preprocess" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Configuration" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "PREFIX = \"domain-\"\n", + "DATASET_KEY = \"data\"\n", + "MAX_FILE_COUNT = 5\n", + "MODEL_KEY = \"small-domain-prediction-v2\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "configure()\n", + "corpus_path = LargeFile(DATASET_KEY).get()\n", + "\n", + "set_config(display=\"diagram\")\n", + "plt.rcParams[\"figure.figsize\"] = (30, 15)\n", + "plt.rcParams[\"figure.facecolor\"] = \"white\"\n", + "plt.rcParams[\"font.size\"] = 12\n", + "plt.rcParams[\"axes.xmargin\"] = 0" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Preprocessing" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def clean_file(p: Path) -> None:\n", + " try:\n", + " processed_path = p.with_name(f\"{PREFIX}{p.stem}{p.suffix}\")\n", + "\n", + " if processed_path.exists():\n", + " return\n", + "\n", + " with open(p) as f:\n", + " content = json.load(f)\n", + "\n", + " result = {\n", + " preprocess(\n", + " clean(f'{c[\"title\"]} {c[\"abstract\"]}', convert_to_ascii=True)\n", + " ): c[\"domain\"]\n", + " for c in content\n", + " if (\n", + " c[\"domain\"]\n", + " and c[\"abstract\"]\n", + " and is_english(predict_language(c[\"abstract\"]))\n", + " )\n", + " }\n", + "\n", + " with open(processed_path, \"w\") as f:\n", + " json.dump(result, f)\n", + " except Exception as e:\n", + " print(f\"Error ({e}) processing {p}\")\n", + "\n", + "\n", + "parallel_map(\n", + " clean_file,\n", + " list(corpus_path.glob(\"s2-corpus-*.json\"))[:MAX_FILE_COUNT],\n", + " chunk_size=1,\n", + ")\n", + "None" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "corpora = list(corpus_path.glob(f\"{PREFIX}*.json\"))[:MAX_FILE_COUNT]\n", + "print(f\"Found {len(corpora)} files\")\n", + "\n", + "data = []\n", + "for p in corpora:\n", + " with open(p) as f:\n", + " data.extend(json.load(f).items())\n", + "\n", + "print(f\"Found {len(data)} documents\")\n", + "\n", + "X_train, X_test, y_train, y_test = train_test_split(\n", + " [d[0] for d in data], [d[1] for d in data], test_size=0.1, random_state=1\n", + ")\n", + "\n", + "X_train = [x for x, y in zip(X_train, y_train) for domain in y]\n", + "y_train = [domain for x, y in zip(X_train, y_train) for domain in y]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Naive Bayes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "classifier = GridSearchCV(\n", + " Pipeline(\n", + " steps=[\n", + " (\"vectorizer\", TfidfVectorizer(token_pattern=r\"[^ ]+\")),\n", + " (\"classifier\", MultinomialNB()),\n", + " ]\n", + " ),\n", + " {\n", + " \"vectorizer__max_df\": [0.05, 0.1, 0.3],\n", + " \"vectorizer__min_df\": [5, 10, 30],\n", + " \"vectorizer__sublinear_tf\": [True, False],\n", + " \"classifier__alpha\": [0.1, 0.25, 0.5, 0.75, 1],\n", + " \"classifier__fit_prior\": [True, False],\n", + " },\n", + " scoring=\"f1_macro\",\n", + " cv=3,\n", + " n_jobs=4,\n", + " verbose=1,\n", + ")\n", + "classifier.fit(X_train, y_train)\n", + "\n", + "results = pd.DataFrame(classifier.cv_results_)\n", + "results.sort_values(\"rank_test_score\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "classifier = Pipeline(\n", + " steps=[\n", + " (\n", + " \"vectorizer\",\n", + " TfidfVectorizer(\n", + " min_df=10, max_df=0.05, sublinear_tf=True, token_pattern=r\"[^ ]+\"\n", + " ),\n", + " ),\n", + " (\"classifier\", MultinomialNB(alpha=0.5, fit_prior=False)),\n", + " ]\n", + ")\n", + "\n", + "classifier.fit(X_train, y_train)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "predicted = classifier.predict(X_test)\n", + "\n", + "y_test_aligned = [p if p in y else y[0] for p, y in zip(predicted, y_test)]\n", + "\n", + "print(metrics.classification_report(y_test_aligned, predicted))\n", + "metrics.ConfusionMatrixDisplay.from_predictions(\n", + " y_true=y_test_aligned,\n", + " y_pred=predicted,\n", + " xticks_rotation=\"vertical\",\n", + " normalize=\"pred\",\n", + " values_format=\".2f\",\n", + ")\n", + "None" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "save_model(classifier, key=MODEL_KEY, keep_last_n=1)" + ] + } + ], + "metadata": { + "interpreter": { + "hash": "acc70e949538f42041ccc57bc4df2261507e3fd7d6b9ce5dcc28e3bcf9d48274" + }, + "kernelspec": { + "display_name": "Python 3.8.5 ('.env': venv)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 }