andras/great-ai
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases 4 Packages Wiki Activity Actions

Split examples

Browse source
This commit is contained in:
Andras Schmelczer 2022-05-25 22:54:21 +02:00
parent b4492da2e6
commit df1c9418ee
No known key found for this signature in database
GPG key ID: 39260B5B0614A13E
11 changed files with 401 additions and 252 deletions
Download patch file Download diff file Expand all files Collapse all files

6
.vscode/tasks.json vendored
View file

@ -4,9 +4,9 @@
{ {
"label": "Format and lint Python", "label": "Format and lint Python",
"type": "shell", "type": "shell",
"command": "source .env/bin/activate && scripts/format-python.sh great_ai && scripts/format-python.sh examples", "command": "source .env/bin/activate && scripts/format-python.sh great_ai && scripts/format-python.sh examples/simple && scripts/format-python.sh examples/complex",
"windows": { "windows": {
"command": ".env/bin/activate.bat; scripts/format-python.sh great_ai; scripts\\format-python.sh examples" "command": ".env\\bin\\activate.bat; scripts\\format-python.sh great_ai; scripts\\format-python.sh examples\\simple; scripts\\format-python.sh examples\\complex"
}, },
"group": "test", "group": "test",
"presentation": { "presentation": {
@ -22,7 +22,7 @@
"type": "shell", "type": "shell",
"command": "source .env/bin/activate && python3 -m pytest great_ai", "command": "source .env/bin/activate && python3 -m pytest great_ai",
"windows": { "windows": {
"command": ".env/bin/activate.bat; python3 -m pytest great_ai" "command": ".env\\bin\\activate.bat; python3 -m pytest great_ai"
}, },
"group": "test", "group": "test",
"presentation": { "presentation": {

25
examples/complex/Dockerfile Normal file
View file

@ -0,0 +1,25 @@
FROM python:3.8.12-slim-bullseye
ENV ENVIRONMENT production
WORKDIR /app
RUN apt-get update &&\
apt-get install curl -y &&\
rm -rf /var/lib/apt/lists/*
RUN pip install --no-cache-dir en-core-web-sm@https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl
COPY sus sus
RUN pip install --no-cache-dir --use-feature=in-tree-build ./sus
COPY requirements.txt ./
RUN pip install --no-cache-dir --requirement requirements.txt
COPY . .
RUN rm -rf sus secrets
EXPOSE 5000
HEALTHCHECK --interval=60s --timeout=60s --start-period=90s --retries=5 CMD [ "curl", "--fail", "http://localhost:5000/health" ]
CMD ["uvicorn", "main:app", "--proxy-headers", "--backlog", "8196", "--timeout-keep-alive", "900", "--host", "0.0.0.0", "--port", "5000"]

22
examples/complex/Dockerfile.development Normal file
View file

@ -0,0 +1,22 @@
FROM python:3.8.12-slim-bullseye
ENV ENVIRONMENT development
VOLUME /app
WORKDIR /app
RUN apt-get update &&\
apt-get install build-essential -y &&\
rm -rf /var/lib/apt/lists/*
RUN python3 -m pip install --no-cache-dir --upgrade pip &&\
python3 -m pip install --no-cache-dir en-core-web-sm@https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl
COPY requirements.txt ./
RUN python3 -m pip install --no-cache-dir -r requirements.txt
EXPOSE 5000
VOLUME /dependencies
CMD ["/bin/bash", "-c", "python3 -m pip install -e /dependencies/sus && python3 -m uvicorn --reload --host 0.0.0.0 --port 5000 main:app"]

0
examples/README.md → examples/complex/README.md
View file

0
examples/main_batch.py → examples/complex/main_batch.py
View file

4
examples/main.py → examples/complex/main_service.py
View file

@ -1,8 +1,6 @@
#!/usr/bin/env python3
from great_ai import configure, create_service from great_ai import configure, create_service
# configure(development_mode_override=True) configure(development_mode_override=True)
from predict_domain import predict_domain from predict_domain import predict_domain

2
examples/predict_domain.py → examples/complex/predict_domain.py
View file

@ -1,5 +1,5 @@
import re import re
from typing import Dict, Iterable, List from typing import Dict, Iterable, List
from preprocess import preprocess from preprocess import preprocess
from pydantic import BaseModel from pydantic import BaseModel

0
examples/preprocess.py → examples/complex/preprocess.py
View file

86
examples/simple/predict_domain.py Normal file
View file

@ -0,0 +1,86 @@
from great_ai import configure, create_service, log_argument, log_metric, use_model
configure(development_mode_override=True)
import re
from typing import Dict, Iterable, List
from preprocess import preprocess
from pydantic import BaseModel
from sklearn.pipeline import Pipeline
from great_ai.utilities.clean import clean
class DomainPrediction(BaseModel):
domain: str
probability: float
explanation: List[str]
@create_service
@use_model("small-domain-prediction-v2", version="latest")
@log_argument("text", validator=lambda t: len(t) > 0)
def predict_domain(
text: str, model: Pipeline, cut_off_probability: float = 0.2
) -> List[DomainPrediction]:
assert 0 <= cut_off_probability <= 1
"""
Predict the scientific domain of the input text.
Return labels until their sum likelihood is larger than cut_off_probability.
"""
log_metric("text_length", len(text))
cleaned = clean(text, convert_to_ascii=True)
text = re.sub(r"[^a-zA-Z0-9]", " ", cleaned)
feature_names = model.named_steps["vectorizer"].get_feature_names_out()
token_mapping = {preprocess(original): original for original in text.split(" ")}
features = model.named_steps["vectorizer"].transform(
[" ".join(token_mapping.keys())]
)
prediction = model.named_steps["classifier"].predict_proba(features)[0]
best_classes = sorted(enumerate(prediction), key=lambda v: v[1], reverse=True)
results: List[DomainPrediction] = []
for class_index, probability in best_classes:
weights = model.named_steps["classifier"].feature_log_prob_[class_index]
domain = model.named_steps["classifier"].classes_[class_index]
results.append(
DomainPrediction(
domain=domain,
probability=round(probability * 100),
explanation=_get_explanation(
feature_names=feature_names,
features=features.A[0],
weights=weights,
token_mapping=token_mapping,
),
)
)
if sum(r.probability for r in results) >= cut_off_probability * 100:
break
return results
def _get_explanation(
feature_names: Iterable[str],
features: Iterable[float],
weights: Iterable[float],
token_mapping: Dict[str, str],
) -> List[str]:
influential = [
(weight, name)
for weight, value, name in zip(weights, features, feature_names)
if value
]
most_influential = sorted(influential, reverse=True)[:5]
return [token_mapping[name] for _, name in most_influential]

8
examples/simple/preprocess.py Normal file
View file

@ -0,0 +1,8 @@
import re
from great_ai.utilities.lemmatize_text import lemmatize_text
def preprocess(text: str) -> str:
lemmas = [re.sub(r"\d[\d.,]*", "NUM", lemma) for lemma in lemmatize_text(text)]
return " ".join(lemmas)

500
examples/train.ipynb
View file

@ -1,248 +1,258 @@
{ {
"cells": [ "cells": [
{ {
"cell_type": "markdown", "cell_type": "markdown",
"metadata": {}, "metadata": {},
"source": [ "source": [
"# Train Domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)" "# Train Domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)"
] ]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import json\n",
"import matplotlib.pyplot as plt\n",
"from pathlib import Path\n",
"import pandas as pd\n",
"from sklearn import metrics, set_config\n",
"from sklearn.model_selection import train_test_split\n",
"from sklearn.naive_bayes import MultinomialNB\n",
"from sklearn.pipeline import Pipeline\n",
"from sklearn.feature_extraction.text import TfidfVectorizer\n",
"from sklearn.model_selection import GridSearchCV\n",
"\n",
"from great_ai.utilities.clean import clean\n",
"from great_ai.utilities.parallel_map import parallel_map\n",
"from great_ai.utilities.language import is_english, predict_language\n",
"from great_ai import save_model, configure, LargeFile\n",
"\n",
"from preprocess import preprocess"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Configuration"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"PREFIX = \"domain-\"\n",
"DATASET_KEY = \"data\"\n",
"MAX_FILE_COUNT = 5\n",
"MODEL_KEY = \"small-domain-prediction-v2\""
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"configure()\n",
"corpus_path = LargeFile(DATASET_KEY).get()\n",
"\n",
"set_config(display=\"diagram\")\n",
"plt.rcParams[\"figure.figsize\"] = (30, 15)\n",
"plt.rcParams[\"figure.facecolor\"] = \"white\"\n",
"plt.rcParams[\"font.size\"] = 12\n",
"plt.rcParams[\"axes.xmargin\"] = 0"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Preprocessing"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def clean_file(p: Path) -> None:\n",
" try:\n",
" processed_path = p.with_name(f\"{PREFIX}{p.stem}{p.suffix}\")\n",
"\n",
" if processed_path.exists():\n",
" return\n",
"\n",
" with open(p) as f:\n",
" content = json.load(f)\n",
"\n",
" result = {\n",
" preprocess(\n",
" clean(f'{c[\"title\"]} {c[\"abstract\"]}', convert_to_ascii=True)\n",
" ): c[\"domain\"]\n",
" for c in content\n",
" if (\n",
" c[\"domain\"]\n",
" and c[\"abstract\"]\n",
" and is_english(predict_language(c[\"abstract\"]))\n",
" )\n",
" }\n",
"\n",
" with open(processed_path, \"w\") as f:\n",
" json.dump(result, f)\n",
" except Exception as e:\n",
" print(f\"Error ({e}) processing {p}\")\n",
"\n",
"\n",
"parallel_map(\n",
" clean_file,\n",
" list(corpus_path.glob(\"s2-corpus-*.json\"))[:MAX_FILE_COUNT],\n",
" chunk_size=1,\n",
")\n",
"None"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"corpora = list(corpus_path.glob(f\"{PREFIX}*.json\"))[:MAX_FILE_COUNT]\n",
"print(f\"Found {len(corpora)} files\")\n",
"\n",
"data = []\n",
"for p in corpora:\n",
" with open(p) as f:\n",
" data.extend(json.load(f).items())\n",
"\n",
"print(f\"Found {len(data)} documents\")\n",
"\n",
"X_train, X_test, y_train, y_test = train_test_split(\n",
" [d[0] for d in data], [d[1] for d in data], test_size=0.1, random_state=1\n",
")\n",
"\n",
"X_train = [x for x, y in zip(X_train, y_train) for domain in y]\n",
"y_train = [domain for x, y in zip(X_train, y_train) for domain in y]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Naive Bayes"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"classifier = GridSearchCV(\n",
" Pipeline(steps=[(\"vectorizer\", TfidfVectorizer(token_pattern=r\"[^ ]+\")), (\"classifier\", MultinomialNB())]),\n",
" {\n",
" \"vectorizer__max_df\": [0.05, 0.1, 0.3],\n",
" \"vectorizer__min_df\": [5, 10, 30],\n",
" \"vectorizer__sublinear_tf\": [True, False],\n",
" \"classifier__alpha\": [0.1, 0.25, 0.5, 0.75, 1],\n",
" \"classifier__fit_prior\": [True, False],\n",
" },\n",
" scoring=\"f1_macro\",\n",
" cv=3,\n",
" n_jobs=4,\n",
" verbose=1,\n",
")\n",
"classifier.fit(X_train, y_train)\n",
"\n",
"results = pd.DataFrame(classifier.cv_results_)\n",
"results.sort_values(\"rank_test_score\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"classifier = Pipeline(\n",
" steps=[\n",
" (\"vectorizer\", TfidfVectorizer(min_df=10, max_df=0.05, sublinear_tf=True, token_pattern=r\"[^ ]+\")),\n",
" (\"classifier\", MultinomialNB(alpha=0.5, fit_prior=False)),\n",
" ]\n",
")\n",
"\n",
"classifier.fit(X_train, y_train)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"predicted = classifier.predict(X_test)\n",
"\n",
"y_test_aligned = [p if p in y else y[0] for p, y in zip(predicted, y_test)]\n",
"\n",
"print(metrics.classification_report(y_test_aligned, predicted))\n",
"metrics.ConfusionMatrixDisplay.from_predictions(\n",
" y_true=y_test_aligned,\n",
" y_pred=predicted,\n",
" xticks_rotation=\"vertical\",\n",
" normalize=\"pred\",\n",
" values_format=\".2f\",\n",
")\n",
"None"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"save_model(classifier, key=MODEL_KEY, keep_last_n=1)"
]
}
],
"metadata": {
"interpreter": {
"hash": "acc70e949538f42041ccc57bc4df2261507e3fd7d6b9ce5dcc28e3bcf9d48274"
},
"kernelspec": {
"display_name": "Python 3.8.5 ('.env': venv)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.5"
},
"orig_nbformat": 4
}, },
"nbformat": 4, {
"nbformat_minor": 2 "cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import json\n",
"import matplotlib.pyplot as plt\n",
"from pathlib import Path\n",
"import pandas as pd\n",
"from sklearn import metrics, set_config\n",
"from sklearn.model_selection import train_test_split\n",
"from sklearn.naive_bayes import MultinomialNB\n",
"from sklearn.pipeline import Pipeline\n",
"from sklearn.feature_extraction.text import TfidfVectorizer\n",
"from sklearn.model_selection import GridSearchCV\n",
"\n",
"from great_ai.utilities.clean import clean\n",
"from great_ai.utilities.parallel_map import parallel_map\n",
"from great_ai.utilities.language import is_english, predict_language\n",
"from great_ai import save_model, configure, LargeFile\n",
"\n",
"from preprocess import preprocess"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Configuration"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"PREFIX = \"domain-\"\n",
"DATASET_KEY = \"data\"\n",
"MAX_FILE_COUNT = 5\n",
"MODEL_KEY = \"small-domain-prediction-v2\""
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"configure()\n",
"corpus_path = LargeFile(DATASET_KEY).get()\n",
"\n",
"set_config(display=\"diagram\")\n",
"plt.rcParams[\"figure.figsize\"] = (30, 15)\n",
"plt.rcParams[\"figure.facecolor\"] = \"white\"\n",
"plt.rcParams[\"font.size\"] = 12\n",
"plt.rcParams[\"axes.xmargin\"] = 0"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Preprocessing"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def clean_file(p: Path) -> None:\n",
" try:\n",
" processed_path = p.with_name(f\"{PREFIX}{p.stem}{p.suffix}\")\n",
"\n",
" if processed_path.exists():\n",
" return\n",
"\n",
" with open(p) as f:\n",
" content = json.load(f)\n",
"\n",
" result = {\n",
" preprocess(\n",
" clean(f'{c[\"title\"]} {c[\"abstract\"]}', convert_to_ascii=True)\n",
" ): c[\"domain\"]\n",
" for c in content\n",
" if (\n",
" c[\"domain\"]\n",
" and c[\"abstract\"]\n",
" and is_english(predict_language(c[\"abstract\"]))\n",
" )\n",
" }\n",
"\n",
" with open(processed_path, \"w\") as f:\n",
" json.dump(result, f)\n",
" except Exception as e:\n",
" print(f\"Error ({e}) processing {p}\")\n",
"\n",
"\n",
"parallel_map(\n",
" clean_file,\n",
" list(corpus_path.glob(\"s2-corpus-*.json\"))[:MAX_FILE_COUNT],\n",
" chunk_size=1,\n",
")\n",
"None"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"corpora = list(corpus_path.glob(f\"{PREFIX}*.json\"))[:MAX_FILE_COUNT]\n",
"print(f\"Found {len(corpora)} files\")\n",
"\n",
"data = []\n",
"for p in corpora:\n",
" with open(p) as f:\n",
" data.extend(json.load(f).items())\n",
"\n",
"print(f\"Found {len(data)} documents\")\n",
"\n",
"X_train, X_test, y_train, y_test = train_test_split(\n",
" [d[0] for d in data], [d[1] for d in data], test_size=0.1, random_state=1\n",
")\n",
"\n",
"X_train = [x for x, y in zip(X_train, y_train) for domain in y]\n",
"y_train = [domain for x, y in zip(X_train, y_train) for domain in y]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Naive Bayes"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"classifier = GridSearchCV(\n",
" Pipeline(\n",
" steps=[\n",
" (\"vectorizer\", TfidfVectorizer(token_pattern=r\"[^ ]+\")),\n",
" (\"classifier\", MultinomialNB()),\n",
" ]\n",
" ),\n",
" {\n",
" \"vectorizer__max_df\": [0.05, 0.1, 0.3],\n",
" \"vectorizer__min_df\": [5, 10, 30],\n",
" \"vectorizer__sublinear_tf\": [True, False],\n",
" \"classifier__alpha\": [0.1, 0.25, 0.5, 0.75, 1],\n",
" \"classifier__fit_prior\": [True, False],\n",
" },\n",
" scoring=\"f1_macro\",\n",
" cv=3,\n",
" n_jobs=4,\n",
" verbose=1,\n",
")\n",
"classifier.fit(X_train, y_train)\n",
"\n",
"results = pd.DataFrame(classifier.cv_results_)\n",
"results.sort_values(\"rank_test_score\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"classifier = Pipeline(\n",
" steps=[\n",
" (\n",
" \"vectorizer\",\n",
" TfidfVectorizer(\n",
" min_df=10, max_df=0.05, sublinear_tf=True, token_pattern=r\"[^ ]+\"\n",
" ),\n",
" ),\n",
" (\"classifier\", MultinomialNB(alpha=0.5, fit_prior=False)),\n",
" ]\n",
")\n",
"\n",
"classifier.fit(X_train, y_train)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"predicted = classifier.predict(X_test)\n",
"\n",
"y_test_aligned = [p if p in y else y[0] for p, y in zip(predicted, y_test)]\n",
"\n",
"print(metrics.classification_report(y_test_aligned, predicted))\n",
"metrics.ConfusionMatrixDisplay.from_predictions(\n",
" y_true=y_test_aligned,\n",
" y_pred=predicted,\n",
" xticks_rotation=\"vertical\",\n",
" normalize=\"pred\",\n",
" values_format=\".2f\",\n",
")\n",
"None"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"save_model(classifier, key=MODEL_KEY, keep_last_n=1)"
]
}
],
"metadata": {
"interpreter": {
"hash": "acc70e949538f42041ccc57bc4df2261507e3fd7d6b9ce5dcc28e3bcf9d48274"
},
"kernelspec": {
"display_name": "Python 3.8.5 ('.env': venv)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.5"
},
"orig_nbformat": 4
},
"nbformat": 4,
"nbformat_minor": 2
} }