Split examples
This commit is contained in:
parent
12407923c4
commit
9a8144d19a
11 changed files with 401 additions and 252 deletions
6
.vscode/tasks.json
vendored
6
.vscode/tasks.json
vendored
|
|
@ -4,9 +4,9 @@
|
||||||
{
|
{
|
||||||
"label": "Format and lint Python",
|
"label": "Format and lint Python",
|
||||||
"type": "shell",
|
"type": "shell",
|
||||||
"command": "source .env/bin/activate && scripts/format-python.sh great_ai && scripts/format-python.sh examples",
|
"command": "source .env/bin/activate && scripts/format-python.sh great_ai && scripts/format-python.sh examples/simple && scripts/format-python.sh examples/complex",
|
||||||
"windows": {
|
"windows": {
|
||||||
"command": ".env/bin/activate.bat; scripts/format-python.sh great_ai; scripts\\format-python.sh examples"
|
"command": ".env\\bin\\activate.bat; scripts\\format-python.sh great_ai; scripts\\format-python.sh examples\\simple; scripts\\format-python.sh examples\\complex"
|
||||||
},
|
},
|
||||||
"group": "test",
|
"group": "test",
|
||||||
"presentation": {
|
"presentation": {
|
||||||
|
|
@ -22,7 +22,7 @@
|
||||||
"type": "shell",
|
"type": "shell",
|
||||||
"command": "source .env/bin/activate && python3 -m pytest great_ai",
|
"command": "source .env/bin/activate && python3 -m pytest great_ai",
|
||||||
"windows": {
|
"windows": {
|
||||||
"command": ".env/bin/activate.bat; python3 -m pytest great_ai"
|
"command": ".env\\bin\\activate.bat; python3 -m pytest great_ai"
|
||||||
},
|
},
|
||||||
"group": "test",
|
"group": "test",
|
||||||
"presentation": {
|
"presentation": {
|
||||||
|
|
|
||||||
25
examples/complex/Dockerfile
Normal file
25
examples/complex/Dockerfile
Normal file
|
|
@ -0,0 +1,25 @@
|
||||||
|
FROM python:3.8.12-slim-bullseye
|
||||||
|
|
||||||
|
ENV ENVIRONMENT production
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
RUN apt-get update &&\
|
||||||
|
apt-get install curl -y &&\
|
||||||
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
RUN pip install --no-cache-dir en-core-web-sm@https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl
|
||||||
|
COPY sus sus
|
||||||
|
RUN pip install --no-cache-dir --use-feature=in-tree-build ./sus
|
||||||
|
|
||||||
|
COPY requirements.txt ./
|
||||||
|
RUN pip install --no-cache-dir --requirement requirements.txt
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
RUN rm -rf sus secrets
|
||||||
|
|
||||||
|
EXPOSE 5000
|
||||||
|
|
||||||
|
HEALTHCHECK --interval=60s --timeout=60s --start-period=90s --retries=5 CMD [ "curl", "--fail", "http://localhost:5000/health" ]
|
||||||
|
|
||||||
|
CMD ["uvicorn", "main:app", "--proxy-headers", "--backlog", "8196", "--timeout-keep-alive", "900", "--host", "0.0.0.0", "--port", "5000"]
|
||||||
22
examples/complex/Dockerfile.development
Normal file
22
examples/complex/Dockerfile.development
Normal file
|
|
@ -0,0 +1,22 @@
|
||||||
|
FROM python:3.8.12-slim-bullseye
|
||||||
|
|
||||||
|
ENV ENVIRONMENT development
|
||||||
|
|
||||||
|
VOLUME /app
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
RUN apt-get update &&\
|
||||||
|
apt-get install build-essential -y &&\
|
||||||
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
RUN python3 -m pip install --no-cache-dir --upgrade pip &&\
|
||||||
|
python3 -m pip install --no-cache-dir en-core-web-sm@https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl
|
||||||
|
|
||||||
|
COPY requirements.txt ./
|
||||||
|
RUN python3 -m pip install --no-cache-dir -r requirements.txt
|
||||||
|
|
||||||
|
EXPOSE 5000
|
||||||
|
|
||||||
|
VOLUME /dependencies
|
||||||
|
|
||||||
|
CMD ["/bin/bash", "-c", "python3 -m pip install -e /dependencies/sus && python3 -m uvicorn --reload --host 0.0.0.0 --port 5000 main:app"]
|
||||||
|
|
@ -1,8 +1,6 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
|
|
||||||
from great_ai import configure, create_service
|
from great_ai import configure, create_service
|
||||||
|
|
||||||
# configure(development_mode_override=True)
|
configure(development_mode_override=True)
|
||||||
|
|
||||||
from predict_domain import predict_domain
|
from predict_domain import predict_domain
|
||||||
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
import re
|
import re
|
||||||
from typing import Dict, Iterable, List
|
from typing import Dict, Iterable, List
|
||||||
|
|
||||||
from preprocess import preprocess
|
from preprocess import preprocess
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
86
examples/simple/predict_domain.py
Normal file
86
examples/simple/predict_domain.py
Normal file
|
|
@ -0,0 +1,86 @@
|
||||||
|
from great_ai import configure, create_service, log_argument, log_metric, use_model
|
||||||
|
|
||||||
|
configure(development_mode_override=True)
|
||||||
|
|
||||||
|
import re
|
||||||
|
from typing import Dict, Iterable, List
|
||||||
|
|
||||||
|
from preprocess import preprocess
|
||||||
|
from pydantic import BaseModel
|
||||||
|
from sklearn.pipeline import Pipeline
|
||||||
|
|
||||||
|
from great_ai.utilities.clean import clean
|
||||||
|
|
||||||
|
|
||||||
|
class DomainPrediction(BaseModel):
|
||||||
|
domain: str
|
||||||
|
probability: float
|
||||||
|
explanation: List[str]
|
||||||
|
|
||||||
|
|
||||||
|
@create_service
|
||||||
|
@use_model("small-domain-prediction-v2", version="latest")
|
||||||
|
@log_argument("text", validator=lambda t: len(t) > 0)
|
||||||
|
def predict_domain(
|
||||||
|
text: str, model: Pipeline, cut_off_probability: float = 0.2
|
||||||
|
) -> List[DomainPrediction]:
|
||||||
|
assert 0 <= cut_off_probability <= 1
|
||||||
|
|
||||||
|
"""
|
||||||
|
Predict the scientific domain of the input text.
|
||||||
|
Return labels until their sum likelihood is larger than cut_off_probability.
|
||||||
|
"""
|
||||||
|
log_metric("text_length", len(text))
|
||||||
|
|
||||||
|
cleaned = clean(text, convert_to_ascii=True)
|
||||||
|
text = re.sub(r"[^a-zA-Z0-9]", " ", cleaned)
|
||||||
|
|
||||||
|
feature_names = model.named_steps["vectorizer"].get_feature_names_out()
|
||||||
|
|
||||||
|
token_mapping = {preprocess(original): original for original in text.split(" ")}
|
||||||
|
|
||||||
|
features = model.named_steps["vectorizer"].transform(
|
||||||
|
[" ".join(token_mapping.keys())]
|
||||||
|
)
|
||||||
|
prediction = model.named_steps["classifier"].predict_proba(features)[0]
|
||||||
|
best_classes = sorted(enumerate(prediction), key=lambda v: v[1], reverse=True)
|
||||||
|
|
||||||
|
results: List[DomainPrediction] = []
|
||||||
|
for class_index, probability in best_classes:
|
||||||
|
weights = model.named_steps["classifier"].feature_log_prob_[class_index]
|
||||||
|
domain = model.named_steps["classifier"].classes_[class_index]
|
||||||
|
|
||||||
|
results.append(
|
||||||
|
DomainPrediction(
|
||||||
|
domain=domain,
|
||||||
|
probability=round(probability * 100),
|
||||||
|
explanation=_get_explanation(
|
||||||
|
feature_names=feature_names,
|
||||||
|
features=features.A[0],
|
||||||
|
weights=weights,
|
||||||
|
token_mapping=token_mapping,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if sum(r.probability for r in results) >= cut_off_probability * 100:
|
||||||
|
break
|
||||||
|
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
def _get_explanation(
|
||||||
|
feature_names: Iterable[str],
|
||||||
|
features: Iterable[float],
|
||||||
|
weights: Iterable[float],
|
||||||
|
token_mapping: Dict[str, str],
|
||||||
|
) -> List[str]:
|
||||||
|
influential = [
|
||||||
|
(weight, name)
|
||||||
|
for weight, value, name in zip(weights, features, feature_names)
|
||||||
|
if value
|
||||||
|
]
|
||||||
|
|
||||||
|
most_influential = sorted(influential, reverse=True)[:5]
|
||||||
|
|
||||||
|
return [token_mapping[name] for _, name in most_influential]
|
||||||
8
examples/simple/preprocess.py
Normal file
8
examples/simple/preprocess.py
Normal file
|
|
@ -0,0 +1,8 @@
|
||||||
|
import re
|
||||||
|
|
||||||
|
from great_ai.utilities.lemmatize_text import lemmatize_text
|
||||||
|
|
||||||
|
|
||||||
|
def preprocess(text: str) -> str:
|
||||||
|
lemmas = [re.sub(r"\d[\d.,]*", "NUM", lemma) for lemma in lemmatize_text(text)]
|
||||||
|
return " ".join(lemmas)
|
||||||
|
|
@ -1,248 +1,258 @@
|
||||||
{
|
{
|
||||||
"cells": [
|
"cells": [
|
||||||
{
|
{
|
||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"# Train Domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)"
|
"# Train Domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)"
|
||||||
]
|
]
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"import json\n",
|
|
||||||
"import matplotlib.pyplot as plt\n",
|
|
||||||
"from pathlib import Path\n",
|
|
||||||
"import pandas as pd\n",
|
|
||||||
"from sklearn import metrics, set_config\n",
|
|
||||||
"from sklearn.model_selection import train_test_split\n",
|
|
||||||
"from sklearn.naive_bayes import MultinomialNB\n",
|
|
||||||
"from sklearn.pipeline import Pipeline\n",
|
|
||||||
"from sklearn.feature_extraction.text import TfidfVectorizer\n",
|
|
||||||
"from sklearn.model_selection import GridSearchCV\n",
|
|
||||||
"\n",
|
|
||||||
"from great_ai.utilities.clean import clean\n",
|
|
||||||
"from great_ai.utilities.parallel_map import parallel_map\n",
|
|
||||||
"from great_ai.utilities.language import is_english, predict_language\n",
|
|
||||||
"from great_ai import save_model, configure, LargeFile\n",
|
|
||||||
"\n",
|
|
||||||
"from preprocess import preprocess"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Configuration"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"PREFIX = \"domain-\"\n",
|
|
||||||
"DATASET_KEY = \"data\"\n",
|
|
||||||
"MAX_FILE_COUNT = 5\n",
|
|
||||||
"MODEL_KEY = \"small-domain-prediction-v2\""
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"configure()\n",
|
|
||||||
"corpus_path = LargeFile(DATASET_KEY).get()\n",
|
|
||||||
"\n",
|
|
||||||
"set_config(display=\"diagram\")\n",
|
|
||||||
"plt.rcParams[\"figure.figsize\"] = (30, 15)\n",
|
|
||||||
"plt.rcParams[\"figure.facecolor\"] = \"white\"\n",
|
|
||||||
"plt.rcParams[\"font.size\"] = 12\n",
|
|
||||||
"plt.rcParams[\"axes.xmargin\"] = 0"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Preprocessing"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"def clean_file(p: Path) -> None:\n",
|
|
||||||
" try:\n",
|
|
||||||
" processed_path = p.with_name(f\"{PREFIX}{p.stem}{p.suffix}\")\n",
|
|
||||||
"\n",
|
|
||||||
" if processed_path.exists():\n",
|
|
||||||
" return\n",
|
|
||||||
"\n",
|
|
||||||
" with open(p) as f:\n",
|
|
||||||
" content = json.load(f)\n",
|
|
||||||
"\n",
|
|
||||||
" result = {\n",
|
|
||||||
" preprocess(\n",
|
|
||||||
" clean(f'{c[\"title\"]} {c[\"abstract\"]}', convert_to_ascii=True)\n",
|
|
||||||
" ): c[\"domain\"]\n",
|
|
||||||
" for c in content\n",
|
|
||||||
" if (\n",
|
|
||||||
" c[\"domain\"]\n",
|
|
||||||
" and c[\"abstract\"]\n",
|
|
||||||
" and is_english(predict_language(c[\"abstract\"]))\n",
|
|
||||||
" )\n",
|
|
||||||
" }\n",
|
|
||||||
"\n",
|
|
||||||
" with open(processed_path, \"w\") as f:\n",
|
|
||||||
" json.dump(result, f)\n",
|
|
||||||
" except Exception as e:\n",
|
|
||||||
" print(f\"Error ({e}) processing {p}\")\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"parallel_map(\n",
|
|
||||||
" clean_file,\n",
|
|
||||||
" list(corpus_path.glob(\"s2-corpus-*.json\"))[:MAX_FILE_COUNT],\n",
|
|
||||||
" chunk_size=1,\n",
|
|
||||||
")\n",
|
|
||||||
"None"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"corpora = list(corpus_path.glob(f\"{PREFIX}*.json\"))[:MAX_FILE_COUNT]\n",
|
|
||||||
"print(f\"Found {len(corpora)} files\")\n",
|
|
||||||
"\n",
|
|
||||||
"data = []\n",
|
|
||||||
"for p in corpora:\n",
|
|
||||||
" with open(p) as f:\n",
|
|
||||||
" data.extend(json.load(f).items())\n",
|
|
||||||
"\n",
|
|
||||||
"print(f\"Found {len(data)} documents\")\n",
|
|
||||||
"\n",
|
|
||||||
"X_train, X_test, y_train, y_test = train_test_split(\n",
|
|
||||||
" [d[0] for d in data], [d[1] for d in data], test_size=0.1, random_state=1\n",
|
|
||||||
")\n",
|
|
||||||
"\n",
|
|
||||||
"X_train = [x for x, y in zip(X_train, y_train) for domain in y]\n",
|
|
||||||
"y_train = [domain for x, y in zip(X_train, y_train) for domain in y]"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Naive Bayes"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"classifier = GridSearchCV(\n",
|
|
||||||
" Pipeline(steps=[(\"vectorizer\", TfidfVectorizer(token_pattern=r\"[^ ]+\")), (\"classifier\", MultinomialNB())]),\n",
|
|
||||||
" {\n",
|
|
||||||
" \"vectorizer__max_df\": [0.05, 0.1, 0.3],\n",
|
|
||||||
" \"vectorizer__min_df\": [5, 10, 30],\n",
|
|
||||||
" \"vectorizer__sublinear_tf\": [True, False],\n",
|
|
||||||
" \"classifier__alpha\": [0.1, 0.25, 0.5, 0.75, 1],\n",
|
|
||||||
" \"classifier__fit_prior\": [True, False],\n",
|
|
||||||
" },\n",
|
|
||||||
" scoring=\"f1_macro\",\n",
|
|
||||||
" cv=3,\n",
|
|
||||||
" n_jobs=4,\n",
|
|
||||||
" verbose=1,\n",
|
|
||||||
")\n",
|
|
||||||
"classifier.fit(X_train, y_train)\n",
|
|
||||||
"\n",
|
|
||||||
"results = pd.DataFrame(classifier.cv_results_)\n",
|
|
||||||
"results.sort_values(\"rank_test_score\")"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"classifier = Pipeline(\n",
|
|
||||||
" steps=[\n",
|
|
||||||
" (\"vectorizer\", TfidfVectorizer(min_df=10, max_df=0.05, sublinear_tf=True, token_pattern=r\"[^ ]+\")),\n",
|
|
||||||
" (\"classifier\", MultinomialNB(alpha=0.5, fit_prior=False)),\n",
|
|
||||||
" ]\n",
|
|
||||||
")\n",
|
|
||||||
"\n",
|
|
||||||
"classifier.fit(X_train, y_train)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"predicted = classifier.predict(X_test)\n",
|
|
||||||
"\n",
|
|
||||||
"y_test_aligned = [p if p in y else y[0] for p, y in zip(predicted, y_test)]\n",
|
|
||||||
"\n",
|
|
||||||
"print(metrics.classification_report(y_test_aligned, predicted))\n",
|
|
||||||
"metrics.ConfusionMatrixDisplay.from_predictions(\n",
|
|
||||||
" y_true=y_test_aligned,\n",
|
|
||||||
" y_pred=predicted,\n",
|
|
||||||
" xticks_rotation=\"vertical\",\n",
|
|
||||||
" normalize=\"pred\",\n",
|
|
||||||
" values_format=\".2f\",\n",
|
|
||||||
")\n",
|
|
||||||
"None"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"save_model(classifier, key=MODEL_KEY, keep_last_n=1)"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"interpreter": {
|
|
||||||
"hash": "acc70e949538f42041ccc57bc4df2261507e3fd7d6b9ce5dcc28e3bcf9d48274"
|
|
||||||
},
|
|
||||||
"kernelspec": {
|
|
||||||
"display_name": "Python 3.8.5 ('.env': venv)",
|
|
||||||
"language": "python",
|
|
||||||
"name": "python3"
|
|
||||||
},
|
|
||||||
"language_info": {
|
|
||||||
"codemirror_mode": {
|
|
||||||
"name": "ipython",
|
|
||||||
"version": 3
|
|
||||||
},
|
|
||||||
"file_extension": ".py",
|
|
||||||
"mimetype": "text/x-python",
|
|
||||||
"name": "python",
|
|
||||||
"nbconvert_exporter": "python",
|
|
||||||
"pygments_lexer": "ipython3",
|
|
||||||
"version": "3.8.5"
|
|
||||||
},
|
|
||||||
"orig_nbformat": 4
|
|
||||||
},
|
},
|
||||||
"nbformat": 4,
|
{
|
||||||
"nbformat_minor": 2
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import json\n",
|
||||||
|
"import matplotlib.pyplot as plt\n",
|
||||||
|
"from pathlib import Path\n",
|
||||||
|
"import pandas as pd\n",
|
||||||
|
"from sklearn import metrics, set_config\n",
|
||||||
|
"from sklearn.model_selection import train_test_split\n",
|
||||||
|
"from sklearn.naive_bayes import MultinomialNB\n",
|
||||||
|
"from sklearn.pipeline import Pipeline\n",
|
||||||
|
"from sklearn.feature_extraction.text import TfidfVectorizer\n",
|
||||||
|
"from sklearn.model_selection import GridSearchCV\n",
|
||||||
|
"\n",
|
||||||
|
"from great_ai.utilities.clean import clean\n",
|
||||||
|
"from great_ai.utilities.parallel_map import parallel_map\n",
|
||||||
|
"from great_ai.utilities.language import is_english, predict_language\n",
|
||||||
|
"from great_ai import save_model, configure, LargeFile\n",
|
||||||
|
"\n",
|
||||||
|
"from preprocess import preprocess"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Configuration"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"PREFIX = \"domain-\"\n",
|
||||||
|
"DATASET_KEY = \"data\"\n",
|
||||||
|
"MAX_FILE_COUNT = 5\n",
|
||||||
|
"MODEL_KEY = \"small-domain-prediction-v2\""
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"configure()\n",
|
||||||
|
"corpus_path = LargeFile(DATASET_KEY).get()\n",
|
||||||
|
"\n",
|
||||||
|
"set_config(display=\"diagram\")\n",
|
||||||
|
"plt.rcParams[\"figure.figsize\"] = (30, 15)\n",
|
||||||
|
"plt.rcParams[\"figure.facecolor\"] = \"white\"\n",
|
||||||
|
"plt.rcParams[\"font.size\"] = 12\n",
|
||||||
|
"plt.rcParams[\"axes.xmargin\"] = 0"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Preprocessing"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"def clean_file(p: Path) -> None:\n",
|
||||||
|
" try:\n",
|
||||||
|
" processed_path = p.with_name(f\"{PREFIX}{p.stem}{p.suffix}\")\n",
|
||||||
|
"\n",
|
||||||
|
" if processed_path.exists():\n",
|
||||||
|
" return\n",
|
||||||
|
"\n",
|
||||||
|
" with open(p) as f:\n",
|
||||||
|
" content = json.load(f)\n",
|
||||||
|
"\n",
|
||||||
|
" result = {\n",
|
||||||
|
" preprocess(\n",
|
||||||
|
" clean(f'{c[\"title\"]} {c[\"abstract\"]}', convert_to_ascii=True)\n",
|
||||||
|
" ): c[\"domain\"]\n",
|
||||||
|
" for c in content\n",
|
||||||
|
" if (\n",
|
||||||
|
" c[\"domain\"]\n",
|
||||||
|
" and c[\"abstract\"]\n",
|
||||||
|
" and is_english(predict_language(c[\"abstract\"]))\n",
|
||||||
|
" )\n",
|
||||||
|
" }\n",
|
||||||
|
"\n",
|
||||||
|
" with open(processed_path, \"w\") as f:\n",
|
||||||
|
" json.dump(result, f)\n",
|
||||||
|
" except Exception as e:\n",
|
||||||
|
" print(f\"Error ({e}) processing {p}\")\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"parallel_map(\n",
|
||||||
|
" clean_file,\n",
|
||||||
|
" list(corpus_path.glob(\"s2-corpus-*.json\"))[:MAX_FILE_COUNT],\n",
|
||||||
|
" chunk_size=1,\n",
|
||||||
|
")\n",
|
||||||
|
"None"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"corpora = list(corpus_path.glob(f\"{PREFIX}*.json\"))[:MAX_FILE_COUNT]\n",
|
||||||
|
"print(f\"Found {len(corpora)} files\")\n",
|
||||||
|
"\n",
|
||||||
|
"data = []\n",
|
||||||
|
"for p in corpora:\n",
|
||||||
|
" with open(p) as f:\n",
|
||||||
|
" data.extend(json.load(f).items())\n",
|
||||||
|
"\n",
|
||||||
|
"print(f\"Found {len(data)} documents\")\n",
|
||||||
|
"\n",
|
||||||
|
"X_train, X_test, y_train, y_test = train_test_split(\n",
|
||||||
|
" [d[0] for d in data], [d[1] for d in data], test_size=0.1, random_state=1\n",
|
||||||
|
")\n",
|
||||||
|
"\n",
|
||||||
|
"X_train = [x for x, y in zip(X_train, y_train) for domain in y]\n",
|
||||||
|
"y_train = [domain for x, y in zip(X_train, y_train) for domain in y]"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Naive Bayes"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"classifier = GridSearchCV(\n",
|
||||||
|
" Pipeline(\n",
|
||||||
|
" steps=[\n",
|
||||||
|
" (\"vectorizer\", TfidfVectorizer(token_pattern=r\"[^ ]+\")),\n",
|
||||||
|
" (\"classifier\", MultinomialNB()),\n",
|
||||||
|
" ]\n",
|
||||||
|
" ),\n",
|
||||||
|
" {\n",
|
||||||
|
" \"vectorizer__max_df\": [0.05, 0.1, 0.3],\n",
|
||||||
|
" \"vectorizer__min_df\": [5, 10, 30],\n",
|
||||||
|
" \"vectorizer__sublinear_tf\": [True, False],\n",
|
||||||
|
" \"classifier__alpha\": [0.1, 0.25, 0.5, 0.75, 1],\n",
|
||||||
|
" \"classifier__fit_prior\": [True, False],\n",
|
||||||
|
" },\n",
|
||||||
|
" scoring=\"f1_macro\",\n",
|
||||||
|
" cv=3,\n",
|
||||||
|
" n_jobs=4,\n",
|
||||||
|
" verbose=1,\n",
|
||||||
|
")\n",
|
||||||
|
"classifier.fit(X_train, y_train)\n",
|
||||||
|
"\n",
|
||||||
|
"results = pd.DataFrame(classifier.cv_results_)\n",
|
||||||
|
"results.sort_values(\"rank_test_score\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"classifier = Pipeline(\n",
|
||||||
|
" steps=[\n",
|
||||||
|
" (\n",
|
||||||
|
" \"vectorizer\",\n",
|
||||||
|
" TfidfVectorizer(\n",
|
||||||
|
" min_df=10, max_df=0.05, sublinear_tf=True, token_pattern=r\"[^ ]+\"\n",
|
||||||
|
" ),\n",
|
||||||
|
" ),\n",
|
||||||
|
" (\"classifier\", MultinomialNB(alpha=0.5, fit_prior=False)),\n",
|
||||||
|
" ]\n",
|
||||||
|
")\n",
|
||||||
|
"\n",
|
||||||
|
"classifier.fit(X_train, y_train)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"predicted = classifier.predict(X_test)\n",
|
||||||
|
"\n",
|
||||||
|
"y_test_aligned = [p if p in y else y[0] for p, y in zip(predicted, y_test)]\n",
|
||||||
|
"\n",
|
||||||
|
"print(metrics.classification_report(y_test_aligned, predicted))\n",
|
||||||
|
"metrics.ConfusionMatrixDisplay.from_predictions(\n",
|
||||||
|
" y_true=y_test_aligned,\n",
|
||||||
|
" y_pred=predicted,\n",
|
||||||
|
" xticks_rotation=\"vertical\",\n",
|
||||||
|
" normalize=\"pred\",\n",
|
||||||
|
" values_format=\".2f\",\n",
|
||||||
|
")\n",
|
||||||
|
"None"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"save_model(classifier, key=MODEL_KEY, keep_last_n=1)"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"interpreter": {
|
||||||
|
"hash": "acc70e949538f42041ccc57bc4df2261507e3fd7d6b9ce5dcc28e3bcf9d48274"
|
||||||
|
},
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3.8.5 ('.env': venv)",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.8.5"
|
||||||
|
},
|
||||||
|
"orig_nbformat": 4
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 2
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue