andras/great-ai
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases 4 Packages Wiki Activity Actions

Format

Browse source 
Signed-off-by: András Schmelczer <andras@schmelczer.dev>
This commit is contained in:
Andras Schmelczer 2022-04-02 13:57:16 +02:00
parent d79a75d352
commit ea54cd9c8c
13 changed files with 168 additions and 116 deletions
Download patch file Download diff file Expand all files Collapse all files

62
example/train.ipynb
View file

@ -85,10 +85,7 @@
"source": [
"def preprocess(text: str) -> str:\n",
" cleaned = clean(text, convert_to_ascii=True)\n",
" lemmas = [\n",
" re.sub(r'\\d+', 'NUM', lemma)\n",
" for lemma in lemmatize_text(cleaned)\n",
" ]\n",
" lemmas = [re.sub(r\"\\d+\", \"NUM\", lemma) for lemma in lemmatize_text(cleaned)]\n",
" return \" \".join(lemmas)"
]
},
@ -157,14 +154,14 @@
"source": [
"corpora = list(SS_CORPUS_PATH.glob(f\"{PREFIX}*.json\"))\n",
"shuffle(corpora)\n",
"print(f'Found {len(corpora)} files')\n",
"print(f\"Found {len(corpora)} files\")\n",
"\n",
"data = []\n",
"for p in corpora[:MAX_FILE_COUNT]:\n",
" with open(p) as f:\n",
" data.extend(json.load(f).items())\n",
"\n",
"print(f'Found {len(data)} documents')"
"print(f\"Found {len(data)} documents\")"
]
},
{
@ -174,23 +171,12 @@
"outputs": [],
"source": [
"X_train, X_test, y_train, y_test = train_test_split(\n",
" [d[0] for d in data],\n",
" [d[1] for d in data],\n",
" test_size=0.1, \n",
" random_state=SEED\n",
" [d[0] for d in data], [d[1] for d in data], test_size=0.1, random_state=SEED\n",
")\n",
"\n",
"X_train = [\n",
" x\n",
" for x, y in zip(X_train, y_train)\n",
" for domain in y\n",
"]\n",
"X_train = [x for x, y in zip(X_train, y_train) for domain in y]\n",
"\n",
"y_train = [\n",
" domain\n",
" for x, y in zip(X_train, y_train)\n",
" for domain in y\n",
"]"
"y_train = [domain for x, y in zip(X_train, y_train) for domain in y]"
]
},
{
@ -207,23 +193,23 @@
"outputs": [],
"source": [
"classifier = GridSearchCV(\n",
" Pipeline(steps=[\n",
" ('vectorizer', TfidfVectorizer()),\n",
" ('classifier', ComplementNB())\n",
" ]),\n",
" Pipeline(steps=[(\"vectorizer\", TfidfVectorizer()), (\"classifier\", ComplementNB())]),\n",
" {\n",
" \"vectorizer__max_df\": [0.05, 0.1, 0.3],\n",
" \"vectorizer__min_df\": [5, 10, 30, 100],\n",
" \"vectorizer__sublinear_tf\": [True, False],\n",
" \"classifier__alpha\": [0.001, 0.1, 0.5, 1],\n",
" \"classifier__fit_prior\": [True, False]\n",
" \"classifier__fit_prior\": [True, False],\n",
" },\n",
" scoring=\"f1_macro\",\n",
" cv=3,\n",
" n_jobs=12,\n",
" verbose=7,\n",
" cv=3,\n",
" n_jobs=12,\n",
" verbose=7,\n",
")\n",
"classifier.fit(\n",
" X_train[:HYPERPARAMETER_OPTIMISATION_SIZE],\n",
" y_train[:HYPERPARAMETER_OPTIMISATION_SIZE],\n",
")\n",
"classifier.fit(X_train[:HYPERPARAMETER_OPTIMISATION_SIZE], y_train[:HYPERPARAMETER_OPTIMISATION_SIZE])\n",
"\n",
"results = pd.DataFrame(classifier.cv_results_)\n",
"results.sort_values(\"rank_test_score\")"
@ -247,10 +233,12 @@
}
],
"source": [
"classifier = Pipeline(steps=[\n",
" ('vectorizer', TfidfVectorizer(min_df=10, max_df=0.05)),\n",
" ('classifier', ComplementNB(alpha=0.5, fit_prior=False))\n",
"])\n",
"classifier = Pipeline(\n",
" steps=[\n",
" (\"vectorizer\", TfidfVectorizer(min_df=10, max_df=0.05)),\n",
" (\"classifier\", ComplementNB(alpha=0.5, fit_prior=False)),\n",
" ]\n",
")\n",
"\n",
"classifier.fit(X_train, y_train)"
]
@ -310,7 +298,11 @@
"\n",
"print(metrics.classification_report(y_test_aligned, predicted))\n",
"metrics.ConfusionMatrixDisplay.from_predictions(\n",
" y_true=y_test_aligned, y_pred=predicted, xticks_rotation=\"vertical\", normalize=\"pred\", values_format='.2f'\n",
" y_true=y_test_aligned,\n",
" y_pred=predicted,\n",
" xticks_rotation=\"vertical\",\n",
" normalize=\"pred\",\n",
" values_format=\".2f\",\n",
")\n",
"None"
]
@ -333,7 +325,7 @@
"outputs": [],
"source": [
"for X, y in zip(X_test[:50], y_test):\n",
" print(', '.join(y))\n",
" print(\", \".join(y))\n",
" pprint(predict(X))\n",
" print()"
]