1796 lines
59 KiB
Text
1796 lines
59 KiB
Text
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Train a domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)\n",
|
|
"\n",
|
|
"> Part 2: train a model\n",
|
|
"\n",
|
|
"\n",
|
|
"> The blue boxes show the steps implemented in this notebook.\n",
|
|
"\n",
|
|
"In [Part 1](data.ipynb), we have cleaned and transformed our training data. We can now access this data using `great_ai.LargeFile`. Locally, it will gives us the cached version, otherwise, the latest version is downloaded from S3. \n",
|
|
"\n",
|
|
"In this part, we hyperparameter-optimise and train a simple, Naive Bayes classifier which we then export for deployment using `great_ai.save_model`."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"MODEL_KEY = \"small-domain-prediction\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Load data that has been extracted in [part 1](data.ipynb)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\u001b[38;5;226m2022-06-19 15:08:22,338 | WARNING | Environment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
|
|
"\u001b[38;5;226m2022-06-19 15:08:22,338 | WARNING | The selected persistence driver (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n",
|
|
"\u001b[38;5;39m2022-06-19 15:08:22,339 | INFO | Options: configured ✅\u001b[0m\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"from great_ai import query_ground_truth\n",
|
|
"\n",
|
|
"data = query_ground_truth(\"train\")\n",
|
|
"X = [d.input for d in data for domain in d.feedback]\n",
|
|
"y = [domain for d in data for domain in d.feedback]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"application/vnd.plotly.v1+json": {
|
|
"config": {
|
|
"plotlyServerURL": "https://plot.ly"
|
|
},
|
|
"data": [
|
|
{
|
|
"alignmentgroup": "True",
|
|
"hovertemplate": "x=%{x}<br>y=%{y}<extra></extra>",
|
|
"legendgroup": "",
|
|
"marker": {
|
|
"color": "#636efa",
|
|
"pattern": {
|
|
"shape": ""
|
|
}
|
|
},
|
|
"name": "",
|
|
"offsetgroup": "",
|
|
"orientation": "v",
|
|
"showlegend": false,
|
|
"textposition": "auto",
|
|
"type": "bar",
|
|
"x": [
|
|
"Medicine",
|
|
"Computer Science",
|
|
"Biology",
|
|
"Chemistry",
|
|
"Materials Science",
|
|
"Engineering",
|
|
"Physics",
|
|
"Psychology",
|
|
"Mathematics",
|
|
"Business",
|
|
"Political Science",
|
|
"Sociology",
|
|
"Economics",
|
|
"Geography",
|
|
"Environmental Science",
|
|
"Geology",
|
|
"Art",
|
|
"History",
|
|
"Philosophy"
|
|
],
|
|
"xaxis": "x",
|
|
"y": [
|
|
16996,
|
|
5836,
|
|
5354,
|
|
5135,
|
|
4200,
|
|
3663,
|
|
2643,
|
|
2463,
|
|
2386,
|
|
1362,
|
|
1304,
|
|
1297,
|
|
1087,
|
|
1060,
|
|
968,
|
|
943,
|
|
515,
|
|
497,
|
|
332
|
|
],
|
|
"yaxis": "y"
|
|
}
|
|
],
|
|
"layout": {
|
|
"barmode": "relative",
|
|
"height": 400,
|
|
"legend": {
|
|
"tracegroupgap": 0
|
|
},
|
|
"margin": {
|
|
"t": 60
|
|
},
|
|
"template": {
|
|
"data": {
|
|
"bar": [
|
|
{
|
|
"error_x": {
|
|
"color": "#2a3f5f"
|
|
},
|
|
"error_y": {
|
|
"color": "#2a3f5f"
|
|
},
|
|
"marker": {
|
|
"line": {
|
|
"color": "#E5ECF6",
|
|
"width": 0.5
|
|
},
|
|
"pattern": {
|
|
"fillmode": "overlay",
|
|
"size": 10,
|
|
"solidity": 0.2
|
|
}
|
|
},
|
|
"type": "bar"
|
|
}
|
|
],
|
|
"barpolar": [
|
|
{
|
|
"marker": {
|
|
"line": {
|
|
"color": "#E5ECF6",
|
|
"width": 0.5
|
|
},
|
|
"pattern": {
|
|
"fillmode": "overlay",
|
|
"size": 10,
|
|
"solidity": 0.2
|
|
}
|
|
},
|
|
"type": "barpolar"
|
|
}
|
|
],
|
|
"carpet": [
|
|
{
|
|
"aaxis": {
|
|
"endlinecolor": "#2a3f5f",
|
|
"gridcolor": "white",
|
|
"linecolor": "white",
|
|
"minorgridcolor": "white",
|
|
"startlinecolor": "#2a3f5f"
|
|
},
|
|
"baxis": {
|
|
"endlinecolor": "#2a3f5f",
|
|
"gridcolor": "white",
|
|
"linecolor": "white",
|
|
"minorgridcolor": "white",
|
|
"startlinecolor": "#2a3f5f"
|
|
},
|
|
"type": "carpet"
|
|
}
|
|
],
|
|
"choropleth": [
|
|
{
|
|
"colorbar": {
|
|
"outlinewidth": 0,
|
|
"ticks": ""
|
|
},
|
|
"type": "choropleth"
|
|
}
|
|
],
|
|
"contour": [
|
|
{
|
|
"colorbar": {
|
|
"outlinewidth": 0,
|
|
"ticks": ""
|
|
},
|
|
"colorscale": [
|
|
[
|
|
0,
|
|
"#0d0887"
|
|
],
|
|
[
|
|
0.1111111111111111,
|
|
"#46039f"
|
|
],
|
|
[
|
|
0.2222222222222222,
|
|
"#7201a8"
|
|
],
|
|
[
|
|
0.3333333333333333,
|
|
"#9c179e"
|
|
],
|
|
[
|
|
0.4444444444444444,
|
|
"#bd3786"
|
|
],
|
|
[
|
|
0.5555555555555556,
|
|
"#d8576b"
|
|
],
|
|
[
|
|
0.6666666666666666,
|
|
"#ed7953"
|
|
],
|
|
[
|
|
0.7777777777777778,
|
|
"#fb9f3a"
|
|
],
|
|
[
|
|
0.8888888888888888,
|
|
"#fdca26"
|
|
],
|
|
[
|
|
1,
|
|
"#f0f921"
|
|
]
|
|
],
|
|
"type": "contour"
|
|
}
|
|
],
|
|
"contourcarpet": [
|
|
{
|
|
"colorbar": {
|
|
"outlinewidth": 0,
|
|
"ticks": ""
|
|
},
|
|
"type": "contourcarpet"
|
|
}
|
|
],
|
|
"heatmap": [
|
|
{
|
|
"colorbar": {
|
|
"outlinewidth": 0,
|
|
"ticks": ""
|
|
},
|
|
"colorscale": [
|
|
[
|
|
0,
|
|
"#0d0887"
|
|
],
|
|
[
|
|
0.1111111111111111,
|
|
"#46039f"
|
|
],
|
|
[
|
|
0.2222222222222222,
|
|
"#7201a8"
|
|
],
|
|
[
|
|
0.3333333333333333,
|
|
"#9c179e"
|
|
],
|
|
[
|
|
0.4444444444444444,
|
|
"#bd3786"
|
|
],
|
|
[
|
|
0.5555555555555556,
|
|
"#d8576b"
|
|
],
|
|
[
|
|
0.6666666666666666,
|
|
"#ed7953"
|
|
],
|
|
[
|
|
0.7777777777777778,
|
|
"#fb9f3a"
|
|
],
|
|
[
|
|
0.8888888888888888,
|
|
"#fdca26"
|
|
],
|
|
[
|
|
1,
|
|
"#f0f921"
|
|
]
|
|
],
|
|
"type": "heatmap"
|
|
}
|
|
],
|
|
"heatmapgl": [
|
|
{
|
|
"colorbar": {
|
|
"outlinewidth": 0,
|
|
"ticks": ""
|
|
},
|
|
"colorscale": [
|
|
[
|
|
0,
|
|
"#0d0887"
|
|
],
|
|
[
|
|
0.1111111111111111,
|
|
"#46039f"
|
|
],
|
|
[
|
|
0.2222222222222222,
|
|
"#7201a8"
|
|
],
|
|
[
|
|
0.3333333333333333,
|
|
"#9c179e"
|
|
],
|
|
[
|
|
0.4444444444444444,
|
|
"#bd3786"
|
|
],
|
|
[
|
|
0.5555555555555556,
|
|
"#d8576b"
|
|
],
|
|
[
|
|
0.6666666666666666,
|
|
"#ed7953"
|
|
],
|
|
[
|
|
0.7777777777777778,
|
|
"#fb9f3a"
|
|
],
|
|
[
|
|
0.8888888888888888,
|
|
"#fdca26"
|
|
],
|
|
[
|
|
1,
|
|
"#f0f921"
|
|
]
|
|
],
|
|
"type": "heatmapgl"
|
|
}
|
|
],
|
|
"histogram": [
|
|
{
|
|
"marker": {
|
|
"pattern": {
|
|
"fillmode": "overlay",
|
|
"size": 10,
|
|
"solidity": 0.2
|
|
}
|
|
},
|
|
"type": "histogram"
|
|
}
|
|
],
|
|
"histogram2d": [
|
|
{
|
|
"colorbar": {
|
|
"outlinewidth": 0,
|
|
"ticks": ""
|
|
},
|
|
"colorscale": [
|
|
[
|
|
0,
|
|
"#0d0887"
|
|
],
|
|
[
|
|
0.1111111111111111,
|
|
"#46039f"
|
|
],
|
|
[
|
|
0.2222222222222222,
|
|
"#7201a8"
|
|
],
|
|
[
|
|
0.3333333333333333,
|
|
"#9c179e"
|
|
],
|
|
[
|
|
0.4444444444444444,
|
|
"#bd3786"
|
|
],
|
|
[
|
|
0.5555555555555556,
|
|
"#d8576b"
|
|
],
|
|
[
|
|
0.6666666666666666,
|
|
"#ed7953"
|
|
],
|
|
[
|
|
0.7777777777777778,
|
|
"#fb9f3a"
|
|
],
|
|
[
|
|
0.8888888888888888,
|
|
"#fdca26"
|
|
],
|
|
[
|
|
1,
|
|
"#f0f921"
|
|
]
|
|
],
|
|
"type": "histogram2d"
|
|
}
|
|
],
|
|
"histogram2dcontour": [
|
|
{
|
|
"colorbar": {
|
|
"outlinewidth": 0,
|
|
"ticks": ""
|
|
},
|
|
"colorscale": [
|
|
[
|
|
0,
|
|
"#0d0887"
|
|
],
|
|
[
|
|
0.1111111111111111,
|
|
"#46039f"
|
|
],
|
|
[
|
|
0.2222222222222222,
|
|
"#7201a8"
|
|
],
|
|
[
|
|
0.3333333333333333,
|
|
"#9c179e"
|
|
],
|
|
[
|
|
0.4444444444444444,
|
|
"#bd3786"
|
|
],
|
|
[
|
|
0.5555555555555556,
|
|
"#d8576b"
|
|
],
|
|
[
|
|
0.6666666666666666,
|
|
"#ed7953"
|
|
],
|
|
[
|
|
0.7777777777777778,
|
|
"#fb9f3a"
|
|
],
|
|
[
|
|
0.8888888888888888,
|
|
"#fdca26"
|
|
],
|
|
[
|
|
1,
|
|
"#f0f921"
|
|
]
|
|
],
|
|
"type": "histogram2dcontour"
|
|
}
|
|
],
|
|
"mesh3d": [
|
|
{
|
|
"colorbar": {
|
|
"outlinewidth": 0,
|
|
"ticks": ""
|
|
},
|
|
"type": "mesh3d"
|
|
}
|
|
],
|
|
"parcoords": [
|
|
{
|
|
"line": {
|
|
"colorbar": {
|
|
"outlinewidth": 0,
|
|
"ticks": ""
|
|
}
|
|
},
|
|
"type": "parcoords"
|
|
}
|
|
],
|
|
"pie": [
|
|
{
|
|
"automargin": true,
|
|
"type": "pie"
|
|
}
|
|
],
|
|
"scatter": [
|
|
{
|
|
"fillpattern": {
|
|
"fillmode": "overlay",
|
|
"size": 10,
|
|
"solidity": 0.2
|
|
},
|
|
"type": "scatter"
|
|
}
|
|
],
|
|
"scatter3d": [
|
|
{
|
|
"line": {
|
|
"colorbar": {
|
|
"outlinewidth": 0,
|
|
"ticks": ""
|
|
}
|
|
},
|
|
"marker": {
|
|
"colorbar": {
|
|
"outlinewidth": 0,
|
|
"ticks": ""
|
|
}
|
|
},
|
|
"type": "scatter3d"
|
|
}
|
|
],
|
|
"scattercarpet": [
|
|
{
|
|
"marker": {
|
|
"colorbar": {
|
|
"outlinewidth": 0,
|
|
"ticks": ""
|
|
}
|
|
},
|
|
"type": "scattercarpet"
|
|
}
|
|
],
|
|
"scattergeo": [
|
|
{
|
|
"marker": {
|
|
"colorbar": {
|
|
"outlinewidth": 0,
|
|
"ticks": ""
|
|
}
|
|
},
|
|
"type": "scattergeo"
|
|
}
|
|
],
|
|
"scattergl": [
|
|
{
|
|
"marker": {
|
|
"colorbar": {
|
|
"outlinewidth": 0,
|
|
"ticks": ""
|
|
}
|
|
},
|
|
"type": "scattergl"
|
|
}
|
|
],
|
|
"scattermapbox": [
|
|
{
|
|
"marker": {
|
|
"colorbar": {
|
|
"outlinewidth": 0,
|
|
"ticks": ""
|
|
}
|
|
},
|
|
"type": "scattermapbox"
|
|
}
|
|
],
|
|
"scatterpolar": [
|
|
{
|
|
"marker": {
|
|
"colorbar": {
|
|
"outlinewidth": 0,
|
|
"ticks": ""
|
|
}
|
|
},
|
|
"type": "scatterpolar"
|
|
}
|
|
],
|
|
"scatterpolargl": [
|
|
{
|
|
"marker": {
|
|
"colorbar": {
|
|
"outlinewidth": 0,
|
|
"ticks": ""
|
|
}
|
|
},
|
|
"type": "scatterpolargl"
|
|
}
|
|
],
|
|
"scatterternary": [
|
|
{
|
|
"marker": {
|
|
"colorbar": {
|
|
"outlinewidth": 0,
|
|
"ticks": ""
|
|
}
|
|
},
|
|
"type": "scatterternary"
|
|
}
|
|
],
|
|
"surface": [
|
|
{
|
|
"colorbar": {
|
|
"outlinewidth": 0,
|
|
"ticks": ""
|
|
},
|
|
"colorscale": [
|
|
[
|
|
0,
|
|
"#0d0887"
|
|
],
|
|
[
|
|
0.1111111111111111,
|
|
"#46039f"
|
|
],
|
|
[
|
|
0.2222222222222222,
|
|
"#7201a8"
|
|
],
|
|
[
|
|
0.3333333333333333,
|
|
"#9c179e"
|
|
],
|
|
[
|
|
0.4444444444444444,
|
|
"#bd3786"
|
|
],
|
|
[
|
|
0.5555555555555556,
|
|
"#d8576b"
|
|
],
|
|
[
|
|
0.6666666666666666,
|
|
"#ed7953"
|
|
],
|
|
[
|
|
0.7777777777777778,
|
|
"#fb9f3a"
|
|
],
|
|
[
|
|
0.8888888888888888,
|
|
"#fdca26"
|
|
],
|
|
[
|
|
1,
|
|
"#f0f921"
|
|
]
|
|
],
|
|
"type": "surface"
|
|
}
|
|
],
|
|
"table": [
|
|
{
|
|
"cells": {
|
|
"fill": {
|
|
"color": "#EBF0F8"
|
|
},
|
|
"line": {
|
|
"color": "white"
|
|
}
|
|
},
|
|
"header": {
|
|
"fill": {
|
|
"color": "#C8D4E3"
|
|
},
|
|
"line": {
|
|
"color": "white"
|
|
}
|
|
},
|
|
"type": "table"
|
|
}
|
|
]
|
|
},
|
|
"layout": {
|
|
"annotationdefaults": {
|
|
"arrowcolor": "#2a3f5f",
|
|
"arrowhead": 0,
|
|
"arrowwidth": 1
|
|
},
|
|
"autotypenumbers": "strict",
|
|
"coloraxis": {
|
|
"colorbar": {
|
|
"outlinewidth": 0,
|
|
"ticks": ""
|
|
}
|
|
},
|
|
"colorscale": {
|
|
"diverging": [
|
|
[
|
|
0,
|
|
"#8e0152"
|
|
],
|
|
[
|
|
0.1,
|
|
"#c51b7d"
|
|
],
|
|
[
|
|
0.2,
|
|
"#de77ae"
|
|
],
|
|
[
|
|
0.3,
|
|
"#f1b6da"
|
|
],
|
|
[
|
|
0.4,
|
|
"#fde0ef"
|
|
],
|
|
[
|
|
0.5,
|
|
"#f7f7f7"
|
|
],
|
|
[
|
|
0.6,
|
|
"#e6f5d0"
|
|
],
|
|
[
|
|
0.7,
|
|
"#b8e186"
|
|
],
|
|
[
|
|
0.8,
|
|
"#7fbc41"
|
|
],
|
|
[
|
|
0.9,
|
|
"#4d9221"
|
|
],
|
|
[
|
|
1,
|
|
"#276419"
|
|
]
|
|
],
|
|
"sequential": [
|
|
[
|
|
0,
|
|
"#0d0887"
|
|
],
|
|
[
|
|
0.1111111111111111,
|
|
"#46039f"
|
|
],
|
|
[
|
|
0.2222222222222222,
|
|
"#7201a8"
|
|
],
|
|
[
|
|
0.3333333333333333,
|
|
"#9c179e"
|
|
],
|
|
[
|
|
0.4444444444444444,
|
|
"#bd3786"
|
|
],
|
|
[
|
|
0.5555555555555556,
|
|
"#d8576b"
|
|
],
|
|
[
|
|
0.6666666666666666,
|
|
"#ed7953"
|
|
],
|
|
[
|
|
0.7777777777777778,
|
|
"#fb9f3a"
|
|
],
|
|
[
|
|
0.8888888888888888,
|
|
"#fdca26"
|
|
],
|
|
[
|
|
1,
|
|
"#f0f921"
|
|
]
|
|
],
|
|
"sequentialminus": [
|
|
[
|
|
0,
|
|
"#0d0887"
|
|
],
|
|
[
|
|
0.1111111111111111,
|
|
"#46039f"
|
|
],
|
|
[
|
|
0.2222222222222222,
|
|
"#7201a8"
|
|
],
|
|
[
|
|
0.3333333333333333,
|
|
"#9c179e"
|
|
],
|
|
[
|
|
0.4444444444444444,
|
|
"#bd3786"
|
|
],
|
|
[
|
|
0.5555555555555556,
|
|
"#d8576b"
|
|
],
|
|
[
|
|
0.6666666666666666,
|
|
"#ed7953"
|
|
],
|
|
[
|
|
0.7777777777777778,
|
|
"#fb9f3a"
|
|
],
|
|
[
|
|
0.8888888888888888,
|
|
"#fdca26"
|
|
],
|
|
[
|
|
1,
|
|
"#f0f921"
|
|
]
|
|
]
|
|
},
|
|
"colorway": [
|
|
"#636efa",
|
|
"#EF553B",
|
|
"#00cc96",
|
|
"#ab63fa",
|
|
"#FFA15A",
|
|
"#19d3f3",
|
|
"#FF6692",
|
|
"#B6E880",
|
|
"#FF97FF",
|
|
"#FECB52"
|
|
],
|
|
"font": {
|
|
"color": "#2a3f5f"
|
|
},
|
|
"geo": {
|
|
"bgcolor": "white",
|
|
"lakecolor": "white",
|
|
"landcolor": "#E5ECF6",
|
|
"showlakes": true,
|
|
"showland": true,
|
|
"subunitcolor": "white"
|
|
},
|
|
"hoverlabel": {
|
|
"align": "left"
|
|
},
|
|
"hovermode": "closest",
|
|
"mapbox": {
|
|
"style": "light"
|
|
},
|
|
"paper_bgcolor": "white",
|
|
"plot_bgcolor": "#E5ECF6",
|
|
"polar": {
|
|
"angularaxis": {
|
|
"gridcolor": "white",
|
|
"linecolor": "white",
|
|
"ticks": ""
|
|
},
|
|
"bgcolor": "#E5ECF6",
|
|
"radialaxis": {
|
|
"gridcolor": "white",
|
|
"linecolor": "white",
|
|
"ticks": ""
|
|
}
|
|
},
|
|
"scene": {
|
|
"xaxis": {
|
|
"backgroundcolor": "#E5ECF6",
|
|
"gridcolor": "white",
|
|
"gridwidth": 2,
|
|
"linecolor": "white",
|
|
"showbackground": true,
|
|
"ticks": "",
|
|
"zerolinecolor": "white"
|
|
},
|
|
"yaxis": {
|
|
"backgroundcolor": "#E5ECF6",
|
|
"gridcolor": "white",
|
|
"gridwidth": 2,
|
|
"linecolor": "white",
|
|
"showbackground": true,
|
|
"ticks": "",
|
|
"zerolinecolor": "white"
|
|
},
|
|
"zaxis": {
|
|
"backgroundcolor": "#E5ECF6",
|
|
"gridcolor": "white",
|
|
"gridwidth": 2,
|
|
"linecolor": "white",
|
|
"showbackground": true,
|
|
"ticks": "",
|
|
"zerolinecolor": "white"
|
|
}
|
|
},
|
|
"shapedefaults": {
|
|
"line": {
|
|
"color": "#2a3f5f"
|
|
}
|
|
},
|
|
"ternary": {
|
|
"aaxis": {
|
|
"gridcolor": "white",
|
|
"linecolor": "white",
|
|
"ticks": ""
|
|
},
|
|
"baxis": {
|
|
"gridcolor": "white",
|
|
"linecolor": "white",
|
|
"ticks": ""
|
|
},
|
|
"bgcolor": "#E5ECF6",
|
|
"caxis": {
|
|
"gridcolor": "white",
|
|
"linecolor": "white",
|
|
"ticks": ""
|
|
}
|
|
},
|
|
"title": {
|
|
"x": 0.05
|
|
},
|
|
"xaxis": {
|
|
"automargin": true,
|
|
"gridcolor": "white",
|
|
"linecolor": "white",
|
|
"ticks": "",
|
|
"title": {
|
|
"standoff": 15
|
|
},
|
|
"zerolinecolor": "white",
|
|
"zerolinewidth": 2
|
|
},
|
|
"yaxis": {
|
|
"automargin": true,
|
|
"gridcolor": "white",
|
|
"linecolor": "white",
|
|
"ticks": "",
|
|
"title": {
|
|
"standoff": 15
|
|
},
|
|
"zerolinecolor": "white",
|
|
"zerolinewidth": 2
|
|
}
|
|
}
|
|
},
|
|
"width": 1200,
|
|
"xaxis": {
|
|
"anchor": "y",
|
|
"domain": [
|
|
0,
|
|
1
|
|
],
|
|
"title": {
|
|
"text": "x"
|
|
}
|
|
},
|
|
"yaxis": {
|
|
"anchor": "x",
|
|
"domain": [
|
|
0,
|
|
1
|
|
],
|
|
"title": {
|
|
"text": "y"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"import pandas as pd\n",
|
|
"from collections import Counter\n",
|
|
"import plotly.express as px\n",
|
|
"\n",
|
|
"df = pd.DataFrame(Counter(y).most_common(), columns=[\"domain\", \"count\"])\n",
|
|
"px.bar(x=df[\"domain\"], y=df[\"count\"], width=1200, height=400).show()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Optimise and train Multinomial Naive Bayes classifier"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from sklearn.naive_bayes import MultinomialNB\n",
|
|
"from sklearn.pipeline import Pipeline\n",
|
|
"from sklearn.feature_extraction.text import TfidfVectorizer\n",
|
|
"\n",
|
|
"\n",
|
|
"def create_pipeline() -> Pipeline:\n",
|
|
" return Pipeline(\n",
|
|
" steps=[\n",
|
|
" (\"vectorizer\", TfidfVectorizer(sublinear_tf=True)),\n",
|
|
" (\"classifier\", MultinomialNB()),\n",
|
|
" ]\n",
|
|
" )"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Fitting 3 folds for each of 24 candidates, totalling 72 fits\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<div>\n",
|
|
"<style scoped>\n",
|
|
" .dataframe tbody tr th:only-of-type {\n",
|
|
" vertical-align: middle;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe tbody tr th {\n",
|
|
" vertical-align: top;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe thead th {\n",
|
|
" text-align: right;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>mean_fit_time</th>\n",
|
|
" <th>std_fit_time</th>\n",
|
|
" <th>mean_score_time</th>\n",
|
|
" <th>std_score_time</th>\n",
|
|
" <th>param_classifier__alpha</th>\n",
|
|
" <th>param_classifier__fit_prior</th>\n",
|
|
" <th>param_vectorizer__max_df</th>\n",
|
|
" <th>param_vectorizer__min_df</th>\n",
|
|
" <th>params</th>\n",
|
|
" <th>split0_test_score</th>\n",
|
|
" <th>split1_test_score</th>\n",
|
|
" <th>split2_test_score</th>\n",
|
|
" <th>mean_test_score</th>\n",
|
|
" <th>std_test_score</th>\n",
|
|
" <th>rank_test_score</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>7</th>\n",
|
|
" <td>7.796924</td>\n",
|
|
" <td>0.321314</td>\n",
|
|
" <td>3.756043</td>\n",
|
|
" <td>0.027860</td>\n",
|
|
" <td>0.5</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>0.05</td>\n",
|
|
" <td>20</td>\n",
|
|
" <td>{'classifier__alpha': 0.5, 'classifier__fit_pr...</td>\n",
|
|
" <td>0.508013</td>\n",
|
|
" <td>0.509086</td>\n",
|
|
" <td>0.514455</td>\n",
|
|
" <td>0.510518</td>\n",
|
|
" <td>0.002818</td>\n",
|
|
" <td>1</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>10</th>\n",
|
|
" <td>8.055664</td>\n",
|
|
" <td>0.206984</td>\n",
|
|
" <td>3.748517</td>\n",
|
|
" <td>0.088012</td>\n",
|
|
" <td>0.5</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>0.1</td>\n",
|
|
" <td>20</td>\n",
|
|
" <td>{'classifier__alpha': 0.5, 'classifier__fit_pr...</td>\n",
|
|
" <td>0.503729</td>\n",
|
|
" <td>0.506417</td>\n",
|
|
" <td>0.511895</td>\n",
|
|
" <td>0.507347</td>\n",
|
|
" <td>0.003398</td>\n",
|
|
" <td>2</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>11</th>\n",
|
|
" <td>7.748360</td>\n",
|
|
" <td>0.484361</td>\n",
|
|
" <td>3.863216</td>\n",
|
|
" <td>0.072048</td>\n",
|
|
" <td>0.5</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>0.1</td>\n",
|
|
" <td>100</td>\n",
|
|
" <td>{'classifier__alpha': 0.5, 'classifier__fit_pr...</td>\n",
|
|
" <td>0.502211</td>\n",
|
|
" <td>0.498949</td>\n",
|
|
" <td>0.503744</td>\n",
|
|
" <td>0.501635</td>\n",
|
|
" <td>0.002000</td>\n",
|
|
" <td>3</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>8</th>\n",
|
|
" <td>7.400649</td>\n",
|
|
" <td>0.087320</td>\n",
|
|
" <td>3.658442</td>\n",
|
|
" <td>0.011735</td>\n",
|
|
" <td>0.5</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>0.05</td>\n",
|
|
" <td>100</td>\n",
|
|
" <td>{'classifier__alpha': 0.5, 'classifier__fit_pr...</td>\n",
|
|
" <td>0.501432</td>\n",
|
|
" <td>0.493970</td>\n",
|
|
" <td>0.501386</td>\n",
|
|
" <td>0.498929</td>\n",
|
|
" <td>0.003507</td>\n",
|
|
" <td>4</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>19</th>\n",
|
|
" <td>8.147969</td>\n",
|
|
" <td>0.401980</td>\n",
|
|
" <td>3.977119</td>\n",
|
|
" <td>0.284028</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>0.05</td>\n",
|
|
" <td>20</td>\n",
|
|
" <td>{'classifier__alpha': 1, 'classifier__fit_prio...</td>\n",
|
|
" <td>0.486410</td>\n",
|
|
" <td>0.491891</td>\n",
|
|
" <td>0.492515</td>\n",
|
|
" <td>0.490272</td>\n",
|
|
" <td>0.002743</td>\n",
|
|
" <td>5</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>20</th>\n",
|
|
" <td>7.472414</td>\n",
|
|
" <td>0.130320</td>\n",
|
|
" <td>3.771136</td>\n",
|
|
" <td>0.146406</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>0.05</td>\n",
|
|
" <td>100</td>\n",
|
|
" <td>{'classifier__alpha': 1, 'classifier__fit_prio...</td>\n",
|
|
" <td>0.486868</td>\n",
|
|
" <td>0.489142</td>\n",
|
|
" <td>0.492665</td>\n",
|
|
" <td>0.489558</td>\n",
|
|
" <td>0.002385</td>\n",
|
|
" <td>6</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>23</th>\n",
|
|
" <td>7.395585</td>\n",
|
|
" <td>0.326162</td>\n",
|
|
" <td>2.332031</td>\n",
|
|
" <td>0.254146</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>0.1</td>\n",
|
|
" <td>100</td>\n",
|
|
" <td>{'classifier__alpha': 1, 'classifier__fit_prio...</td>\n",
|
|
" <td>0.489489</td>\n",
|
|
" <td>0.489987</td>\n",
|
|
" <td>0.488543</td>\n",
|
|
" <td>0.489340</td>\n",
|
|
" <td>0.000599</td>\n",
|
|
" <td>7</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>22</th>\n",
|
|
" <td>7.452060</td>\n",
|
|
" <td>0.162072</td>\n",
|
|
" <td>2.937473</td>\n",
|
|
" <td>0.116443</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>0.1</td>\n",
|
|
" <td>20</td>\n",
|
|
" <td>{'classifier__alpha': 1, 'classifier__fit_prio...</td>\n",
|
|
" <td>0.478748</td>\n",
|
|
" <td>0.485174</td>\n",
|
|
" <td>0.484685</td>\n",
|
|
" <td>0.482869</td>\n",
|
|
" <td>0.002921</td>\n",
|
|
" <td>8</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>6</th>\n",
|
|
" <td>7.836380</td>\n",
|
|
" <td>0.374669</td>\n",
|
|
" <td>4.007429</td>\n",
|
|
" <td>0.251199</td>\n",
|
|
" <td>0.5</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>0.05</td>\n",
|
|
" <td>5</td>\n",
|
|
" <td>{'classifier__alpha': 0.5, 'classifier__fit_pr...</td>\n",
|
|
" <td>0.472793</td>\n",
|
|
" <td>0.476460</td>\n",
|
|
" <td>0.479583</td>\n",
|
|
" <td>0.476279</td>\n",
|
|
" <td>0.002775</td>\n",
|
|
" <td>9</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td>7.839444</td>\n",
|
|
" <td>0.174964</td>\n",
|
|
" <td>3.914105</td>\n",
|
|
" <td>0.379735</td>\n",
|
|
" <td>0.5</td>\n",
|
|
" <td>True</td>\n",
|
|
" <td>0.05</td>\n",
|
|
" <td>100</td>\n",
|
|
" <td>{'classifier__alpha': 0.5, 'classifier__fit_pr...</td>\n",
|
|
" <td>0.469224</td>\n",
|
|
" <td>0.472179</td>\n",
|
|
" <td>0.476758</td>\n",
|
|
" <td>0.472720</td>\n",
|
|
" <td>0.003100</td>\n",
|
|
" <td>10</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>5</th>\n",
|
|
" <td>7.948454</td>\n",
|
|
" <td>0.411364</td>\n",
|
|
" <td>3.968444</td>\n",
|
|
" <td>0.090030</td>\n",
|
|
" <td>0.5</td>\n",
|
|
" <td>True</td>\n",
|
|
" <td>0.1</td>\n",
|
|
" <td>100</td>\n",
|
|
" <td>{'classifier__alpha': 0.5, 'classifier__fit_pr...</td>\n",
|
|
" <td>0.469310</td>\n",
|
|
" <td>0.471916</td>\n",
|
|
" <td>0.476900</td>\n",
|
|
" <td>0.472708</td>\n",
|
|
" <td>0.003149</td>\n",
|
|
" <td>11</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>9</th>\n",
|
|
" <td>7.373592</td>\n",
|
|
" <td>0.143028</td>\n",
|
|
" <td>3.777698</td>\n",
|
|
" <td>0.008990</td>\n",
|
|
" <td>0.5</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>0.1</td>\n",
|
|
" <td>5</td>\n",
|
|
" <td>{'classifier__alpha': 0.5, 'classifier__fit_pr...</td>\n",
|
|
" <td>0.463156</td>\n",
|
|
" <td>0.466783</td>\n",
|
|
" <td>0.463460</td>\n",
|
|
" <td>0.464466</td>\n",
|
|
" <td>0.001643</td>\n",
|
|
" <td>12</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td>7.406839</td>\n",
|
|
" <td>0.041973</td>\n",
|
|
" <td>3.838634</td>\n",
|
|
" <td>0.116634</td>\n",
|
|
" <td>0.5</td>\n",
|
|
" <td>True</td>\n",
|
|
" <td>0.05</td>\n",
|
|
" <td>20</td>\n",
|
|
" <td>{'classifier__alpha': 0.5, 'classifier__fit_pr...</td>\n",
|
|
" <td>0.462315</td>\n",
|
|
" <td>0.462975</td>\n",
|
|
" <td>0.463192</td>\n",
|
|
" <td>0.462827</td>\n",
|
|
" <td>0.000373</td>\n",
|
|
" <td>13</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>4</th>\n",
|
|
" <td>7.481533</td>\n",
|
|
" <td>0.224344</td>\n",
|
|
" <td>3.789821</td>\n",
|
|
" <td>0.098098</td>\n",
|
|
" <td>0.5</td>\n",
|
|
" <td>True</td>\n",
|
|
" <td>0.1</td>\n",
|
|
" <td>20</td>\n",
|
|
" <td>{'classifier__alpha': 0.5, 'classifier__fit_pr...</td>\n",
|
|
" <td>0.457504</td>\n",
|
|
" <td>0.460853</td>\n",
|
|
" <td>0.459941</td>\n",
|
|
" <td>0.459433</td>\n",
|
|
" <td>0.001414</td>\n",
|
|
" <td>14</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>14</th>\n",
|
|
" <td>7.749725</td>\n",
|
|
" <td>0.468469</td>\n",
|
|
" <td>4.139534</td>\n",
|
|
" <td>0.140173</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>True</td>\n",
|
|
" <td>0.05</td>\n",
|
|
" <td>100</td>\n",
|
|
" <td>{'classifier__alpha': 1, 'classifier__fit_prio...</td>\n",
|
|
" <td>0.442930</td>\n",
|
|
" <td>0.443183</td>\n",
|
|
" <td>0.449577</td>\n",
|
|
" <td>0.445230</td>\n",
|
|
" <td>0.003076</td>\n",
|
|
" <td>15</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>17</th>\n",
|
|
" <td>7.703846</td>\n",
|
|
" <td>0.357508</td>\n",
|
|
" <td>4.034154</td>\n",
|
|
" <td>0.265509</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>True</td>\n",
|
|
" <td>0.1</td>\n",
|
|
" <td>100</td>\n",
|
|
" <td>{'classifier__alpha': 1, 'classifier__fit_prio...</td>\n",
|
|
" <td>0.439058</td>\n",
|
|
" <td>0.443429</td>\n",
|
|
" <td>0.449336</td>\n",
|
|
" <td>0.443941</td>\n",
|
|
" <td>0.004212</td>\n",
|
|
" <td>16</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>18</th>\n",
|
|
" <td>7.553105</td>\n",
|
|
" <td>0.056077</td>\n",
|
|
" <td>4.094978</td>\n",
|
|
" <td>0.198813</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>0.05</td>\n",
|
|
" <td>5</td>\n",
|
|
" <td>{'classifier__alpha': 1, 'classifier__fit_prio...</td>\n",
|
|
" <td>0.421488</td>\n",
|
|
" <td>0.427530</td>\n",
|
|
" <td>0.422329</td>\n",
|
|
" <td>0.423782</td>\n",
|
|
" <td>0.002672</td>\n",
|
|
" <td>17</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>13</th>\n",
|
|
" <td>7.438030</td>\n",
|
|
" <td>0.237545</td>\n",
|
|
" <td>3.915824</td>\n",
|
|
" <td>0.029823</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>True</td>\n",
|
|
" <td>0.05</td>\n",
|
|
" <td>20</td>\n",
|
|
" <td>{'classifier__alpha': 1, 'classifier__fit_prio...</td>\n",
|
|
" <td>0.404152</td>\n",
|
|
" <td>0.411373</td>\n",
|
|
" <td>0.406858</td>\n",
|
|
" <td>0.407461</td>\n",
|
|
" <td>0.002979</td>\n",
|
|
" <td>18</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>21</th>\n",
|
|
" <td>8.057189</td>\n",
|
|
" <td>0.435052</td>\n",
|
|
" <td>3.305222</td>\n",
|
|
" <td>0.348995</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>0.1</td>\n",
|
|
" <td>5</td>\n",
|
|
" <td>{'classifier__alpha': 1, 'classifier__fit_prio...</td>\n",
|
|
" <td>0.397182</td>\n",
|
|
" <td>0.405247</td>\n",
|
|
" <td>0.401505</td>\n",
|
|
" <td>0.401311</td>\n",
|
|
" <td>0.003295</td>\n",
|
|
" <td>19</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>16</th>\n",
|
|
" <td>7.710748</td>\n",
|
|
" <td>0.559283</td>\n",
|
|
" <td>4.034421</td>\n",
|
|
" <td>0.073522</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>True</td>\n",
|
|
" <td>0.1</td>\n",
|
|
" <td>20</td>\n",
|
|
" <td>{'classifier__alpha': 1, 'classifier__fit_prio...</td>\n",
|
|
" <td>0.392046</td>\n",
|
|
" <td>0.397995</td>\n",
|
|
" <td>0.396114</td>\n",
|
|
" <td>0.395385</td>\n",
|
|
" <td>0.002483</td>\n",
|
|
" <td>20</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td>7.614647</td>\n",
|
|
" <td>0.466252</td>\n",
|
|
" <td>3.898220</td>\n",
|
|
" <td>0.098618</td>\n",
|
|
" <td>0.5</td>\n",
|
|
" <td>True</td>\n",
|
|
" <td>0.05</td>\n",
|
|
" <td>5</td>\n",
|
|
" <td>{'classifier__alpha': 0.5, 'classifier__fit_pr...</td>\n",
|
|
" <td>0.385156</td>\n",
|
|
" <td>0.388866</td>\n",
|
|
" <td>0.386509</td>\n",
|
|
" <td>0.386844</td>\n",
|
|
" <td>0.001533</td>\n",
|
|
" <td>21</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>3</th>\n",
|
|
" <td>8.176893</td>\n",
|
|
" <td>0.252821</td>\n",
|
|
" <td>4.071952</td>\n",
|
|
" <td>0.265773</td>\n",
|
|
" <td>0.5</td>\n",
|
|
" <td>True</td>\n",
|
|
" <td>0.1</td>\n",
|
|
" <td>5</td>\n",
|
|
" <td>{'classifier__alpha': 0.5, 'classifier__fit_pr...</td>\n",
|
|
" <td>0.371079</td>\n",
|
|
" <td>0.377072</td>\n",
|
|
" <td>0.374531</td>\n",
|
|
" <td>0.374228</td>\n",
|
|
" <td>0.002456</td>\n",
|
|
" <td>22</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>12</th>\n",
|
|
" <td>7.606435</td>\n",
|
|
" <td>0.239001</td>\n",
|
|
" <td>3.875793</td>\n",
|
|
" <td>0.109225</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>True</td>\n",
|
|
" <td>0.05</td>\n",
|
|
" <td>5</td>\n",
|
|
" <td>{'classifier__alpha': 1, 'classifier__fit_prio...</td>\n",
|
|
" <td>0.277031</td>\n",
|
|
" <td>0.288065</td>\n",
|
|
" <td>0.287913</td>\n",
|
|
" <td>0.284336</td>\n",
|
|
" <td>0.005166</td>\n",
|
|
" <td>23</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>15</th>\n",
|
|
" <td>8.077971</td>\n",
|
|
" <td>0.733700</td>\n",
|
|
" <td>4.135721</td>\n",
|
|
" <td>0.235307</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>True</td>\n",
|
|
" <td>0.1</td>\n",
|
|
" <td>5</td>\n",
|
|
" <td>{'classifier__alpha': 1, 'classifier__fit_prio...</td>\n",
|
|
" <td>0.260415</td>\n",
|
|
" <td>0.267201</td>\n",
|
|
" <td>0.266981</td>\n",
|
|
" <td>0.264866</td>\n",
|
|
" <td>0.003148</td>\n",
|
|
" <td>24</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"text/plain": [
|
|
" mean_fit_time std_fit_time mean_score_time std_score_time \\\n",
|
|
"7 7.796924 0.321314 3.756043 0.027860 \n",
|
|
"10 8.055664 0.206984 3.748517 0.088012 \n",
|
|
"11 7.748360 0.484361 3.863216 0.072048 \n",
|
|
"8 7.400649 0.087320 3.658442 0.011735 \n",
|
|
"19 8.147969 0.401980 3.977119 0.284028 \n",
|
|
"20 7.472414 0.130320 3.771136 0.146406 \n",
|
|
"23 7.395585 0.326162 2.332031 0.254146 \n",
|
|
"22 7.452060 0.162072 2.937473 0.116443 \n",
|
|
"6 7.836380 0.374669 4.007429 0.251199 \n",
|
|
"2 7.839444 0.174964 3.914105 0.379735 \n",
|
|
"5 7.948454 0.411364 3.968444 0.090030 \n",
|
|
"9 7.373592 0.143028 3.777698 0.008990 \n",
|
|
"1 7.406839 0.041973 3.838634 0.116634 \n",
|
|
"4 7.481533 0.224344 3.789821 0.098098 \n",
|
|
"14 7.749725 0.468469 4.139534 0.140173 \n",
|
|
"17 7.703846 0.357508 4.034154 0.265509 \n",
|
|
"18 7.553105 0.056077 4.094978 0.198813 \n",
|
|
"13 7.438030 0.237545 3.915824 0.029823 \n",
|
|
"21 8.057189 0.435052 3.305222 0.348995 \n",
|
|
"16 7.710748 0.559283 4.034421 0.073522 \n",
|
|
"0 7.614647 0.466252 3.898220 0.098618 \n",
|
|
"3 8.176893 0.252821 4.071952 0.265773 \n",
|
|
"12 7.606435 0.239001 3.875793 0.109225 \n",
|
|
"15 8.077971 0.733700 4.135721 0.235307 \n",
|
|
"\n",
|
|
" param_classifier__alpha param_classifier__fit_prior \\\n",
|
|
"7 0.5 False \n",
|
|
"10 0.5 False \n",
|
|
"11 0.5 False \n",
|
|
"8 0.5 False \n",
|
|
"19 1 False \n",
|
|
"20 1 False \n",
|
|
"23 1 False \n",
|
|
"22 1 False \n",
|
|
"6 0.5 False \n",
|
|
"2 0.5 True \n",
|
|
"5 0.5 True \n",
|
|
"9 0.5 False \n",
|
|
"1 0.5 True \n",
|
|
"4 0.5 True \n",
|
|
"14 1 True \n",
|
|
"17 1 True \n",
|
|
"18 1 False \n",
|
|
"13 1 True \n",
|
|
"21 1 False \n",
|
|
"16 1 True \n",
|
|
"0 0.5 True \n",
|
|
"3 0.5 True \n",
|
|
"12 1 True \n",
|
|
"15 1 True \n",
|
|
"\n",
|
|
" param_vectorizer__max_df param_vectorizer__min_df \\\n",
|
|
"7 0.05 20 \n",
|
|
"10 0.1 20 \n",
|
|
"11 0.1 100 \n",
|
|
"8 0.05 100 \n",
|
|
"19 0.05 20 \n",
|
|
"20 0.05 100 \n",
|
|
"23 0.1 100 \n",
|
|
"22 0.1 20 \n",
|
|
"6 0.05 5 \n",
|
|
"2 0.05 100 \n",
|
|
"5 0.1 100 \n",
|
|
"9 0.1 5 \n",
|
|
"1 0.05 20 \n",
|
|
"4 0.1 20 \n",
|
|
"14 0.05 100 \n",
|
|
"17 0.1 100 \n",
|
|
"18 0.05 5 \n",
|
|
"13 0.05 20 \n",
|
|
"21 0.1 5 \n",
|
|
"16 0.1 20 \n",
|
|
"0 0.05 5 \n",
|
|
"3 0.1 5 \n",
|
|
"12 0.05 5 \n",
|
|
"15 0.1 5 \n",
|
|
"\n",
|
|
" params split0_test_score \\\n",
|
|
"7 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.508013 \n",
|
|
"10 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.503729 \n",
|
|
"11 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.502211 \n",
|
|
"8 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.501432 \n",
|
|
"19 {'classifier__alpha': 1, 'classifier__fit_prio... 0.486410 \n",
|
|
"20 {'classifier__alpha': 1, 'classifier__fit_prio... 0.486868 \n",
|
|
"23 {'classifier__alpha': 1, 'classifier__fit_prio... 0.489489 \n",
|
|
"22 {'classifier__alpha': 1, 'classifier__fit_prio... 0.478748 \n",
|
|
"6 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.472793 \n",
|
|
"2 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.469224 \n",
|
|
"5 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.469310 \n",
|
|
"9 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.463156 \n",
|
|
"1 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.462315 \n",
|
|
"4 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.457504 \n",
|
|
"14 {'classifier__alpha': 1, 'classifier__fit_prio... 0.442930 \n",
|
|
"17 {'classifier__alpha': 1, 'classifier__fit_prio... 0.439058 \n",
|
|
"18 {'classifier__alpha': 1, 'classifier__fit_prio... 0.421488 \n",
|
|
"13 {'classifier__alpha': 1, 'classifier__fit_prio... 0.404152 \n",
|
|
"21 {'classifier__alpha': 1, 'classifier__fit_prio... 0.397182 \n",
|
|
"16 {'classifier__alpha': 1, 'classifier__fit_prio... 0.392046 \n",
|
|
"0 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.385156 \n",
|
|
"3 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.371079 \n",
|
|
"12 {'classifier__alpha': 1, 'classifier__fit_prio... 0.277031 \n",
|
|
"15 {'classifier__alpha': 1, 'classifier__fit_prio... 0.260415 \n",
|
|
"\n",
|
|
" split1_test_score split2_test_score mean_test_score std_test_score \\\n",
|
|
"7 0.509086 0.514455 0.510518 0.002818 \n",
|
|
"10 0.506417 0.511895 0.507347 0.003398 \n",
|
|
"11 0.498949 0.503744 0.501635 0.002000 \n",
|
|
"8 0.493970 0.501386 0.498929 0.003507 \n",
|
|
"19 0.491891 0.492515 0.490272 0.002743 \n",
|
|
"20 0.489142 0.492665 0.489558 0.002385 \n",
|
|
"23 0.489987 0.488543 0.489340 0.000599 \n",
|
|
"22 0.485174 0.484685 0.482869 0.002921 \n",
|
|
"6 0.476460 0.479583 0.476279 0.002775 \n",
|
|
"2 0.472179 0.476758 0.472720 0.003100 \n",
|
|
"5 0.471916 0.476900 0.472708 0.003149 \n",
|
|
"9 0.466783 0.463460 0.464466 0.001643 \n",
|
|
"1 0.462975 0.463192 0.462827 0.000373 \n",
|
|
"4 0.460853 0.459941 0.459433 0.001414 \n",
|
|
"14 0.443183 0.449577 0.445230 0.003076 \n",
|
|
"17 0.443429 0.449336 0.443941 0.004212 \n",
|
|
"18 0.427530 0.422329 0.423782 0.002672 \n",
|
|
"13 0.411373 0.406858 0.407461 0.002979 \n",
|
|
"21 0.405247 0.401505 0.401311 0.003295 \n",
|
|
"16 0.397995 0.396114 0.395385 0.002483 \n",
|
|
"0 0.388866 0.386509 0.386844 0.001533 \n",
|
|
"3 0.377072 0.374531 0.374228 0.002456 \n",
|
|
"12 0.288065 0.287913 0.284336 0.005166 \n",
|
|
"15 0.267201 0.266981 0.264866 0.003148 \n",
|
|
"\n",
|
|
" rank_test_score \n",
|
|
"7 1 \n",
|
|
"10 2 \n",
|
|
"11 3 \n",
|
|
"8 4 \n",
|
|
"19 5 \n",
|
|
"20 6 \n",
|
|
"23 7 \n",
|
|
"22 8 \n",
|
|
"6 9 \n",
|
|
"2 10 \n",
|
|
"5 11 \n",
|
|
"9 12 \n",
|
|
"1 13 \n",
|
|
"4 14 \n",
|
|
"14 15 \n",
|
|
"17 16 \n",
|
|
"18 17 \n",
|
|
"13 18 \n",
|
|
"21 19 \n",
|
|
"16 20 \n",
|
|
"0 21 \n",
|
|
"3 22 \n",
|
|
"12 23 \n",
|
|
"15 24 "
|
|
]
|
|
},
|
|
"execution_count": 6,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"from sklearn.model_selection import GridSearchCV\n",
|
|
"\n",
|
|
"optimisation_pipeline = GridSearchCV(\n",
|
|
" create_pipeline(),\n",
|
|
" {\n",
|
|
" \"vectorizer__min_df\": [5, 20, 100],\n",
|
|
" \"vectorizer__max_df\": [0.05, 0.1],\n",
|
|
" \"classifier__alpha\": [0.5, 1],\n",
|
|
" \"classifier__fit_prior\": [True, False],\n",
|
|
" },\n",
|
|
" scoring=\"f1_macro\",\n",
|
|
" cv=3,\n",
|
|
" n_jobs=-1,\n",
|
|
" verbose=1,\n",
|
|
")\n",
|
|
"optimisation_pipeline.fit(X, y)\n",
|
|
"\n",
|
|
"results = pd.DataFrame(optimisation_pipeline.cv_results_)\n",
|
|
"results.sort_values(\"rank_test_score\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 7,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>Pipeline(steps=[('vectorizer',\n",
|
|
" TfidfVectorizer(max_df=0.05, min_df=20, sublinear_tf=True)),\n",
|
|
" ('classifier', MultinomialNB(alpha=0.5, fit_prior=False))])</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[('vectorizer',\n",
|
|
" TfidfVectorizer(max_df=0.05, min_df=20, sublinear_tf=True)),\n",
|
|
" ('classifier', MultinomialNB(alpha=0.5, fit_prior=False))])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">TfidfVectorizer</label><div class=\"sk-toggleable__content\"><pre>TfidfVectorizer(max_df=0.05, min_df=20, sublinear_tf=True)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MultinomialNB</label><div class=\"sk-toggleable__content\"><pre>MultinomialNB(alpha=0.5, fit_prior=False)</pre></div></div></div></div></div></div></div>"
|
|
],
|
|
"text/plain": [
|
|
"Pipeline(steps=[('vectorizer',\n",
|
|
" TfidfVectorizer(max_df=0.05, min_df=20, sublinear_tf=True)),\n",
|
|
" ('classifier', MultinomialNB(alpha=0.5, fit_prior=False))])"
|
|
]
|
|
},
|
|
"execution_count": 7,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"from sklearn import set_config\n",
|
|
"\n",
|
|
"set_config(display=\"diagram\")\n",
|
|
"\n",
|
|
"classifier = create_pipeline()\n",
|
|
"classifier.set_params(**optimisation_pipeline.best_params_)\n",
|
|
"classifier.fit(X, y)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Export the model using GreatAI"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 8,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\u001b[38;5;39m2022-06-19 15:12:58,312 | INFO | Fetching cached versions of small-domain-prediction\u001b[0m\n",
|
|
"\u001b[38;5;39m2022-06-19 15:12:59,027 | INFO | Copying file for small-domain-prediction-12\u001b[0m\n",
|
|
"\u001b[38;5;39m2022-06-19 15:12:59,039 | INFO | Compressing small-domain-prediction-12\u001b[0m\n",
|
|
"\u001b[38;5;39m2022-06-19 15:12:59,842 | INFO | Model small-domain-prediction uploaded with version 12\u001b[0m\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"'small-domain-prediction:12'"
|
|
]
|
|
},
|
|
"execution_count": 8,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"from great_ai import save_model\n",
|
|
"\n",
|
|
"\n",
|
|
"save_model(classifier, key=MODEL_KEY, keep_last_n=5)"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3.10.4 ('.env': venv)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.10.4"
|
|
},
|
|
"orig_nbformat": 4,
|
|
"vscode": {
|
|
"interpreter": {
|
|
"hash": "02dd6d3afbfa9fbbe1037d64ad9014965528a1ccad21929d6e72f466389a68ad"
|
|
}
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|