diff --git a/.vscode/settings.json b/.vscode/settings.json index 98e49e4..ee745e6 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -13,6 +13,7 @@ "iloc", "initialisation", "initialised", + "initialising", "inplace", "ipynb", "joblib", diff --git a/examples/simple/data.ipynb b/examples/simple/data.ipynb index 51419cd..8c55da6 100644 --- a/examples/simple/data.ipynb +++ b/examples/simple/data.ipynb @@ -11,15 +11,6 @@ "> The blue boxes show the steps implemented in this notebook." ] }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "MAX_CHUNK_COUNT = 4" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -31,6 +22,15 @@ "In this case, we download the semantic scholar dataset from a public S3 bucket." ] }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "MAX_CHUNK_COUNT = 1" + ] + }, { "cell_type": "code", "execution_count": 2, @@ -39,7 +39,7 @@ { "data": { "text/plain": [ - "'Processing 4 out of the 6002 available chunks'" + "'Processing 1 out of the 6002 available chunks'" ] }, "execution_count": 2, @@ -49,6 +49,7 @@ ], "source": [ "import urllib.request\n", + "from random import shuffle\n", "\n", "manifest = (\n", " urllib.request.urlopen(\n", @@ -58,7 +59,9 @@ " .decode()\n", ") # a list of available chunks separated by '\\n' characters\n", "\n", - "chunks = manifest.split()[:MAX_CHUNK_COUNT]\n", + "lines = manifest.split()\n", + "shuffle(lines)\n", + "chunks = lines[:MAX_CHUNK_COUNT]\n", "\n", "f\"Processing {len(chunks)} out of the {len(manifest.split())} available chunks\"" ] @@ -83,23 +86,11 @@ "name": "stderr", "output_type": "stream", "text": [ - "\u001b[38;5;226m2022-06-19 14:59:12,562 | WARNING | Limiting concurrency to 4 because there are only 4 chunks\u001b[0m\n", - "\u001b[38;5;39m2022-06-19 14:59:12,563 | INFO | Starting parallel map (concurrency: 4, chunk size: 1)\u001b[0m\n" + "\u001b[38;5;226m2022-06-25 11:20:01,955 | WARNING | Limiting concurrency to 1 because there are only 1 chunks\u001b[0m\n", + "\u001b[38;5;39m2022-06-25 11:20:01,956 | INFO | Starting parallel map (concurrency: 1, chunk size: 1)\u001b[0m\n", + "\u001b[38;5;226m2022-06-25 11:20:01,956 | WARNING | Running in series, there is no reason for parallelism\u001b[0m\n", + "100%|██████████| 1/1 [04:02<00:00, 242.61s/it]\n" ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "ff8fc113515944cfa75127f4aba3d491", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/4 [00:00 You can install MongoDB from [here](https://www.mongodb.com/docs/manual/installation) or [use it as a service](https://www.mongodb.com/cloud/atlas/register)\n" ] }, { @@ -181,9 +169,17 @@ "name": "stderr", "output_type": "stream", "text": [ - "\u001b[38;5;226m2022-06-19 15:03:30,300 | WARNING | Environment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n", - "\u001b[38;5;226m2022-06-19 15:03:30,301 | WARNING | The selected persistence driver (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n", - "\u001b[38;5;39m2022-06-19 15:03:30,301 | INFO | Options: configured ✅\u001b[0m\n" + "\u001b[38;5;226m2022-06-25 11:24:04,668 | WARNING | Environment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n", + "\u001b[38;5;39m2022-06-25 11:24:04,669 | INFO | Found credentials file (/data/projects/great-ai/examples/simple/mongo.ini), initialising MongodbDriver\u001b[0m\n", + "\u001b[38;5;39m2022-06-25 11:24:04,670 | INFO | Found credentials file (/data/projects/great-ai/examples/simple/mongo.ini), initialising LargeFileMongo\u001b[0m\n", + "\u001b[38;5;39m2022-06-25 11:24:04,671 | INFO | Settings: configured ✅\u001b[0m\n", + "\u001b[38;5;39m2022-06-25 11:24:04,672 | INFO | 🔩 tracing_database: MongodbDriver\u001b[0m\n", + "\u001b[38;5;39m2022-06-25 11:24:04,672 | INFO | 🔩 large_file_implementation: LargeFileMongo\u001b[0m\n", + "\u001b[38;5;39m2022-06-25 11:24:04,673 | INFO | 🔩 is_production: False\u001b[0m\n", + "\u001b[38;5;39m2022-06-25 11:24:04,673 | INFO | 🔩 should_log_exception_stack: True\u001b[0m\n", + "\u001b[38;5;39m2022-06-25 11:24:04,674 | INFO | 🔩 prediction_cache_size: 512\u001b[0m\n", + "\u001b[38;5;226m2022-06-25 11:24:04,674 | WARNING | You still need to check whether you follow all best practices before trusting your deployment.\u001b[0m\n", + "\u001b[38;5;226m2022-06-25 11:24:04,674 | WARNING | > Find out more at https://se-ml.github.io/practices/\u001b[0m\n" ] } ], diff --git a/examples/simple/deploy.ipynb b/examples/simple/deploy.ipynb index b946ae5..387ea54 100644 --- a/examples/simple/deploy.ipynb +++ b/examples/simple/deploy.ipynb @@ -23,17 +23,25 @@ "name": "stderr", "output_type": "stream", "text": [ - "\u001b[38;5;226m2022-06-19 15:16:23,856 | WARNING | Environment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n", - "\u001b[38;5;226m2022-06-19 15:16:23,857 | WARNING | The selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n", - "\u001b[38;5;39m2022-06-19 15:16:23,858 | INFO | Options: configured ✅\u001b[0m\n", - "\u001b[38;5;39m2022-06-19 15:16:23,858 | INFO | Fetching cached versions of small-domain-prediction\u001b[0m\n", - "\u001b[38;5;39m2022-06-19 15:16:23,859 | INFO | Latest version of small-domain-prediction is 12 (from versions: 9, 10, 11, 12)\u001b[0m\n", - "\u001b[38;5;39m2022-06-19 15:16:23,860 | INFO | File small-domain-prediction-12 found in cache\u001b[0m\n" + "\u001b[38;5;226m2022-06-24 22:38:15,518 | WARNING | Environment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n", + "\u001b[38;5;39m2022-06-24 22:38:15,518 | INFO | Found credentials file (/data/projects/great-ai/examples/simple/mongo.ini), initialising MongodbDriver\u001b[0m\n", + "\u001b[38;5;39m2022-06-24 22:38:15,519 | INFO | Found credentials file (/data/projects/great-ai/examples/simple/mongo.ini), initialising LargeFileMongo\u001b[0m\n", + "\u001b[38;5;39m2022-06-24 22:38:15,520 | INFO | Settings: configured ✅\u001b[0m\n", + "\u001b[38;5;39m2022-06-24 22:38:15,520 | INFO | 🔩 tracing_database: MongodbDriver\u001b[0m\n", + "\u001b[38;5;39m2022-06-24 22:38:15,520 | INFO | 🔩 large_file_implementation: LargeFileMongo\u001b[0m\n", + "\u001b[38;5;39m2022-06-24 22:38:15,521 | INFO | 🔩 is_production: False\u001b[0m\n", + "\u001b[38;5;39m2022-06-24 22:38:15,521 | INFO | 🔩 should_log_exception_stack: True\u001b[0m\n", + "\u001b[38;5;39m2022-06-24 22:38:15,521 | INFO | 🔩 prediction_cache_size: 512\u001b[0m\n", + "\u001b[38;5;226m2022-06-24 22:38:15,521 | WARNING | You still need to check whether you follow all best practices before trusting your deployment.\u001b[0m\n", + "\u001b[38;5;226m2022-06-24 22:38:15,522 | WARNING | > Find out more at https://se-ml.github.io/practices/\u001b[0m\n", + "\u001b[38;5;39m2022-06-24 22:38:15,531 | INFO | Latest version of small-domain-prediction is 0 (from versions: 0)\u001b[0m\n", + "\u001b[38;5;39m2022-06-24 22:38:15,532 | INFO | File small-domain-prediction-0 found in cache\u001b[0m\n" ] } ], "source": [ "import re\n", + "import numpy as np\n", "from sklearn.pipeline import Pipeline\n", "from great_ai.utilities import clean\n", "from great_ai import (\n", @@ -47,15 +55,14 @@ "\n", "@GreatAI.deploy\n", "@use_model(\"small-domain-prediction\", version=\"latest\")\n", - "@parameter(\"explanation_length\", disable_logging=True)\n", + "@parameter(\"target_confidence\", validator=lambda c: 0 <= c <= 100)\n", "def predict_domain(\n", - " text: str, model: Pipeline, target_confidence: int = 50, explanation_length: int = 5\n", + " text: str, model: Pipeline, target_confidence: int = 50\n", ") -> MultiLabelClassificationOutput:\n", " \"\"\"\n", " Predict the scientific domain of the input text.\n", " Return labels until their sum likelihood is larger than `target_confidence`.\n", " \"\"\"\n", - " assert 0 <= target_confidence <= 100, \"invalid argument\"\n", "\n", " preprocessed = re.sub(r\"[^a-zA-Z\\s]\", \"\", clean(text, convert_to_ascii=True))\n", " features = model.named_steps[\"vectorizer\"].transform([preprocessed])\n", @@ -66,33 +73,42 @@ " results = MultiLabelClassificationOutput()\n", " for class_index, probability in best_classes:\n", " results.labels.append(\n", - " ClassificationOutput(\n", - " label=model.named_steps[\"classifier\"].classes_[class_index],\n", - " confidence=round(probability * 100),\n", - " explanation=[\n", - " word\n", - " for _, word in sorted(\n", - " (\n", - " (weight, word)\n", - " for weight, word, count in zip(\n", - " model.named_steps[\"classifier\"].feature_log_prob_[\n", - " class_index\n", - " ],\n", - " model.named_steps[\"vectorizer\"].get_feature_names_out(),\n", - " features.A[0],\n", - " )\n", - " if count > 0\n", - " ),\n", - " reverse=True,\n", - " )\n", - " ][:explanation_length],\n", + " get_label(\n", + " model=model,\n", + " features=features,\n", + " class_index=class_index,\n", + " probability=probability,\n", " )\n", " )\n", "\n", " if sum(r.confidence for r in results.labels) >= target_confidence:\n", " break\n", "\n", - " return results" + " return results\n", + "\n", + "\n", + "def get_label(\n", + " model: Pipeline, features: np.ndarray, class_index: int, probability: float\n", + ") -> ClassificationOutput:\n", + " return ClassificationOutput(\n", + " label=model.named_steps[\"classifier\"].classes_[class_index],\n", + " confidence=round(probability * 100),\n", + " explanation=[\n", + " word\n", + " for _, word in sorted(\n", + " (\n", + " (weight, word)\n", + " for weight, word, count in zip(\n", + " model.named_steps[\"classifier\"].feature_log_prob_[class_index],\n", + " model.named_steps[\"vectorizer\"].get_feature_names_out(),\n", + " features.A[0],\n", + " )\n", + " if count > 0\n", + " ),\n", + " reverse=True,\n", + " )\n", + " ][:5],\n", + " )" ] }, { @@ -104,72 +120,59 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": {}, "outputs": [ { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "\u001b[38;5;226m2022-06-19 15:16:29,300 | WARNING | Limiting concurrency to 10 because there are only 10 chunks\u001b[0m\n", - "\u001b[38;5;39m2022-06-19 15:16:29,301 | INFO | Starting parallel map (concurrency: 10, chunk size: 1)\u001b[0m\n" + "{'filter': {'$and': [{'tags': 'test'}, {'feedback': {'$ne': None}}]}, 'sort': []}\n" ] }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "34e54eabd5f945b8aa2809907964eec8", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/10 [00:00" ] @@ -183,7 +186,7 @@ " from great_ai import query_ground_truth\n", " from sklearn import metrics\n", "\n", - " data = query_ground_truth(\"test\", return_max_count=10)\n", + " data = query_ground_truth(\"test\")\n", "\n", " X = [d.input for d in data]\n", " y_actual = [d.feedback for d in data]\n", @@ -193,7 +196,7 @@ "\n", " import matplotlib.pyplot as plt\n", "\n", - " # Configure matplotlib to have nice, high-resolution charts.\n", + " # Configure matplotlib to have nice, high-resolution charts\n", "\n", " %matplotlib inline\n", "\n", diff --git a/examples/simple/diagrams/ss-confusion.png b/examples/simple/diagrams/ss-confusion.png new file mode 100644 index 0000000..4610198 Binary files /dev/null and b/examples/simple/diagrams/ss-confusion.png differ diff --git a/examples/simple/diagrams/ss-distribution.png b/examples/simple/diagrams/ss-distribution.png new file mode 100644 index 0000000..721684d Binary files /dev/null and b/examples/simple/diagrams/ss-distribution.png differ diff --git a/examples/simple/mongo.ini b/examples/simple/mongo.ini index 3f5dfb1..8151869 100644 --- a/examples/simple/mongo.ini +++ b/examples/simple/mongo.ini @@ -1,3 +1,2 @@ -[DEFAULT] -connection_string=mongodb://localhost:27017/ -database=large_file_tests +mongo_connection_string=mongodb://localhost:27017/ # change this +mongo_database=great_ai_db2 # this will be automatically created diff --git a/examples/simple/train.ipynb b/examples/simple/train.ipynb index e558aa6..4f951f7 100644 --- a/examples/simple/train.ipynb +++ b/examples/simple/train.ipynb @@ -16,15 +16,6 @@ "In this part, we hyperparameter-optimise and train a simple, Naive Bayes classifier which we then export for deployment using `great_ai.save_model`." ] }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "MODEL_KEY = \"small-domain-prediction\"" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -34,16 +25,31 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "\u001b[38;5;226m2022-06-19 15:08:22,338 | WARNING | Environment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n", - "\u001b[38;5;226m2022-06-19 15:08:22,338 | WARNING | The selected persistence driver (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n", - "\u001b[38;5;39m2022-06-19 15:08:22,339 | INFO | Options: configured ✅\u001b[0m\n" + "\u001b[38;5;226m2022-06-25 11:25:08,430 | WARNING | Environment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n", + "\u001b[38;5;39m2022-06-25 11:25:08,431 | INFO | Found credentials file (/data/projects/great-ai/examples/simple/mongo.ini), initialising MongodbDriver\u001b[0m\n", + "\u001b[38;5;39m2022-06-25 11:25:08,432 | INFO | Found credentials file (/data/projects/great-ai/examples/simple/mongo.ini), initialising LargeFileMongo\u001b[0m\n", + "\u001b[38;5;39m2022-06-25 11:25:08,432 | INFO | Settings: configured ✅\u001b[0m\n", + "\u001b[38;5;39m2022-06-25 11:25:08,433 | INFO | 🔩 tracing_database: MongodbDriver\u001b[0m\n", + "\u001b[38;5;39m2022-06-25 11:25:08,433 | INFO | 🔩 large_file_implementation: LargeFileMongo\u001b[0m\n", + "\u001b[38;5;39m2022-06-25 11:25:08,434 | INFO | 🔩 is_production: False\u001b[0m\n", + "\u001b[38;5;39m2022-06-25 11:25:08,434 | INFO | 🔩 should_log_exception_stack: True\u001b[0m\n", + "\u001b[38;5;39m2022-06-25 11:25:08,434 | INFO | 🔩 prediction_cache_size: 512\u001b[0m\n", + "\u001b[38;5;226m2022-06-25 11:25:08,435 | WARNING | You still need to check whether you follow all best practices before trusting your deployment.\u001b[0m\n", + "\u001b[38;5;226m2022-06-25 11:25:08,435 | WARNING | > Find out more at https://se-ml.github.io/practices/\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'filter': {'$and': [{'tags': 'train'}, {'feedback': {'$ne': None}}]}, 'sort': []}\n" ] } ], @@ -57,7 +63,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 2, "metadata": {}, "outputs": [ { @@ -69,7 +75,7 @@ "data": [ { "alignmentgroup": "True", - "hovertemplate": "x=%{x}
y=%{y}", + "hovertemplate": "domain=%{x}
count=%{y}", "legendgroup": "", "marker": { "color": "#636efa", @@ -93,38 +99,38 @@ "Physics", "Psychology", "Mathematics", - "Business", "Political Science", + "Business", "Sociology", "Economics", "Geography", - "Environmental Science", "Geology", - "Art", + "Environmental Science", "History", + "Art", "Philosophy" ], "xaxis": "x", "y": [ - 16996, - 5836, - 5354, - 5135, - 4200, - 3663, - 2643, - 2463, - 2386, - 1362, - 1304, - 1297, - 1087, - 1060, - 968, - 943, - 515, - 497, - 332 + 4409, + 1422, + 1369, + 1352, + 1026, + 887, + 638, + 619, + 582, + 340, + 333, + 303, + 256, + 254, + 229, + 217, + 113, + 112, + 65 ], "yaxis": "y" } @@ -962,7 +968,7 @@ 1 ], "title": { - "text": "x" + "text": "domain" } }, "yaxis": { @@ -972,7 +978,7 @@ 1 ], "title": { - "text": "y" + "text": "count" } } } @@ -988,7 +994,7 @@ "import plotly.express as px\n", "\n", "df = pd.DataFrame(Counter(y).most_common(), columns=[\"domain\", \"count\"])\n", - "px.bar(x=df[\"domain\"], y=df[\"count\"], width=1200, height=400).show()" + "px.bar(df, \"domain\", \"count\", width=1200, height=400).show()" ] }, { @@ -1000,7 +1006,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -1020,7 +1026,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -1071,434 +1077,434 @@ " \n", " \n", " 7\n", - " 7.796924\n", - " 0.321314\n", - " 3.756043\n", - " 0.027860\n", + " 1.986588\n", + " 0.050896\n", + " 1.090251\n", + " 0.135508\n", " 0.5\n", " False\n", " 0.05\n", " 20\n", " {'classifier__alpha': 0.5, 'classifier__fit_pr...\n", - " 0.508013\n", - " 0.509086\n", - " 0.514455\n", - " 0.510518\n", - " 0.002818\n", + " 0.459165\n", + " 0.473024\n", + " 0.475462\n", + " 0.469217\n", + " 0.007177\n", " 1\n", " \n", " \n", " 10\n", - " 8.055664\n", - " 0.206984\n", - " 3.748517\n", - " 0.088012\n", + " 2.070333\n", + " 0.038396\n", + " 0.976315\n", + " 0.033742\n", " 0.5\n", " False\n", " 0.1\n", " 20\n", " {'classifier__alpha': 0.5, 'classifier__fit_pr...\n", - " 0.503729\n", - " 0.506417\n", - " 0.511895\n", - " 0.507347\n", - " 0.003398\n", + " 0.457524\n", + " 0.463575\n", + " 0.458007\n", + " 0.459702\n", + " 0.002745\n", " 2\n", " \n", " \n", - " 11\n", - " 7.748360\n", - " 0.484361\n", - " 3.863216\n", - " 0.072048\n", - " 0.5\n", - " False\n", - " 0.1\n", - " 100\n", - " {'classifier__alpha': 0.5, 'classifier__fit_pr...\n", - " 0.502211\n", - " 0.498949\n", - " 0.503744\n", - " 0.501635\n", - " 0.002000\n", - " 3\n", - " \n", - " \n", - " 8\n", - " 7.400649\n", - " 0.087320\n", - " 3.658442\n", - " 0.011735\n", - " 0.5\n", - " False\n", - " 0.05\n", - " 100\n", - " {'classifier__alpha': 0.5, 'classifier__fit_pr...\n", - " 0.501432\n", - " 0.493970\n", - " 0.501386\n", - " 0.498929\n", - " 0.003507\n", - " 4\n", - " \n", - " \n", " 19\n", - " 8.147969\n", - " 0.401980\n", - " 3.977119\n", - " 0.284028\n", + " 2.049166\n", + " 0.193113\n", + " 1.064576\n", + " 0.087034\n", " 1\n", " False\n", " 0.05\n", " 20\n", " {'classifier__alpha': 1, 'classifier__fit_prio...\n", - " 0.486410\n", - " 0.491891\n", - " 0.492515\n", - " 0.490272\n", - " 0.002743\n", + " 0.441657\n", + " 0.452933\n", + " 0.451286\n", + " 0.448625\n", + " 0.004973\n", + " 3\n", + " \n", + " \n", + " 6\n", + " 2.288145\n", + " 0.131338\n", + " 1.133445\n", + " 0.099583\n", + " 0.5\n", + " False\n", + " 0.05\n", + " 5\n", + " {'classifier__alpha': 0.5, 'classifier__fit_pr...\n", + " 0.436976\n", + " 0.449693\n", + " 0.437911\n", + " 0.441527\n", + " 0.005787\n", + " 4\n", + " \n", + " \n", + " 22\n", + " 1.872234\n", + " 0.072997\n", + " 0.748070\n", + " 0.085882\n", + " 1\n", + " False\n", + " 0.1\n", + " 20\n", + " {'classifier__alpha': 1, 'classifier__fit_prio...\n", + " 0.432322\n", + " 0.438805\n", + " 0.428209\n", + " 0.433112\n", + " 0.004362\n", " 5\n", " \n", " \n", - " 20\n", - " 7.472414\n", - " 0.130320\n", - " 3.771136\n", - " 0.146406\n", - " 1\n", + " 11\n", + " 2.067691\n", + " 0.126923\n", + " 0.910947\n", + " 0.078748\n", + " 0.5\n", " False\n", - " 0.05\n", + " 0.1\n", " 100\n", - " {'classifier__alpha': 1, 'classifier__fit_prio...\n", - " 0.486868\n", - " 0.489142\n", - " 0.492665\n", - " 0.489558\n", - " 0.002385\n", + " {'classifier__alpha': 0.5, 'classifier__fit_pr...\n", + " 0.426436\n", + " 0.429182\n", + " 0.437410\n", + " 0.431009\n", + " 0.004662\n", " 6\n", " \n", " \n", " 23\n", - " 7.395585\n", - " 0.326162\n", - " 2.332031\n", - " 0.254146\n", + " 1.847330\n", + " 0.147504\n", + " 0.495354\n", + " 0.018589\n", " 1\n", " False\n", " 0.1\n", " 100\n", " {'classifier__alpha': 1, 'classifier__fit_prio...\n", - " 0.489489\n", - " 0.489987\n", - " 0.488543\n", - " 0.489340\n", - " 0.000599\n", + " 0.422130\n", + " 0.430875\n", + " 0.430829\n", + " 0.427945\n", + " 0.004112\n", " 7\n", " \n", " \n", - " 22\n", - " 7.452060\n", - " 0.162072\n", - " 2.937473\n", - " 0.116443\n", - " 1\n", + " 9\n", + " 2.071489\n", + " 0.256086\n", + " 1.055936\n", + " 0.037198\n", + " 0.5\n", " False\n", " 0.1\n", - " 20\n", - " {'classifier__alpha': 1, 'classifier__fit_prio...\n", - " 0.478748\n", - " 0.485174\n", - " 0.484685\n", - " 0.482869\n", - " 0.002921\n", + " 5\n", + " {'classifier__alpha': 0.5, 'classifier__fit_pr...\n", + " 0.416746\n", + " 0.425938\n", + " 0.417381\n", + " 0.420022\n", + " 0.004192\n", " 8\n", " \n", " \n", - " 6\n", - " 7.836380\n", - " 0.374669\n", - " 4.007429\n", - " 0.251199\n", - " 0.5\n", + " 20\n", + " 1.776546\n", + " 0.064677\n", + " 0.888485\n", + " 0.093302\n", + " 1\n", " False\n", " 0.05\n", - " 5\n", - " {'classifier__alpha': 0.5, 'classifier__fit_pr...\n", - " 0.472793\n", - " 0.476460\n", - " 0.479583\n", - " 0.476279\n", - " 0.002775\n", + " 100\n", + " {'classifier__alpha': 1, 'classifier__fit_prio...\n", + " 0.413441\n", + " 0.417122\n", + " 0.427196\n", + " 0.419253\n", + " 0.005814\n", " 9\n", " \n", " \n", - " 2\n", - " 7.839444\n", - " 0.174964\n", - " 3.914105\n", - " 0.379735\n", + " 8\n", + " 2.015992\n", + " 0.062583\n", + " 0.974434\n", + " 0.082582\n", " 0.5\n", - " True\n", + " False\n", " 0.05\n", " 100\n", " {'classifier__alpha': 0.5, 'classifier__fit_pr...\n", - " 0.469224\n", - " 0.472179\n", - " 0.476758\n", - " 0.472720\n", - " 0.003100\n", + " 0.412522\n", + " 0.410409\n", + " 0.425047\n", + " 0.415993\n", + " 0.006460\n", " 10\n", " \n", " \n", - " 5\n", - " 7.948454\n", - " 0.411364\n", - " 3.968444\n", - " 0.090030\n", - " 0.5\n", - " True\n", - " 0.1\n", - " 100\n", - " {'classifier__alpha': 0.5, 'classifier__fit_pr...\n", - " 0.469310\n", - " 0.471916\n", - " 0.476900\n", - " 0.472708\n", - " 0.003149\n", + " 18\n", + " 2.032050\n", + " 0.104313\n", + " 0.969210\n", + " 0.070997\n", + " 1\n", + " False\n", + " 0.05\n", + " 5\n", + " {'classifier__alpha': 1, 'classifier__fit_prio...\n", + " 0.392393\n", + " 0.396143\n", + " 0.386439\n", + " 0.391659\n", + " 0.003996\n", " 11\n", " \n", " \n", - " 9\n", - " 7.373592\n", - " 0.143028\n", - " 3.777698\n", - " 0.008990\n", + " 1\n", + " 2.307611\n", + " 0.142902\n", + " 1.102178\n", + " 0.078755\n", " 0.5\n", - " False\n", - " 0.1\n", - " 5\n", + " True\n", + " 0.05\n", + " 20\n", " {'classifier__alpha': 0.5, 'classifier__fit_pr...\n", - " 0.463156\n", - " 0.466783\n", - " 0.463460\n", - " 0.464466\n", - " 0.001643\n", + " 0.385834\n", + " 0.392225\n", + " 0.366549\n", + " 0.381536\n", + " 0.010914\n", " 12\n", " \n", " \n", - " 1\n", - " 7.406839\n", - " 0.041973\n", - " 3.838634\n", - " 0.116634\n", + " 4\n", + " 1.876958\n", + " 0.050648\n", + " 1.031975\n", + " 0.029341\n", " 0.5\n", " True\n", - " 0.05\n", + " 0.1\n", " 20\n", " {'classifier__alpha': 0.5, 'classifier__fit_pr...\n", - " 0.462315\n", - " 0.462975\n", - " 0.463192\n", - " 0.462827\n", - " 0.000373\n", + " 0.377309\n", + " 0.384224\n", + " 0.366632\n", + " 0.376055\n", + " 0.007236\n", " 13\n", " \n", " \n", - " 4\n", - " 7.481533\n", - " 0.224344\n", - " 3.789821\n", - " 0.098098\n", - " 0.5\n", - " True\n", + " 21\n", + " 1.897479\n", + " 0.094903\n", + " 0.740516\n", + " 0.143821\n", + " 1\n", + " False\n", " 0.1\n", - " 20\n", - " {'classifier__alpha': 0.5, 'classifier__fit_pr...\n", - " 0.457504\n", - " 0.460853\n", - " 0.459941\n", - " 0.459433\n", - " 0.001414\n", + " 5\n", + " {'classifier__alpha': 1, 'classifier__fit_prio...\n", + " 0.360551\n", + " 0.366311\n", + " 0.350439\n", + " 0.359101\n", + " 0.006561\n", " 14\n", " \n", " \n", - " 14\n", - " 7.749725\n", - " 0.468469\n", - " 4.139534\n", - " 0.140173\n", - " 1\n", + " 5\n", + " 1.824982\n", + " 0.038113\n", + " 0.960814\n", + " 0.041330\n", + " 0.5\n", " True\n", - " 0.05\n", + " 0.1\n", " 100\n", - " {'classifier__alpha': 1, 'classifier__fit_prio...\n", - " 0.442930\n", - " 0.443183\n", - " 0.449577\n", - " 0.445230\n", - " 0.003076\n", + " {'classifier__alpha': 0.5, 'classifier__fit_pr...\n", + " 0.347831\n", + " 0.350708\n", + " 0.349963\n", + " 0.349501\n", + " 0.001219\n", " 15\n", " \n", " \n", + " 2\n", + " 2.151446\n", + " 0.120440\n", + " 0.887620\n", + " 0.037900\n", + " 0.5\n", + " True\n", + " 0.05\n", + " 100\n", + " {'classifier__alpha': 0.5, 'classifier__fit_pr...\n", + " 0.335093\n", + " 0.340020\n", + " 0.341502\n", + " 0.338872\n", + " 0.002739\n", + " 16\n", + " \n", + " \n", " 17\n", - " 7.703846\n", - " 0.357508\n", - " 4.034154\n", - " 0.265509\n", + " 2.004961\n", + " 0.196640\n", + " 0.953482\n", + " 0.113914\n", " 1\n", " True\n", " 0.1\n", " 100\n", " {'classifier__alpha': 1, 'classifier__fit_prio...\n", - " 0.439058\n", - " 0.443429\n", - " 0.449336\n", - " 0.443941\n", - " 0.004212\n", - " 16\n", - " \n", - " \n", - " 18\n", - " 7.553105\n", - " 0.056077\n", - " 4.094978\n", - " 0.198813\n", - " 1\n", - " False\n", - " 0.05\n", - " 5\n", - " {'classifier__alpha': 1, 'classifier__fit_prio...\n", - " 0.421488\n", - " 0.427530\n", - " 0.422329\n", - " 0.423782\n", - " 0.002672\n", + " 0.300093\n", + " 0.306020\n", + " 0.305047\n", + " 0.303720\n", + " 0.002595\n", " 17\n", " \n", " \n", " 13\n", - " 7.438030\n", - " 0.237545\n", - " 3.915824\n", - " 0.029823\n", + " 1.890709\n", + " 0.056234\n", + " 1.006219\n", + " 0.042628\n", " 1\n", " True\n", " 0.05\n", " 20\n", " {'classifier__alpha': 1, 'classifier__fit_prio...\n", - " 0.404152\n", - " 0.411373\n", - " 0.406858\n", - " 0.407461\n", - " 0.002979\n", + " 0.297119\n", + " 0.302061\n", + " 0.298975\n", + " 0.299385\n", + " 0.002038\n", " 18\n", " \n", " \n", - " 21\n", - " 8.057189\n", - " 0.435052\n", - " 3.305222\n", - " 0.348995\n", + " 14\n", + " 2.022432\n", + " 0.201745\n", + " 0.931837\n", + " 0.047467\n", " 1\n", - " False\n", - " 0.1\n", - " 5\n", + " True\n", + " 0.05\n", + " 100\n", " {'classifier__alpha': 1, 'classifier__fit_prio...\n", - " 0.397182\n", - " 0.405247\n", - " 0.401505\n", - " 0.401311\n", - " 0.003295\n", + " 0.300746\n", + " 0.291492\n", + " 0.297456\n", + " 0.296565\n", + " 0.003830\n", " 19\n", " \n", " \n", - " 16\n", - " 7.710748\n", - " 0.559283\n", - " 4.034421\n", - " 0.073522\n", - " 1\n", - " True\n", - " 0.1\n", - " 20\n", - " {'classifier__alpha': 1, 'classifier__fit_prio...\n", - " 0.392046\n", - " 0.397995\n", - " 0.396114\n", - " 0.395385\n", - " 0.002483\n", - " 20\n", - " \n", - " \n", " 0\n", - " 7.614647\n", - " 0.466252\n", - " 3.898220\n", - " 0.098618\n", + " 2.081572\n", + " 0.123875\n", + " 1.058692\n", + " 0.034084\n", " 0.5\n", " True\n", " 0.05\n", " 5\n", " {'classifier__alpha': 0.5, 'classifier__fit_pr...\n", - " 0.385156\n", - " 0.388866\n", - " 0.386509\n", - " 0.386844\n", - " 0.001533\n", + " 0.296667\n", + " 0.294715\n", + " 0.292144\n", + " 0.294509\n", + " 0.001852\n", + " 20\n", + " \n", + " \n", + " 16\n", + " 1.961700\n", + " 0.108977\n", + " 0.964816\n", + " 0.089735\n", + " 1\n", + " True\n", + " 0.1\n", + " 20\n", + " {'classifier__alpha': 1, 'classifier__fit_prio...\n", + " 0.278389\n", + " 0.287134\n", + " 0.286320\n", + " 0.283948\n", + " 0.003945\n", " 21\n", " \n", " \n", " 3\n", - " 8.176893\n", - " 0.252821\n", - " 4.071952\n", - " 0.265773\n", + " 2.272779\n", + " 0.155897\n", + " 0.982528\n", + " 0.045744\n", " 0.5\n", " True\n", " 0.1\n", " 5\n", " {'classifier__alpha': 0.5, 'classifier__fit_pr...\n", - " 0.371079\n", - " 0.377072\n", - " 0.374531\n", - " 0.374228\n", - " 0.002456\n", + " 0.275833\n", + " 0.272693\n", + " 0.278427\n", + " 0.275651\n", + " 0.002344\n", " 22\n", " \n", " \n", " 12\n", - " 7.606435\n", - " 0.239001\n", - " 3.875793\n", - " 0.109225\n", + " 1.979826\n", + " 0.106524\n", + " 0.938829\n", + " 0.050975\n", " 1\n", " True\n", " 0.05\n", " 5\n", " {'classifier__alpha': 1, 'classifier__fit_prio...\n", - " 0.277031\n", - " 0.288065\n", - " 0.287913\n", - " 0.284336\n", - " 0.005166\n", + " 0.181309\n", + " 0.186707\n", + " 0.193406\n", + " 0.187141\n", + " 0.004948\n", " 23\n", " \n", " \n", " 15\n", - " 8.077971\n", - " 0.733700\n", - " 4.135721\n", - " 0.235307\n", + " 1.952817\n", + " 0.107900\n", + " 0.938851\n", + " 0.048295\n", " 1\n", " True\n", " 0.1\n", " 5\n", " {'classifier__alpha': 1, 'classifier__fit_prio...\n", - " 0.260415\n", - " 0.267201\n", - " 0.266981\n", - " 0.264866\n", - " 0.003148\n", + " 0.160402\n", + " 0.172025\n", + " 0.171455\n", + " 0.167960\n", + " 0.005350\n", " 24\n", " \n", " \n", @@ -1507,53 +1513,53 @@ ], "text/plain": [ " mean_fit_time std_fit_time mean_score_time std_score_time \\\n", - "7 7.796924 0.321314 3.756043 0.027860 \n", - "10 8.055664 0.206984 3.748517 0.088012 \n", - "11 7.748360 0.484361 3.863216 0.072048 \n", - "8 7.400649 0.087320 3.658442 0.011735 \n", - "19 8.147969 0.401980 3.977119 0.284028 \n", - "20 7.472414 0.130320 3.771136 0.146406 \n", - "23 7.395585 0.326162 2.332031 0.254146 \n", - "22 7.452060 0.162072 2.937473 0.116443 \n", - "6 7.836380 0.374669 4.007429 0.251199 \n", - "2 7.839444 0.174964 3.914105 0.379735 \n", - "5 7.948454 0.411364 3.968444 0.090030 \n", - "9 7.373592 0.143028 3.777698 0.008990 \n", - "1 7.406839 0.041973 3.838634 0.116634 \n", - "4 7.481533 0.224344 3.789821 0.098098 \n", - "14 7.749725 0.468469 4.139534 0.140173 \n", - "17 7.703846 0.357508 4.034154 0.265509 \n", - "18 7.553105 0.056077 4.094978 0.198813 \n", - "13 7.438030 0.237545 3.915824 0.029823 \n", - "21 8.057189 0.435052 3.305222 0.348995 \n", - "16 7.710748 0.559283 4.034421 0.073522 \n", - "0 7.614647 0.466252 3.898220 0.098618 \n", - "3 8.176893 0.252821 4.071952 0.265773 \n", - "12 7.606435 0.239001 3.875793 0.109225 \n", - "15 8.077971 0.733700 4.135721 0.235307 \n", + "7 1.986588 0.050896 1.090251 0.135508 \n", + "10 2.070333 0.038396 0.976315 0.033742 \n", + "19 2.049166 0.193113 1.064576 0.087034 \n", + "6 2.288145 0.131338 1.133445 0.099583 \n", + "22 1.872234 0.072997 0.748070 0.085882 \n", + "11 2.067691 0.126923 0.910947 0.078748 \n", + "23 1.847330 0.147504 0.495354 0.018589 \n", + "9 2.071489 0.256086 1.055936 0.037198 \n", + "20 1.776546 0.064677 0.888485 0.093302 \n", + "8 2.015992 0.062583 0.974434 0.082582 \n", + "18 2.032050 0.104313 0.969210 0.070997 \n", + "1 2.307611 0.142902 1.102178 0.078755 \n", + "4 1.876958 0.050648 1.031975 0.029341 \n", + "21 1.897479 0.094903 0.740516 0.143821 \n", + "5 1.824982 0.038113 0.960814 0.041330 \n", + "2 2.151446 0.120440 0.887620 0.037900 \n", + "17 2.004961 0.196640 0.953482 0.113914 \n", + "13 1.890709 0.056234 1.006219 0.042628 \n", + "14 2.022432 0.201745 0.931837 0.047467 \n", + "0 2.081572 0.123875 1.058692 0.034084 \n", + "16 1.961700 0.108977 0.964816 0.089735 \n", + "3 2.272779 0.155897 0.982528 0.045744 \n", + "12 1.979826 0.106524 0.938829 0.050975 \n", + "15 1.952817 0.107900 0.938851 0.048295 \n", "\n", " param_classifier__alpha param_classifier__fit_prior \\\n", "7 0.5 False \n", "10 0.5 False \n", - "11 0.5 False \n", - "8 0.5 False \n", "19 1 False \n", - "20 1 False \n", - "23 1 False \n", - "22 1 False \n", "6 0.5 False \n", - "2 0.5 True \n", - "5 0.5 True \n", + "22 1 False \n", + "11 0.5 False \n", + "23 1 False \n", "9 0.5 False \n", + "20 1 False \n", + "8 0.5 False \n", + "18 1 False \n", "1 0.5 True \n", "4 0.5 True \n", - "14 1 True \n", - "17 1 True \n", - "18 1 False \n", - "13 1 True \n", "21 1 False \n", - "16 1 True \n", + "5 0.5 True \n", + "2 0.5 True \n", + "17 1 True \n", + "13 1 True \n", + "14 1 True \n", "0 0.5 True \n", + "16 1 True \n", "3 0.5 True \n", "12 1 True \n", "15 1 True \n", @@ -1561,109 +1567,109 @@ " param_vectorizer__max_df param_vectorizer__min_df \\\n", "7 0.05 20 \n", "10 0.1 20 \n", - "11 0.1 100 \n", - "8 0.05 100 \n", "19 0.05 20 \n", - "20 0.05 100 \n", - "23 0.1 100 \n", - "22 0.1 20 \n", "6 0.05 5 \n", - "2 0.05 100 \n", - "5 0.1 100 \n", + "22 0.1 20 \n", + "11 0.1 100 \n", + "23 0.1 100 \n", "9 0.1 5 \n", + "20 0.05 100 \n", + "8 0.05 100 \n", + "18 0.05 5 \n", "1 0.05 20 \n", "4 0.1 20 \n", - "14 0.05 100 \n", - "17 0.1 100 \n", - "18 0.05 5 \n", - "13 0.05 20 \n", "21 0.1 5 \n", - "16 0.1 20 \n", + "5 0.1 100 \n", + "2 0.05 100 \n", + "17 0.1 100 \n", + "13 0.05 20 \n", + "14 0.05 100 \n", "0 0.05 5 \n", + "16 0.1 20 \n", "3 0.1 5 \n", "12 0.05 5 \n", "15 0.1 5 \n", "\n", " params split0_test_score \\\n", - "7 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.508013 \n", - "10 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.503729 \n", - "11 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.502211 \n", - "8 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.501432 \n", - "19 {'classifier__alpha': 1, 'classifier__fit_prio... 0.486410 \n", - "20 {'classifier__alpha': 1, 'classifier__fit_prio... 0.486868 \n", - "23 {'classifier__alpha': 1, 'classifier__fit_prio... 0.489489 \n", - "22 {'classifier__alpha': 1, 'classifier__fit_prio... 0.478748 \n", - "6 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.472793 \n", - "2 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.469224 \n", - "5 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.469310 \n", - "9 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.463156 \n", - "1 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.462315 \n", - "4 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.457504 \n", - "14 {'classifier__alpha': 1, 'classifier__fit_prio... 0.442930 \n", - "17 {'classifier__alpha': 1, 'classifier__fit_prio... 0.439058 \n", - "18 {'classifier__alpha': 1, 'classifier__fit_prio... 0.421488 \n", - "13 {'classifier__alpha': 1, 'classifier__fit_prio... 0.404152 \n", - "21 {'classifier__alpha': 1, 'classifier__fit_prio... 0.397182 \n", - "16 {'classifier__alpha': 1, 'classifier__fit_prio... 0.392046 \n", - "0 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.385156 \n", - "3 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.371079 \n", - "12 {'classifier__alpha': 1, 'classifier__fit_prio... 0.277031 \n", - "15 {'classifier__alpha': 1, 'classifier__fit_prio... 0.260415 \n", + "7 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.459165 \n", + "10 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.457524 \n", + "19 {'classifier__alpha': 1, 'classifier__fit_prio... 0.441657 \n", + "6 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.436976 \n", + "22 {'classifier__alpha': 1, 'classifier__fit_prio... 0.432322 \n", + "11 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.426436 \n", + "23 {'classifier__alpha': 1, 'classifier__fit_prio... 0.422130 \n", + "9 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.416746 \n", + "20 {'classifier__alpha': 1, 'classifier__fit_prio... 0.413441 \n", + "8 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.412522 \n", + "18 {'classifier__alpha': 1, 'classifier__fit_prio... 0.392393 \n", + "1 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.385834 \n", + "4 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.377309 \n", + "21 {'classifier__alpha': 1, 'classifier__fit_prio... 0.360551 \n", + "5 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.347831 \n", + "2 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.335093 \n", + "17 {'classifier__alpha': 1, 'classifier__fit_prio... 0.300093 \n", + "13 {'classifier__alpha': 1, 'classifier__fit_prio... 0.297119 \n", + "14 {'classifier__alpha': 1, 'classifier__fit_prio... 0.300746 \n", + "0 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.296667 \n", + "16 {'classifier__alpha': 1, 'classifier__fit_prio... 0.278389 \n", + "3 {'classifier__alpha': 0.5, 'classifier__fit_pr... 0.275833 \n", + "12 {'classifier__alpha': 1, 'classifier__fit_prio... 0.181309 \n", + "15 {'classifier__alpha': 1, 'classifier__fit_prio... 0.160402 \n", "\n", " split1_test_score split2_test_score mean_test_score std_test_score \\\n", - "7 0.509086 0.514455 0.510518 0.002818 \n", - "10 0.506417 0.511895 0.507347 0.003398 \n", - "11 0.498949 0.503744 0.501635 0.002000 \n", - "8 0.493970 0.501386 0.498929 0.003507 \n", - "19 0.491891 0.492515 0.490272 0.002743 \n", - "20 0.489142 0.492665 0.489558 0.002385 \n", - "23 0.489987 0.488543 0.489340 0.000599 \n", - "22 0.485174 0.484685 0.482869 0.002921 \n", - "6 0.476460 0.479583 0.476279 0.002775 \n", - "2 0.472179 0.476758 0.472720 0.003100 \n", - "5 0.471916 0.476900 0.472708 0.003149 \n", - "9 0.466783 0.463460 0.464466 0.001643 \n", - "1 0.462975 0.463192 0.462827 0.000373 \n", - "4 0.460853 0.459941 0.459433 0.001414 \n", - "14 0.443183 0.449577 0.445230 0.003076 \n", - "17 0.443429 0.449336 0.443941 0.004212 \n", - "18 0.427530 0.422329 0.423782 0.002672 \n", - "13 0.411373 0.406858 0.407461 0.002979 \n", - "21 0.405247 0.401505 0.401311 0.003295 \n", - "16 0.397995 0.396114 0.395385 0.002483 \n", - "0 0.388866 0.386509 0.386844 0.001533 \n", - "3 0.377072 0.374531 0.374228 0.002456 \n", - "12 0.288065 0.287913 0.284336 0.005166 \n", - "15 0.267201 0.266981 0.264866 0.003148 \n", + "7 0.473024 0.475462 0.469217 0.007177 \n", + "10 0.463575 0.458007 0.459702 0.002745 \n", + "19 0.452933 0.451286 0.448625 0.004973 \n", + "6 0.449693 0.437911 0.441527 0.005787 \n", + "22 0.438805 0.428209 0.433112 0.004362 \n", + "11 0.429182 0.437410 0.431009 0.004662 \n", + "23 0.430875 0.430829 0.427945 0.004112 \n", + "9 0.425938 0.417381 0.420022 0.004192 \n", + "20 0.417122 0.427196 0.419253 0.005814 \n", + "8 0.410409 0.425047 0.415993 0.006460 \n", + "18 0.396143 0.386439 0.391659 0.003996 \n", + "1 0.392225 0.366549 0.381536 0.010914 \n", + "4 0.384224 0.366632 0.376055 0.007236 \n", + "21 0.366311 0.350439 0.359101 0.006561 \n", + "5 0.350708 0.349963 0.349501 0.001219 \n", + "2 0.340020 0.341502 0.338872 0.002739 \n", + "17 0.306020 0.305047 0.303720 0.002595 \n", + "13 0.302061 0.298975 0.299385 0.002038 \n", + "14 0.291492 0.297456 0.296565 0.003830 \n", + "0 0.294715 0.292144 0.294509 0.001852 \n", + "16 0.287134 0.286320 0.283948 0.003945 \n", + "3 0.272693 0.278427 0.275651 0.002344 \n", + "12 0.186707 0.193406 0.187141 0.004948 \n", + "15 0.172025 0.171455 0.167960 0.005350 \n", "\n", " rank_test_score \n", "7 1 \n", "10 2 \n", - "11 3 \n", - "8 4 \n", - "19 5 \n", - "20 6 \n", + "19 3 \n", + "6 4 \n", + "22 5 \n", + "11 6 \n", "23 7 \n", - "22 8 \n", - "6 9 \n", - "2 10 \n", - "5 11 \n", - "9 12 \n", - "1 13 \n", - "4 14 \n", - "14 15 \n", - "17 16 \n", - "18 17 \n", + "9 8 \n", + "20 9 \n", + "8 10 \n", + "18 11 \n", + "1 12 \n", + "4 13 \n", + "21 14 \n", + "5 15 \n", + "2 16 \n", + "17 17 \n", "13 18 \n", - "21 19 \n", - "16 20 \n", - "0 21 \n", + "14 19 \n", + "0 20 \n", + "16 21 \n", "3 22 \n", "12 23 \n", "15 24 " ] }, - "execution_count": 6, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -1692,7 +1698,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -1710,7 +1716,7 @@ " ('classifier', MultinomialNB(alpha=0.5, fit_prior=False))])" ] }, - "execution_count": 7, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -1734,26 +1740,34 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "\u001b[38;5;39m2022-06-19 15:12:58,312 | INFO | Fetching cached versions of small-domain-prediction\u001b[0m\n", - "\u001b[38;5;39m2022-06-19 15:12:59,027 | INFO | Copying file for small-domain-prediction-12\u001b[0m\n", - "\u001b[38;5;39m2022-06-19 15:12:59,039 | INFO | Compressing small-domain-prediction-12\u001b[0m\n", - "\u001b[38;5;39m2022-06-19 15:12:59,842 | INFO | Model small-domain-prediction uploaded with version 12\u001b[0m\n" + "\u001b[38;5;39m2022-06-25 11:25:32,714 | INFO | Copying file for small-domain-prediction-0\u001b[0m\n", + "\u001b[38;5;39m2022-06-25 11:25:32,737 | INFO | Compressing small-domain-prediction-0\u001b[0m\n", + "\u001b[38;5;39m2022-06-25 11:25:33,050 | INFO | Uploading /tmp/tmpgerx8x95/small-domain-prediction-0.tar.gz to Mongo (GridFS)\u001b[0m\n", + "\u001b[38;5;39m2022-06-25 11:25:33,107 | INFO | Uploading small-domain-prediction-0.tar.gz 0.26/1.85 MB (14.2%)\u001b[0m\n", + "\u001b[38;5;39m2022-06-25 11:25:33,109 | INFO | Uploading small-domain-prediction-0.tar.gz 0.52/1.85 MB (28.3%)\u001b[0m\n", + "\u001b[38;5;39m2022-06-25 11:25:33,112 | INFO | Uploading small-domain-prediction-0.tar.gz 0.78/1.85 MB (42.5%)\u001b[0m\n", + "\u001b[38;5;39m2022-06-25 11:25:33,114 | INFO | Uploading small-domain-prediction-0.tar.gz 1.04/1.85 MB (56.6%)\u001b[0m\n", + "\u001b[38;5;39m2022-06-25 11:25:33,116 | INFO | Uploading small-domain-prediction-0.tar.gz 1.31/1.85 MB (70.8%)\u001b[0m\n", + "\u001b[38;5;39m2022-06-25 11:25:33,117 | INFO | Uploading small-domain-prediction-0.tar.gz 1.57/1.85 MB (84.9%)\u001b[0m\n", + "\u001b[38;5;39m2022-06-25 11:25:33,120 | INFO | Uploading small-domain-prediction-0.tar.gz 1.83/1.85 MB (99.1%)\u001b[0m\n", + "\u001b[38;5;39m2022-06-25 11:25:33,120 | INFO | Uploading small-domain-prediction-0.tar.gz 1.85/1.85 MB (100.0%)\u001b[0m\n", + "\u001b[38;5;39m2022-06-25 11:25:33,124 | INFO | Model small-domain-prediction uploaded with version 0\u001b[0m\n" ] }, { "data": { "text/plain": [ - "'small-domain-prediction:12'" + "'small-domain-prediction:0'" ] }, - "execution_count": 8, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -1762,7 +1776,7 @@ "from great_ai import save_model\n", "\n", "\n", - "save_model(classifier, key=MODEL_KEY, keep_last_n=5)" + "save_model(classifier, key=\"small-domain-prediction\", keep_last_n=5)" ] } ], diff --git a/great_ai/example_secrets.ini b/great_ai/example_secrets.ini index 439d79a..9d8f335 100644 --- a/great_ai/example_secrets.ini +++ b/great_ai/example_secrets.ini @@ -1,4 +1,3 @@ -[DEFAULT] aws_region_name = your_region_like_eu-west-2 aws_access_key_id = YOUR_ACCESS_KEY_ID aws_secret_access_key = YOUR_VERY_SECRET_ACCESS_KEY diff --git a/great_ai/open_s3.md b/great_ai/open_s3.md index 49acb9e..04708e7 100644 --- a/great_ai/open_s3.md +++ b/great_ai/open_s3.md @@ -78,7 +78,6 @@ The package can be used as a module from the command-line to give you more flexi Create an .ini file (or use _~/.aws/credentials_). It may look like this: ```ini -[DEFAULT] aws_region_name = your_region_like_eu-west-2 aws_access_key_id = YOUR_ACCESS_KEY_ID aws_secret_access_key = YOUR_VERY_SECRET_ACCESS_KEY diff --git a/great_ai/setup.cfg b/great_ai/setup.cfg index 1ca9e8f..f733e59 100644 --- a/great_ai/setup.cfg +++ b/great_ai/setup.cfg @@ -20,7 +20,6 @@ packages = find: include_package_data = True python_requires = >=3.8 install_requires = - click < 8.1.0 unidecode >= 1.3.0 multiprocess >= 0.70.0.0 tqdm >= 4.0.0 diff --git a/great_ai/src/great_ai/__main__.py b/great_ai/src/great_ai/__main__.py index 5d1a3d7..b42c462 100644 --- a/great_ai/src/great_ai/__main__.py +++ b/great_ai/src/great_ai/__main__.py @@ -19,7 +19,7 @@ from .great_ai.context import _is_in_production_mode from .great_ai.deploy import GreatAI from .great_ai.exceptions import ArgumentValidationError, MissingArgumentError from .parse_arguments import parse_arguments -from .utilities.logger import get_logger +from .utilities import get_logger logger = get_logger(SERVER_NAME) diff --git a/great_ai/src/great_ai/great_ai/__init__.py b/great_ai/src/great_ai/great_ai/__init__.py index 210d2f3..edf03ff 100644 --- a/great_ai/src/great_ai/great_ai/__init__.py +++ b/great_ai/src/great_ai/great_ai/__init__.py @@ -7,5 +7,5 @@ from .output_models import ( RegressionOutput, ) from .parameters import log_metric, parameter -from .persistence import MongoDbDriver, ParallelTinyDbDriver, TracingDatabaseDriver -from .tracing import add_ground_truth, query_ground_truth +from .persistence import MongodbDriver, ParallelTinyDbDriver, TracingDatabaseDriver +from .tracing import add_ground_truth, delete_ground_truth, query_ground_truth diff --git a/great_ai/src/great_ai/great_ai/constants.py b/great_ai/src/great_ai/great_ai/constants.py index 170d15e..a60bb55 100644 --- a/great_ai/src/great_ai/great_ai/constants.py +++ b/great_ai/src/great_ai/great_ai/constants.py @@ -1,16 +1,18 @@ -from pathlib import Path - -from great_ai.large_file import LargeFileLocal, LargeFileMongo, LargeFileS3 +from ..large_file import LargeFileMongo, LargeFileS3 +from .persistence.mongodb_driver import MongodbDriver ENV_VAR_KEY = "ENVIRONMENT" PRODUCTION_KEY = "production" -DEFAULT_TRACING_DB_FILENAME = "tracing_database.json" DASHBOARD_PATH = "/dashboard" +MONGO_CONFIG_PATHS = ["mongodb.ini", "mongo.ini", "mongo_db.ini", "mongo-db.ini"] +DEFAULT_TRACING_DATABASE_CONFIG_PATHS = { + MongodbDriver: MONGO_CONFIG_PATHS, +} + DEFAULT_LARGE_FILE_CONFIG_PATHS = { - LargeFileLocal: None, - LargeFileMongo: Path("mongodb.ini"), - LargeFileS3: Path("s3.ini"), + LargeFileS3: ["s3.ini", "b2.ini"], + LargeFileMongo: MONGO_CONFIG_PATHS, } GITHUB_LINK = "https://github.com/ScoutinScience/great-ai" @@ -25,4 +27,4 @@ ONLINE_TAG_NAME = "online" SERVER_NAME = "GreatAI-Server" -SE4ML_WEBSITE = 'https://se-ml.github.io/practices/' +SE4ML_WEBSITE = "https://se-ml.github.io/practices/" diff --git a/great_ai/src/great_ai/great_ai/context.py b/great_ai/src/great_ai/great_ai/context.py index 39d0c3a..d27b0d3 100644 --- a/great_ai/src/great_ai/great_ai/context.py +++ b/great_ai/src/great_ai/great_ai/context.py @@ -7,11 +7,11 @@ from typing import Any, Dict, Optional, Type, cast from pydantic import BaseModel from great_ai.large_file import LargeFile, LargeFileLocal -from great_ai.utilities.logger import get_logger -import yaml +from great_ai.utilities import get_logger + from .constants import ( DEFAULT_LARGE_FILE_CONFIG_PATHS, - DEFAULT_TRACING_DB_FILENAME, + DEFAULT_TRACING_DATABASE_CONFIG_PATHS, ENV_VAR_KEY, PRODUCTION_KEY, SE4ML_WEBSITE, @@ -26,6 +26,7 @@ class Context(BaseModel): logger: Logger should_log_exception_stack: bool prediction_cache_size: int + dashboard_table_size: int class Config: arbitrary_types_allowed = True @@ -37,6 +38,7 @@ class Context(BaseModel): "is_production": self.is_production, "should_log_exception_stack": self.should_log_exception_stack, "prediction_cache_size": self.prediction_cache_size, + "dashboard_table_size": self.dashboard_table_size, } @@ -54,13 +56,12 @@ def configure( *, log_level: int = DEBUG, seed: int = 42, - tracing_database: TracingDatabaseDriver = ParallelTinyDbDriver( - Path(DEFAULT_TRACING_DB_FILENAME) - ), - large_file_implementation: Type[LargeFile] = LargeFileLocal, + tracing_database: Optional[Type[TracingDatabaseDriver]] = None, + large_file_implementation: Optional[Type[LargeFile]] = None, should_log_exception_stack: Optional[bool] = None, prediction_cache_size: int = 512, - disable_se4ml_banner: bool=False + disable_se4ml_banner: bool = False, + dashboard_table_size: int = 50, ) -> None: global _context logger = get_logger("great_ai", level=log_level) @@ -72,9 +73,11 @@ def configure( ) is_production = _is_in_production_mode(logger=logger) - _initialize_large_file(large_file_implementation, logger=logger) + _set_seed(seed) + tracing_database = _initialize_tracing_database(tracing_database, logger=logger)() + if not tracing_database.is_production_ready: if is_production: logger.error( @@ -87,23 +90,27 @@ def configure( _context = Context( tracing_database=tracing_database, - large_file_implementation=large_file_implementation, + large_file_implementation=_initialize_large_file( + large_file_implementation, logger=logger + ), is_production=is_production, logger=logger, should_log_exception_stack=not is_production if should_log_exception_stack is None else should_log_exception_stack, prediction_cache_size=prediction_cache_size, + dashboard_table_size=dashboard_table_size, ) - logger.info("Setting: configured ✅") + logger.info("Settings: configured ✅") for k, v in get_context().to_flat_dict().items(): - logger.info(f'🔩 {k}: {v}') + logger.info(f"🔩 {k}: {v}") if not is_production and not disable_se4ml_banner: - logger.warning(f'You still need to check whether you follow all best practices so that you and others can trust your deployment.') - logger.warning(f'> Find out more at {SE4ML_WEBSITE}') - + logger.warning( + "You still need to check whether you follow all best practices before trusting your deployment." + ) + logger.warning(f"> Find out more at {SE4ML_WEBSITE}") def _is_in_production_mode(logger: Optional[Logger]) -> bool: @@ -129,24 +136,48 @@ def _is_in_production_mode(logger: Optional[Logger]) -> bool: return is_production -def _initialize_large_file(large_file: Type[LargeFile], logger: Logger) -> None: - path = DEFAULT_LARGE_FILE_CONFIG_PATHS[large_file] - if path is None: - return +def _initialize_tracing_database( + selected: Optional[Type[TracingDatabaseDriver]], logger: Logger +) -> Type[TracingDatabaseDriver]: + for tracing_driver, paths in DEFAULT_TRACING_DATABASE_CONFIG_PATHS.items(): + if selected is None or selected == tracing_driver: + if tracing_driver.initialized: + logger.warning( + f"{tracing_driver.__name__} has been already configured: skipping initialisation" + ) + return tracing_driver + for p in paths: + if Path(p).exists(): + logger.info( + f"Found credentials file ({Path(p).absolute()}), initialising {tracing_driver.__name__}" + ) + tracing_driver.configure_credentials_from_file(p) + return tracing_driver + logger.warning( + "Cannot find credentials files, defaulting to using ParallelTinyDbDriver" + ) + return ParallelTinyDbDriver - if large_file.initialized: - logger.warning( - f"{large_file.__name__} has been already configured: skipping initialisation" - ) - return - if path.exists(): - large_file.configure_credentials_from_file(path) - logger.info(f"{large_file.__name__} initialised with config ({path.resolve()})") - else: - logger.warning( - f"Default {large_file.__name__} config ({path.resolve()}) not found, skipping {large_file.__name__} initialisation" - ) +def _initialize_large_file( + selected: Optional[Type[LargeFile]], logger: Logger +) -> Type[LargeFile]: + for large_file, paths in DEFAULT_LARGE_FILE_CONFIG_PATHS.items(): + if selected is None or selected == large_file: + if large_file.initialized: + logger.warning( + f"{large_file.__name__} has been already configured: skipping initialisation" + ) + return large_file + for p in paths: + if Path(p).exists(): + logger.info( + f"Found credentials file ({Path(p).absolute()}), initialising {large_file.__name__}" + ) + large_file.configure_credentials_from_file(p) + return large_file + logger.warning("Cannot find credentials files, defaulting to using LargeFileLocal") + return LargeFileLocal def _set_seed(seed: int) -> None: diff --git a/great_ai/src/great_ai/great_ai/deploy/great_ai.py b/great_ai/src/great_ai/great_ai/deploy/great_ai.py index 92a8181..295b434 100644 --- a/great_ai/src/great_ai/great_ai/deploy/great_ai.py +++ b/great_ai/src/great_ai/great_ai/deploy/great_ai.py @@ -13,13 +13,12 @@ from typing import ( cast, ) -import yaml from fastapi import APIRouter, FastAPI, status from pydantic import BaseModel, create_model from great_ai.great_ai.deploy.routes.bootstrap_dashboard import bootstrap_dashboard from great_ai.great_ai.views.cache_statistics import CacheStatistics -from great_ai.utilities.parallel_map import parallel_map +from great_ai.utilities import parallel_map from ..constants import DASHBOARD_PATH from ..context import get_context @@ -40,6 +39,7 @@ from .routes import ( T = TypeVar("T") + class GreatAI(Generic[T]): def __init__(self, func: Callable[..., Any], version: str): func = automatically_decorate_parameters(func) @@ -70,7 +70,6 @@ class GreatAI(Generic[T]): redoc_url=None, ) - @staticmethod def deploy( func: Optional[Callable[..., T]] = None, diff --git a/great_ai/src/great_ai/great_ai/deploy/routes/bootstrap_trace_endpoints.py b/great_ai/src/great_ai/great_ai/deploy/routes/bootstrap_trace_endpoints.py index 754cea5..7c951a7 100644 --- a/great_ai/src/great_ai/great_ai/deploy/routes/bootstrap_trace_endpoints.py +++ b/great_ai/src/great_ai/great_ai/deploy/routes/bootstrap_trace_endpoints.py @@ -20,6 +20,10 @@ def bootstrap_trace_endpoints(app: FastAPI) -> None: ) -> List[Trace]: return get_context().tracing_database.query( conjunctive_filters=query.filter, + conjunctive_tags=query.conjunctive_tags, + since=query.since, + until=query.until, + has_feedback=query.has_feedback, sort_by=query.sort, skip=skip, take=take, diff --git a/great_ai/src/great_ai/great_ai/deploy/routes/dashboard/create_dash_app.py b/great_ai/src/great_ai/great_ai/deploy/routes/dashboard/create_dash_app.py index 61f8432..45fe646 100644 --- a/great_ai/src/great_ai/great_ai/deploy/routes/dashboard/create_dash_app.py +++ b/great_ai/src/great_ai/great_ai/deploy/routes/dashboard/create_dash_app.py @@ -1,5 +1,5 @@ from math import ceil -from typing import Any, Dict, List, Sequence, Tuple +from typing import Any, Dict, List, Optional, Sequence, Tuple, Union import pandas as pd import plotly.express as px @@ -8,12 +8,12 @@ from dash import Dash, dcc, html from dash.dependencies import Input, Output from flask import Flask -from great_ai.utilities.unique import unique +from great_ai.utilities import unique from ....constants import DASHBOARD_PATH, ONLINE_TAG_NAME from ....context import get_context from ....helper import snake_case_to_text, text_to_hex_color -from ....views import SortBy +from ....views import SortBy, Trace from .get_description import get_description from .get_filter_from_datatable import get_filter_from_datatable from .get_footer import get_footer @@ -105,19 +105,32 @@ def create_dash_app(function_name: str, function_docs: str) -> Flask: @app.callback( Output(table, "data"), Output(table, "page_count"), + Output(table, "columns"), + Output(traces_table_container, "style"), + Output(execution_time_histogram_container, "children"), + Output(parallel_coordinates, "figure"), + Output(parallel_coordinates, "style"), Input(table, "page_current"), Input(table, "page_size"), Input(table, "sort_by"), Input(table, "filter_query"), Input(interval, "n_intervals"), ) - def update_table( + def update_page( page_current: int, page_size: int, - sort_by: List[SortBy], + sort_by: List[Dict[str, Union[str, int]]], filter_query: str, n_intervals: int, - ) -> Tuple[List[Dict[str, Any]], int]: + ) -> Tuple[ + List[Dict[str, Any]], + int, + List[Dict[str, Sequence[str]]], + Dict[str, Any], + Any, + go.Figure, + Dict[str, Any], + ]: conjunctive_filters = ( [get_filter_from_datatable(f) for f in filter_query.split(" && ")] if filter_query @@ -128,114 +141,98 @@ def create_dash_app(function_name: str, function_docs: str) -> Flask: elements, count = get_context().tracing_database.query( skip=page_current * page_size, take=page_size, - conjunctive_tags=[ONLINE_TAG_NAME], conjunctive_filters=non_null_conjunctive_filters, - sort_by=sort_by, + conjunctive_tags=[ONLINE_TAG_NAME], + sort_by=[SortBy.parse_obj(s) for s in sort_by], + ) + + columns, style = update_layout(elements[0] if elements else None) + execution_time_histogram, parallel_coords_fig, style = update_charts( + elements=elements, function_name=function_name, accent_color=accent_color ) return ( - [e.to_flat_dict() for e in elements], + [e.to_flat_dict(include_original=False) for e in elements], max(1, ceil(count / page_size)), - ) - - @app.callback( - Output(table, "columns"), - Output(traces_table_container, "style"), - Input(interval, "n_intervals"), - ) - def update_layout( - n_intervals: int, - ) -> Tuple[List[Dict[str, Sequence[str]]], Dict[str, Any]]: - elements, count = get_context().tracing_database.query( - take=1, conjunctive_tags=[ONLINE_TAG_NAME] - ) - - if elements: - keys = list(elements[0].to_flat_dict().keys()) - header_height = max(len(i.split(":")) for i in keys) - columns = [ - { - "name": [""] * (header_height - len(k.split(":"))) - + k.replace("_flat", "").split(":"), - "id": k, - } - for k in keys - ] - else: - columns = [] - - return ( columns, - {"display": "block" if count > 0 else "none"}, + style, + execution_time_histogram, + parallel_coords_fig, + style, ) - @app.callback( - Output(execution_time_histogram_container, "children"), - Output(parallel_coordinates, "figure"), - Output(parallel_coordinates, "style"), - Input(table, "filter_query"), - Input(interval, "n_intervals"), - ) - def update_charts( - filter_query: str, n_intervals: int - ) -> Tuple[Any, go.Figure, Dict[str, Any]]: - conjunctive_filters = ( - [get_filter_from_datatable(f) for f in filter_query.split(" && ")] - if filter_query - else [] - ) - non_null_conjunctive_filters = [f for f in conjunctive_filters if f is not None] - - elements, count = get_context().tracing_database.query( - conjunctive_tags=[ONLINE_TAG_NAME], - conjunctive_filters=non_null_conjunctive_filters, - ) - - if not elements: - return ( - html.Span( - f"No traces yet: call your function ({function_name}) to create one.", - className="placeholder", - ), - go.Figure(), - {"display": "none"}, - ) - - flat_elements = [e.to_flat_dict() for e in elements] - - execution_time_histogram = dcc.Graph(config={"displaylogo": False}) - df = pd.DataFrame(flat_elements) - fig = px.histogram( - df, - x="original_execution_time_ms", - labels={"original_execution_time_ms": "Execution time (ms)"}, - nbins=20, - height=400, - log_y=True, - color_discrete_sequence=[accent_color], - ) - fig.update_layout( - margin=dict(l=0, r=0, b=0, t=0, pad=0), - ) - execution_time_histogram.figure = fig - - parallel_coords_fig = go.Figure( - go.Parcoords( - dimensions=[ - get_dimension_descriptor(df, c) - for c in df.columns - if c - not in {"trace_id", "created", "output", "exception", "feedback"} - and "_flat" not in c - ], - line_color=accent_color, - ) - ) - return execution_time_histogram, parallel_coords_fig, {} - return app.server +def update_layout( + first_element: Optional[Trace], +) -> Tuple[List[Dict[str, Sequence[str]]], Dict[str, Any]]: + + if first_element: + keys = list(first_element.to_flat_dict(include_original=False).keys()) + header_height = max(len(i.split(":")) for i in keys) + columns = [ + { + "name": [""] * (header_height - len(k.split(":"))) + + k.replace("_flat", "").split(":"), + "id": k, + } + for k in keys + ] + else: + columns = [] + + return ( + columns, + {"display": "none" if first_element is None else "block"}, + ) + + +def update_charts( + elements: List[Trace], function_name: str, accent_color: str +) -> Tuple[Any, go.Figure, Dict[str, Any]]: + if not elements: + return ( + html.Span( + f"No traces yet: call your function ({function_name}) to create one.", + className="placeholder", + ), + go.Figure(), + {"display": "none"}, + ) + + flat_elements = [e.to_flat_dict(include_original=False) for e in elements] + + execution_time_histogram = dcc.Graph(config={"displaylogo": False}) + df = pd.DataFrame(flat_elements) + fig = px.histogram( + df, + x="original_execution_time_ms", + labels={"original_execution_time_ms": "Execution time (ms)"}, + nbins=20, + height=400, + log_y=True, + color_discrete_sequence=[accent_color], + ) + fig.update_layout( + margin=dict(l=0, r=0, b=0, t=0, pad=0), + ) + execution_time_histogram.figure = fig + + parallel_coords_fig = go.Figure( + go.Parcoords( + dimensions=[ + get_dimension_descriptor(df, c) + for c in df.columns + if c not in {"trace_id", "created", "output", "exception", "feedback"} + and "_flat" not in c + ], + line_color=accent_color, + ) + ) + return execution_time_histogram, parallel_coords_fig, {} + + def get_dimension_descriptor(df: pd.DataFrame, column: str) -> Dict[str, Any]: dimension: Dict[str, Any] = { "label": snake_case_to_text(column), diff --git a/great_ai/src/great_ai/great_ai/deploy/routes/dashboard/get_traces_table.py b/great_ai/src/great_ai/great_ai/deploy/routes/dashboard/get_traces_table.py index 629de70..2acdfce 100644 --- a/great_ai/src/great_ai/great_ai/deploy/routes/dashboard/get_traces_table.py +++ b/great_ai/src/great_ai/great_ai/deploy/routes/dashboard/get_traces_table.py @@ -1,10 +1,12 @@ from dash import dash_table +from great_ai.great_ai.context import get_context + def get_traces_table() -> dash_table.DataTable: return dash_table.DataTable( page_current=0, - page_size=20, + page_size=get_context().dashboard_table_size, page_action="custom", filter_action="custom", sort_action="custom", @@ -24,7 +26,6 @@ def get_traces_table() -> dash_table.DataTable: "background-color": "white", "font-weight": "bold", }, - style_table={"max-height": "70vh", "overflow": "auto"}, merge_duplicate_headers=True, style_cell_conditional=[ {"if": {"column_id": "output"}, "width": 1500}, diff --git a/great_ai/src/great_ai/great_ai/persistence/__init__.py b/great_ai/src/great_ai/great_ai/persistence/__init__.py index ff5f8b6..a2a97fe 100644 --- a/great_ai/src/great_ai/great_ai/persistence/__init__.py +++ b/great_ai/src/great_ai/great_ai/persistence/__init__.py @@ -1,3 +1,3 @@ -from .mongodb_driver import MongoDbDriver +from .mongodb_driver import MongodbDriver from .parallel_tinydb_driver import ParallelTinyDbDriver from .tracing_database_driver import TracingDatabaseDriver diff --git a/great_ai/src/great_ai/great_ai/persistence/mongodb_driver.py b/great_ai/src/great_ai/great_ai/persistence/mongodb_driver.py index 6c9cc58..97926ca 100644 --- a/great_ai/src/great_ai/great_ai/persistence/mongodb_driver.py +++ b/great_ai/src/great_ai/great_ai/persistence/mongodb_driver.py @@ -1,5 +1,137 @@ +from datetime import datetime +from typing import Any, List, Mapping, Optional, Sequence, Tuple + +from pymongo import MongoClient + +from ..views import Filter, SortBy, Trace from .tracing_database_driver import TracingDatabaseDriver +operator_mapping = { + "=": "$eq", + "!=": "$ne", + "<": "$lt", + "<=": "$lte", + ">": "$gt", + ">=": "$gte", + "contains": "$regex", +} -class MongoDbDriver(TracingDatabaseDriver): + +class MongodbDriver(TracingDatabaseDriver): is_production_ready = True + + def __init__(self) -> None: + super().__init__() + if self.mongo_connection_string is None or self.mongo_database is None: + raise ValueError( + "Please configure the MongoDB access options by calling MongodbDriver.configure_credentials" + ) + + @classmethod + def configure_credentials( # type: ignore + cls, + *, + mongo_connection_string: str, + mongo_database: str, + **_: Mapping[str, Any], + ) -> None: + cls.mongo_connection_string = mongo_connection_string + cls.mongo_database = mongo_database + super().configure_credentials() + + def save(self, trace: Trace) -> str: + serialized = trace.to_flat_dict() + serialized["_id"] = trace.trace_id + + with MongoClient(self.mongo_connection_string) as client: + return client[self.mongo_database].traces.insert_one(serialized) + + def save_batch(self, documents: List[Trace]) -> List[str]: + serialized = [d.to_flat_dict() for d in documents] + for s in serialized: + s["_id"] = s["trace_id"] + + with MongoClient(self.mongo_connection_string) as client: + return client[self.mongo_database].traces.insert_many( + serialized, ordered=False + ) + + def get(self, id: str) -> Optional[Trace]: + with MongoClient(self.mongo_connection_string) as client: + value = client[self.mongo_database].traces.find_one(id) + + if value: + value = Trace.parse_obj(value) + + return value + + def _get_operator(self, filter: Filter) -> str: + if filter.operator == "contains" and not isinstance(filter.value, str): + return operator_mapping["="] + return operator_mapping[filter.operator] + + def query( + self, + *, + skip: int = 0, + take: Optional[int] = None, + conjunctive_filters: Sequence[Filter] = [], + conjunctive_tags: Sequence[str] = [], + since: Optional[datetime] = None, + until: Optional[datetime] = None, + has_feedback: Optional[bool] = None, + sort_by: Sequence[SortBy] = [], + ) -> Tuple[List[Trace], int]: + + query = { + "filter": { + "$and": [{"tags": tag} for tag in conjunctive_tags] + + [ + {f.property: {self._get_operator(f): f.value}} + for f in conjunctive_filters + ] + + [{}] + }, + "sort": [ + (col.column_id, 1 if col.direction == "asc" else -1) for col in sort_by + ], + } + + if skip: + query["skip"] = skip + + if take: + query["limit"] = take + + if since: + query["filter"]["$and"].append({"created": {"$gte": since}}) + + if until: + query["filter"]["$and"].append({"created": {"$lte": until}}) + + if has_feedback is not None: + query["filter"]["$and"].append( + {"feedback": {"$ne": None}} if has_feedback else {"feedback": None} + ) + + with MongoClient(self.mongo_connection_string) as client: + values = client[self.mongo_database].traces.find(**query) + documents = [Trace.parse_obj(t) for t in values] + + return documents, len(documents) + + def update(self, id: str, new_version: Trace) -> None: + serialized = new_version.dict() + serialized["_id"] = new_version.trace_id + with MongoClient(self.mongo_connection_string) as client: + client[self.mongo_database].traces.update_one(id, new_version) + + def delete(self, id: str) -> None: + with MongoClient(self.mongo_connection_string) as client: + client[self.mongo_database].traces.delete_one(id) + + def delete_batch(self, ids: List[str]) -> List[str]: + delete_filter = {"_id": {"$in": ids}} + + with MongoClient(self.mongo_connection_string) as client: + return client[self.mongo_database].traces.delete_many(delete_filter) diff --git a/great_ai/src/great_ai/great_ai/persistence/parallel_tinydb_driver.py b/great_ai/src/great_ai/great_ai/persistence/parallel_tinydb_driver.py index 11929f5..1243ab0 100644 --- a/great_ai/src/great_ai/great_ai/persistence/parallel_tinydb_driver.py +++ b/great_ai/src/great_ai/great_ai/persistence/parallel_tinydb_driver.py @@ -9,6 +9,7 @@ from tinydb import TinyDB from ..views import Filter, SortBy, Trace from .tracing_database_driver import TracingDatabaseDriver +DEFAULT_TRACING_DB_FILENAME = "tracing_database.json" lock = Lock() @@ -17,10 +18,7 @@ operator_mapping = {"=": "eq", "!=": "ne", "<": "lt", "<=": "le", ">": "gt", ">= class ParallelTinyDbDriver(TracingDatabaseDriver): is_production_ready = False - - def __init__(self, path_to_db: Path) -> None: - super().__init__() - self._path_to_db = path_to_db + path_to_db = Path(DEFAULT_TRACING_DB_FILENAME) def save(self, trace: Trace) -> str: return self._safe_execute(lambda db: db.insert(trace.dict())) @@ -43,8 +41,9 @@ class ParallelTinyDbDriver(TracingDatabaseDriver): conjunctive_filters: Sequence[Filter] = [], conjunctive_tags: Sequence[str] = [], since: Optional[datetime] = None, - sort_by: Sequence[SortBy] = [], - has_feedback: Optional[bool] = None + until: Optional[datetime] = None, + has_feedback: Optional[bool] = None, + sort_by: Sequence[SortBy] = [] ) -> Tuple[List[Trace], int]: def does_match(d: Dict[str, Any]) -> bool: return ( @@ -53,6 +52,10 @@ class ParallelTinyDbDriver(TracingDatabaseDriver): since is None or cast(datetime, datetime.fromisoformat(d["created"])) >= since ) + and ( + until is None + or cast(datetime, datetime.fromisoformat(d["created"])) <= until + ) and ( has_feedback is None or has_feedback == (d["feedback"] is not None) ) @@ -99,7 +102,11 @@ class ParallelTinyDbDriver(TracingDatabaseDriver): def delete(self, id: str) -> None: self._safe_execute(lambda db: db.remove(lambda d: d["trace_id"] == id)) + def delete_batch(self, ids: List[str]) -> List[str]: + for i in ids: + self.delete(i) + def _safe_execute(self, func: Callable[[TinyDB], Any]) -> Any: with lock: - with TinyDB(self._path_to_db) as db: + with TinyDB(self.path_to_db) as db: return func(db) diff --git a/great_ai/src/great_ai/great_ai/persistence/tracing_database_driver.py b/great_ai/src/great_ai/great_ai/persistence/tracing_database_driver.py index 5cc1ac1..a4bab9d 100644 --- a/great_ai/src/great_ai/great_ai/persistence/tracing_database_driver.py +++ b/great_ai/src/great_ai/great_ai/persistence/tracing_database_driver.py @@ -1,12 +1,29 @@ -from abc import ABC, abstractmethod +from abc import ABC, abstractclassmethod, abstractmethod from datetime import datetime -from typing import List, Optional, Sequence, Tuple +from pathlib import Path +from typing import List, Optional, Sequence, Tuple, Union + +from great_ai.utilities import ConfigFile from ..views import Filter, SortBy, Trace class TracingDatabaseDriver(ABC): is_production_ready: bool + initialized: bool = False + + @classmethod + def configure_credentials_from_file( + cls, + secrets_path: Union[Path, str], + ) -> None: + cls.configure_credentials(**ConfigFile(secrets_path)) + + @abstractclassmethod + def configure_credentials( + cls, + ) -> None: + cls.initialized = True @abstractmethod def save(self, document: Trace) -> str: @@ -31,9 +48,10 @@ class TracingDatabaseDriver(ABC): take: Optional[int] = None, conjunctive_filters: Sequence[Filter] = [], conjunctive_tags: Sequence[str] = [], + until: Optional[datetime] = None, since: Optional[datetime] = None, - sort_by: Sequence[SortBy] = [], - has_feedback: Optional[bool] = None + has_feedback: Optional[bool] = None, + sort_by: Sequence[SortBy] = [] ) -> Tuple[List[Trace], int]: pass @@ -44,3 +62,10 @@ class TracingDatabaseDriver(ABC): @abstractmethod def delete(self, id: str) -> None: pass + + @abstractmethod + def delete_batch( + self, + ids: List[str], + ) -> None: + pass diff --git a/great_ai/src/great_ai/great_ai/tracing/__init__.py b/great_ai/src/great_ai/great_ai/tracing/__init__.py index fda4bb0..e34184c 100644 --- a/great_ai/src/great_ai/great_ai/tracing/__init__.py +++ b/great_ai/src/great_ai/great_ai/tracing/__init__.py @@ -1,3 +1,4 @@ from .add_ground_truth import add_ground_truth +from .delete_ground_truth import delete_ground_truth from .query_ground_truth import query_ground_truth from .tracing_context import TracingContext diff --git a/great_ai/src/great_ai/great_ai/tracing/add_ground_truth.py b/great_ai/src/great_ai/great_ai/tracing/add_ground_truth.py index affcc78..3702756 100644 --- a/great_ai/src/great_ai/great_ai/tracing/add_ground_truth.py +++ b/great_ai/src/great_ai/great_ai/tracing/add_ground_truth.py @@ -51,7 +51,7 @@ def add_ground_truth( created = datetime.utcnow().isoformat() traces = [ - Trace[T]( + Trace( created=created, original_execution_time_ms=0, logged_values=X if isinstance(X, dict) else {"input": X}, diff --git a/great_ai/src/great_ai/great_ai/tracing/delete_ground_truth.py b/great_ai/src/great_ai/great_ai/tracing/delete_ground_truth.py new file mode 100644 index 0000000..ef1620e --- /dev/null +++ b/great_ai/src/great_ai/great_ai/tracing/delete_ground_truth.py @@ -0,0 +1,22 @@ +from datetime import datetime +from typing import List, Optional, Union + +from ..context import get_context + + +def delete_ground_truth( + conjunctive_tags: Union[List[str], str] = [], + *, + until: Optional[datetime] = None, + since: Optional[datetime] = None, +) -> None: + tags = ( + conjunctive_tags if isinstance(conjunctive_tags, list) else [conjunctive_tags] + ) + db = get_context().tracing_database + + items, length = db.query( + conjunctive_tags=tags, until=until, since=since, has_feedback=True + ) + + db.delete_batch([i.trace_id for i in items]) diff --git a/great_ai/src/great_ai/great_ai/tracing/query_ground_truth.py b/great_ai/src/great_ai/great_ai/tracing/query_ground_truth.py index b515f51..b58d3ea 100644 --- a/great_ai/src/great_ai/great_ai/tracing/query_ground_truth.py +++ b/great_ai/src/great_ai/great_ai/tracing/query_ground_truth.py @@ -1,10 +1,8 @@ from datetime import datetime -from typing import List, Optional, TypeVar, Union +from typing import List, Optional, Union from ..context import get_context -from ..views.trace import Trace - -T = TypeVar("T") +from ..views import Trace def query_ground_truth( @@ -12,7 +10,7 @@ def query_ground_truth( *, since: Optional[datetime] = None, return_max_count: Optional[int] = None -) -> List[Trace[T]]: +) -> List[Trace]: tags = ( conjunctive_tags if isinstance(conjunctive_tags, list) else [conjunctive_tags] ) diff --git a/great_ai/src/great_ai/great_ai/views/query.py b/great_ai/src/great_ai/great_ai/views/query.py index e46b567..7257a7b 100644 --- a/great_ai/src/great_ai/great_ai/views/query.py +++ b/great_ai/src/great_ai/great_ai/views/query.py @@ -1,4 +1,5 @@ -from typing import List +from datetime import datetime +from typing import List, Optional, Sequence from pydantic import BaseModel @@ -9,6 +10,10 @@ from .sort_by import SortBy class Query(BaseModel): filter: List[Filter] = [] sort: List[SortBy] = [] + conjunctive_tags: Sequence[str] = [] + since: Optional[datetime] = None + until: Optional[datetime] = None + has_feedback: Optional[bool] = None class Config: schema_extra = { @@ -24,5 +29,7 @@ class Query(BaseModel): {"column_id": "original_execution_time_ms", "direction": "asc"}, {"column_id": "id", "direction": "desc"}, ], + "conjunctive_tags": ["online"], + "has_feedback": False, } } diff --git a/great_ai/src/great_ai/great_ai/views/sort_by.py b/great_ai/src/great_ai/great_ai/views/sort_by.py index 868000e..d0f8eff 100644 --- a/great_ai/src/great_ai/great_ai/views/sort_by.py +++ b/great_ai/src/great_ai/great_ai/views/sort_by.py @@ -1,8 +1,8 @@ from typing import Literal -from typing_extensions import TypedDict +from pydantic import BaseModel -class SortBy(TypedDict): +class SortBy(BaseModel): column_id: str direction: Literal["asc", "desc"] diff --git a/great_ai/src/great_ai/great_ai/views/trace.py b/great_ai/src/great_ai/great_ai/views/trace.py index 6182f2a..adc744b 100644 --- a/great_ai/src/great_ai/great_ai/views/trace.py +++ b/great_ai/src/great_ai/great_ai/views/trace.py @@ -1,8 +1,8 @@ +from pprint import pformat from typing import Any, Dict, Generic, List, Optional, TypeVar from uuid import uuid4 -import yaml -from pydantic import validator +from pydantic import Extra, validator from ..helper import HashableBaseModel from .model import Model @@ -10,7 +10,7 @@ from .model import Model T = TypeVar("T") -class Trace(HashableBaseModel, Generic[T]): +class Trace(Generic[T], HashableBaseModel): trace_id: Optional[str] created: str original_execution_time_ms: float @@ -21,6 +21,9 @@ class Trace(HashableBaseModel, Generic[T]): feedback: Any = None tags: List[str] + class Config: + extra = Extra.ignore + @validator("trace_id", always=True) def generate_id(cls, v: Optional[str], values: Dict[str, Any]) -> Optional[str]: if not v: @@ -41,26 +44,42 @@ class Trace(HashableBaseModel, Generic[T]): @property def output_flat(self) -> str: - return yaml.dump(self.output, stream=None) + return pformat(self.output, indent=2, compact=True) + + @property + def exception_flat(self) -> str: + return ( + "null" + if self.exception is None + else pformat(self.exception, indent=2, compact=True) + ) @property def feedback_flat(self) -> str: return ( - "null" if self.feedback is None else yaml.dump(self.feedback, stream=None) + "null" + if self.feedback is None + else pformat(self.feedback, indent=2, compact=True) ) @property def tags_flat(self) -> str: return ",\n".join(self.tags) - def to_flat_dict(self) -> Dict[str, Any]: + def to_flat_dict(self, include_original: bool = True) -> Dict[str, Any]: return { - "trace_id": self.trace_id, - "created": self.created, - "original_execution_time_ms": self.original_execution_time_ms, + **( + self.dict() + if include_original + else { + "trace_id": self.trace_id, + "created": self.created, + "original_execution_time_ms": self.original_execution_time_ms, + } + ), **self.logged_values, "models_flat": self.models_flat, - "exception": "null" if self.exception is None else self.exception, + "exception_flat": self.exception_flat, "output_flat": self.output_flat, "feedback_flat": self.feedback_flat, "tags_flat": self.tags_flat, diff --git a/great_ai/src/great_ai/large_file/large_file/large_file_mongo.py b/great_ai/src/great_ai/large_file/large_file/large_file_mongo.py index 4cf2118..07bb25c 100644 --- a/great_ai/src/great_ai/large_file/large_file/large_file_mongo.py +++ b/great_ai/src/great_ai/large_file/large_file/large_file_mongo.py @@ -19,29 +19,29 @@ MONGO_NAME_VERSION_SEPARATOR = "_" class LargeFileMongo(LargeFile): - connection_string = None - database = None + mongo_connection_string = None + mongo_database = None @classmethod def configure_credentials( # type: ignore cls, *, - connection_string: str, - database: str, + mongo_connection_string: str, + mongo_database: str, **_: Mapping[str, Any], ) -> None: - cls.connection_string = connection_string - cls.database = database + cls.mongo_connection_string = mongo_connection_string + cls.mongo_database = mongo_database super().configure_credentials() @cached_property def _client(self) -> GridFSBucket: - if self.connection_string is None or self.database is None: + if self.mongo_connection_string is None or self.mongo_database is None: raise ValueError( "Please configure the MongoDB access options by calling LargeFileMongo.configure_credentials or set offline_mode=True in the constructor." ) - db: Database = MongoClient(self.connection_string)[self.database] + db: Database = MongoClient(self.mongo_connection_string)[self.mongo_database] return GridFSBucket(db) def _find_remote_instances(self) -> List[DataInstance]: