Move packages
Signed-off-by: András Schmelczer <andras@schmelczer.dev>
This commit is contained in:
parent
cef2e51c67
commit
acbdd96883
78 changed files with 123 additions and 27 deletions
|
|
@ -1,9 +1,8 @@
|
|||
import json
|
||||
|
||||
from open_s3 import LargeFile
|
||||
from predict import predict_domain
|
||||
from good_ai import LargeFile, process_batch
|
||||
from predict_domain import predict_domain
|
||||
|
||||
from good_ai import process_batch
|
||||
|
||||
if __name__ == "__main__":
|
||||
with open("data/s2-corpus-1583.json") as f:
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from models import DomainPrediction
|
|||
from sklearn.pipeline import Pipeline
|
||||
|
||||
from good_ai import use_model
|
||||
from sus.clean import clean
|
||||
from good_ai.utilities.clean import clean
|
||||
|
||||
@use_model(model_key, version="latest")
|
||||
def predict_domain(
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import re
|
||||
|
||||
from sus.clean import clean
|
||||
from sus.lemmatize_text import lemmatize_text
|
||||
from good_ai.utilities.clean import clean
|
||||
from good_ai.utilities.lemmatize_text import lemmatize_text
|
||||
|
||||
|
||||
def preprocess(text: str) -> str:
|
||||
|
|
|
|||
|
|
@ -18,7 +18,23 @@
|
|||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"ename": "ModuleNotFoundError",
|
||||
"evalue": "No module named 'open_s3'",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)",
|
||||
"\u001b[1;32m/Users/andras/Projects/good-ai/example/train.ipynb Cell 2'\u001b[0m in \u001b[0;36m<cell line: 18>\u001b[0;34m()\u001b[0m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/andras/Projects/good-ai/example/train.ipynb#ch0000001?line=15'>16</a>\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mre\u001b[39;00m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/andras/Projects/good-ai/example/train.ipynb#ch0000001?line=16'>17</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mconfig\u001b[39;00m \u001b[39mimport\u001b[39;00m model_key\n\u001b[0;32m---> <a href='vscode-notebook-cell:/Users/andras/Projects/good-ai/example/train.ipynb#ch0000001?line=17'>18</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mgood_ai\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mutilities\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mclean\u001b[39;00m \u001b[39mimport\u001b[39;00m clean\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/andras/Projects/good-ai/example/train.ipynb#ch0000001?line=18'>19</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mgood_ai\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mutilities\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mparallel_map\u001b[39;00m \u001b[39mimport\u001b[39;00m parallel_map\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/andras/Projects/good-ai/example/train.ipynb#ch0000001?line=19'>20</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mgood_ai\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mutilities\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mlanguage\u001b[39;00m \u001b[39mimport\u001b[39;00m is_english, predict_language\n",
|
||||
"File \u001b[0;32m~/Projects/good-ai/good_ai/src/good_ai/__init__.py:1\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/__init__.py?line=0'>1</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39mgood_ai\u001b[39;00m \u001b[39mimport\u001b[39;00m \u001b[39m*\u001b[39m\n\u001b[1;32m <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/__init__.py?line=1'>2</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39mutilities\u001b[39;00m \u001b[39mimport\u001b[39;00m \u001b[39m*\u001b[39m\n\u001b[1;32m <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/__init__.py?line=2'>3</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39mopen_s3\u001b[39;00m \u001b[39mimport\u001b[39;00m \u001b[39m*\u001b[39m\n",
|
||||
"File \u001b[0;32m~/Projects/good-ai/good_ai/src/good_ai/good_ai/__init__.py:2\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/__init__.py?line=0'>1</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39mdeploy\u001b[39;00m \u001b[39mimport\u001b[39;00m process_batch\n\u001b[0;32m----> <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/__init__.py?line=1'>2</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39mmodels\u001b[39;00m \u001b[39mimport\u001b[39;00m save_model, use_model\n",
|
||||
"File \u001b[0;32m~/Projects/good-ai/good_ai/src/good_ai/good_ai/models/__init__.py:1\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/models/__init__.py?line=0'>1</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39msave_model\u001b[39;00m \u001b[39mimport\u001b[39;00m save_model\n\u001b[1;32m <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/models/__init__.py?line=1'>2</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39muse_model\u001b[39;00m \u001b[39mimport\u001b[39;00m use_model\n",
|
||||
"File \u001b[0;32m~/Projects/good-ai/good_ai/src/good_ai/good_ai/models/save_model.py:6\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/models/save_model.py?line=2'>3</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mtyping\u001b[39;00m \u001b[39mimport\u001b[39;00m Optional, Union\n\u001b[1;32m <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/models/save_model.py?line=4'>5</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mjoblib\u001b[39;00m \u001b[39mimport\u001b[39;00m dump\n\u001b[0;32m----> <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/models/save_model.py?line=5'>6</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mopen_s3\u001b[39;00m \u001b[39mimport\u001b[39;00m LargeFile\n\u001b[1;32m <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/models/save_model.py?line=7'>8</a>\u001b[0m logger \u001b[39m=\u001b[39m logging\u001b[39m.\u001b[39mgetLogger(\u001b[39m\"\u001b[39m\u001b[39mmodels\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/models/save_model.py?line=10'>11</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39msave_model\u001b[39m(\n\u001b[1;32m <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/models/save_model.py?line=11'>12</a>\u001b[0m model: Union[Path, \u001b[39mstr\u001b[39m, \u001b[39mobject\u001b[39m], key: \u001b[39mstr\u001b[39m, keep_last_n: Optional[\u001b[39mint\u001b[39m] \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/models/save_model.py?line=12'>13</a>\u001b[0m ) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mint\u001b[39m:\n",
|
||||
"\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'open_s3'"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import json\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
|
|
@ -30,19 +46,16 @@
|
|||
"import pandas as pd\n",
|
||||
"from joblib import dump\n",
|
||||
"from sklearn.pipeline import Pipeline\n",
|
||||
"from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
|
||||
"from sklearn.feature_extraction.text import TfidfVectorizer\n",
|
||||
"from sklearn import metrics\n",
|
||||
"from random import shuffle, seed\n",
|
||||
"from sklearn.model_selection import GridSearchCV\n",
|
||||
"from pprint import pprint\n",
|
||||
"import re\n",
|
||||
"from config import model_key\n",
|
||||
"from sus.clean import clean\n",
|
||||
"from sus.parallel_map import parallel_map\n",
|
||||
"from sus.language import is_english, predict_language\n",
|
||||
"from sus.lemmatize_text import lemmatize_text\n",
|
||||
"from open_s3 import LargeFile\n",
|
||||
"from good_ai import save_model\n",
|
||||
"from good_ai.utilities.clean import clean\n",
|
||||
"from good_ai.utilities.parallel_map import parallel_map\n",
|
||||
"from good_ai.utilities.language import is_english, predict_language\n",
|
||||
"from good_ai import LargeFile, save_model\n",
|
||||
"from preprocess import preprocess\n",
|
||||
"from predict_domain import predict_domain"
|
||||
]
|
||||
|
|
@ -90,7 +103,7 @@
|
|||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 5/5 [00:00<00:00, 43062.67it/s]\n"
|
||||
"100%|██████████| 5/5 [00:00<00:00, 50412.31it/s]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -306,7 +319,89 @@
|
|||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"INFO:open_s3:Fetching online versions of domain-prediction-v2\n",
|
||||
"INFO:open_s3:Found versions: [1, 2]\n",
|
||||
"INFO:open_s3:Copying file for domain-prediction-v2-3\n",
|
||||
"INFO:open_s3:Compressing domain-prediction-v2-3\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3 to S3 from /var/folders/5g/yd_5_wg548q0yb4lnvg0y3q40000gn/T/large-file-hkyimkq5\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 262144/15169091 bytes (1.7%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 524288/15169091 bytes (3.5%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 786432/15169091 bytes (5.2%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 1048576/15169091 bytes (6.9%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 1310720/15169091 bytes (8.6%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 1572864/15169091 bytes (10.4%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 1835008/15169091 bytes (12.1%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 2097152/15169091 bytes (13.8%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 2359296/15169091 bytes (15.6%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 2621440/15169091 bytes (17.3%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 2883584/15169091 bytes (19.0%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 3145728/15169091 bytes (20.7%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 3407872/15169091 bytes (22.5%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 3670016/15169091 bytes (24.2%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 3932160/15169091 bytes (25.9%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 4194304/15169091 bytes (27.7%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 4456448/15169091 bytes (29.4%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 4718592/15169091 bytes (31.1%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 4980736/15169091 bytes (32.8%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 5242880/15169091 bytes (34.6%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 5505024/15169091 bytes (36.3%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 5767168/15169091 bytes (38.0%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 6029312/15169091 bytes (39.7%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 6291456/15169091 bytes (41.5%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 6553600/15169091 bytes (43.2%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 6815744/15169091 bytes (44.9%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 7077888/15169091 bytes (46.7%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 7340032/15169091 bytes (48.4%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 7602176/15169091 bytes (50.1%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 7864320/15169091 bytes (51.8%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 8126464/15169091 bytes (53.6%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 8388608/15169091 bytes (55.3%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 8650752/15169091 bytes (57.0%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 8912896/15169091 bytes (58.8%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 9175040/15169091 bytes (60.5%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 9437184/15169091 bytes (62.2%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 9699328/15169091 bytes (63.9%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 9961472/15169091 bytes (65.7%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 10223616/15169091 bytes (67.4%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 10485760/15169091 bytes (69.1%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 10747904/15169091 bytes (70.9%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 11010048/15169091 bytes (72.6%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 11272192/15169091 bytes (74.3%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 11534336/15169091 bytes (76.0%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 11796480/15169091 bytes (77.8%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 12023363/15169091 bytes (79.3%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 12285507/15169091 bytes (81.0%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 12547651/15169091 bytes (82.7%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 12809795/15169091 bytes (84.4%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 13071939/15169091 bytes (86.2%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 13334083/15169091 bytes (87.9%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 13596227/15169091 bytes (89.6%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 13858371/15169091 bytes (91.4%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 14120515/15169091 bytes (93.1%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 14382659/15169091 bytes (94.8%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 14644803/15169091 bytes (96.5%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 14906947/15169091 bytes (98.3%)\n",
|
||||
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 15169091/15169091 bytes (100.0%)\n",
|
||||
"INFO:open_s3:Removing old version (keep_last_n=1): domain-prediction-v2/1\n",
|
||||
"INFO:models:Model domain-prediction-v2 uploaded with version 3\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"3"
|
||||
]
|
||||
},
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import logging\n",
|
||||
"import sys\n",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue