Move packages

Signed-off-by: András Schmelczer <andras@schmelczer.dev>

example/main.py

@@ -1,9 +1,8 @@
 import json
 
-from open_s3 import LargeFile
-from predict import predict_domain
+from good_ai import LargeFile, process_batch
+from predict_domain import predict_domain
 
-from good_ai import process_batch
 
 if __name__ == "__main__":
     with open("data/s2-corpus-1583.json") as f:

example/predict_domain.py

@@ -7,7 +7,7 @@ from models import DomainPrediction
 from sklearn.pipeline import Pipeline
 
 from good_ai import use_model
-from sus.clean import clean
+from good_ai.utilities.clean import clean
 
 @use_model(model_key, version="latest")
 def predict_domain(

example/preprocess.py

@@ -1,7 +1,7 @@
 import re
 
-from sus.clean import clean
-from sus.lemmatize_text import lemmatize_text
+from good_ai.utilities.clean import clean
+from good_ai.utilities.lemmatize_text import lemmatize_text
 
 
 def preprocess(text: str) -> str:

example/train.ipynb

@@ -18,7 +18,23 @@
    "cell_type": "code",
    "execution_count": 1,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "ename": "ModuleNotFoundError",
+     "evalue": "No module named 'open_s3'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
+      "\u001b[1;32m/Users/andras/Projects/good-ai/example/train.ipynb Cell 2'\u001b[0m in \u001b[0;36m<cell line: 18>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/andras/Projects/good-ai/example/train.ipynb#ch0000001?line=15'>16</a>\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mre\u001b[39;00m\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/andras/Projects/good-ai/example/train.ipynb#ch0000001?line=16'>17</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mconfig\u001b[39;00m \u001b[39mimport\u001b[39;00m model_key\n\u001b[0;32m---> <a href='vscode-notebook-cell:/Users/andras/Projects/good-ai/example/train.ipynb#ch0000001?line=17'>18</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mgood_ai\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mutilities\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mclean\u001b[39;00m \u001b[39mimport\u001b[39;00m clean\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/andras/Projects/good-ai/example/train.ipynb#ch0000001?line=18'>19</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mgood_ai\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mutilities\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mparallel_map\u001b[39;00m \u001b[39mimport\u001b[39;00m parallel_map\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/andras/Projects/good-ai/example/train.ipynb#ch0000001?line=19'>20</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mgood_ai\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mutilities\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mlanguage\u001b[39;00m \u001b[39mimport\u001b[39;00m is_english, predict_language\n",
+      "File \u001b[0;32m~/Projects/good-ai/good_ai/src/good_ai/__init__.py:1\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/__init__.py?line=0'>1</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39mgood_ai\u001b[39;00m \u001b[39mimport\u001b[39;00m \u001b[39m*\u001b[39m\n\u001b[1;32m      <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/__init__.py?line=1'>2</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39mutilities\u001b[39;00m \u001b[39mimport\u001b[39;00m \u001b[39m*\u001b[39m\n\u001b[1;32m      <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/__init__.py?line=2'>3</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39mopen_s3\u001b[39;00m \u001b[39mimport\u001b[39;00m \u001b[39m*\u001b[39m\n",
+      "File \u001b[0;32m~/Projects/good-ai/good_ai/src/good_ai/good_ai/__init__.py:2\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/__init__.py?line=0'>1</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39mdeploy\u001b[39;00m \u001b[39mimport\u001b[39;00m process_batch\n\u001b[0;32m----> <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/__init__.py?line=1'>2</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39mmodels\u001b[39;00m \u001b[39mimport\u001b[39;00m save_model, use_model\n",
+      "File \u001b[0;32m~/Projects/good-ai/good_ai/src/good_ai/good_ai/models/__init__.py:1\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/models/__init__.py?line=0'>1</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39msave_model\u001b[39;00m \u001b[39mimport\u001b[39;00m save_model\n\u001b[1;32m      <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/models/__init__.py?line=1'>2</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39muse_model\u001b[39;00m \u001b[39mimport\u001b[39;00m use_model\n",
+      "File \u001b[0;32m~/Projects/good-ai/good_ai/src/good_ai/good_ai/models/save_model.py:6\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/models/save_model.py?line=2'>3</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mtyping\u001b[39;00m \u001b[39mimport\u001b[39;00m Optional, Union\n\u001b[1;32m      <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/models/save_model.py?line=4'>5</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mjoblib\u001b[39;00m \u001b[39mimport\u001b[39;00m dump\n\u001b[0;32m----> <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/models/save_model.py?line=5'>6</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mopen_s3\u001b[39;00m \u001b[39mimport\u001b[39;00m LargeFile\n\u001b[1;32m      <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/models/save_model.py?line=7'>8</a>\u001b[0m logger \u001b[39m=\u001b[39m logging\u001b[39m.\u001b[39mgetLogger(\u001b[39m\"\u001b[39m\u001b[39mmodels\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m     <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/models/save_model.py?line=10'>11</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39msave_model\u001b[39m(\n\u001b[1;32m     <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/models/save_model.py?line=11'>12</a>\u001b[0m     model: Union[Path, \u001b[39mstr\u001b[39m, \u001b[39mobject\u001b[39m], key: \u001b[39mstr\u001b[39m, keep_last_n: Optional[\u001b[39mint\u001b[39m] \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m     <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/models/save_model.py?line=12'>13</a>\u001b[0m ) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mint\u001b[39m:\n",
+      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'open_s3'"
+     ]
+    }
+   ],
    "source": [
     "import json\n",
     "import matplotlib.pyplot as plt\n",
@@ -30,19 +46,16 @@
     "import pandas as pd\n",
     "from joblib import dump\n",
     "from sklearn.pipeline import Pipeline\n",
-    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
+    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
     "from sklearn import metrics\n",
     "from random import shuffle, seed\n",
     "from sklearn.model_selection import GridSearchCV\n",
     "from pprint import pprint\n",
-    "import re\n",
     "from config import model_key\n",
-    "from sus.clean import clean\n",
-    "from sus.parallel_map import parallel_map\n",
-    "from sus.language import is_english, predict_language\n",
-    "from sus.lemmatize_text import lemmatize_text\n",
-    "from open_s3 import LargeFile\n",
-    "from good_ai import save_model\n",
+    "from good_ai.utilities.clean import clean\n",
+    "from good_ai.utilities.parallel_map import parallel_map\n",
+    "from good_ai.utilities.language import is_english, predict_language\n",
+    "from good_ai import LargeFile, save_model\n",
     "from preprocess import preprocess\n",
     "from predict_domain import predict_domain"
    ]
@@ -90,7 +103,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|██████████| 5/5 [00:00<00:00, 43062.67it/s]\n"
+      "100%|██████████| 5/5 [00:00<00:00, 50412.31it/s]\n"
      ]
     }
    ],
@@ -306,7 +319,89 @@
    "cell_type": "code",
    "execution_count": 8,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "INFO:open_s3:Fetching online versions of domain-prediction-v2\n",
+      "INFO:open_s3:Found versions: [1, 2]\n",
+      "INFO:open_s3:Copying file for domain-prediction-v2-3\n",
+      "INFO:open_s3:Compressing domain-prediction-v2-3\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3 to S3 from /var/folders/5g/yd_5_wg548q0yb4lnvg0y3q40000gn/T/large-file-hkyimkq5\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz   262144/15169091 bytes (1.7%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz   524288/15169091 bytes (3.5%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz   786432/15169091 bytes (5.2%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  1048576/15169091 bytes (6.9%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  1310720/15169091 bytes (8.6%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  1572864/15169091 bytes (10.4%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  1835008/15169091 bytes (12.1%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  2097152/15169091 bytes (13.8%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  2359296/15169091 bytes (15.6%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  2621440/15169091 bytes (17.3%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  2883584/15169091 bytes (19.0%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  3145728/15169091 bytes (20.7%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  3407872/15169091 bytes (22.5%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  3670016/15169091 bytes (24.2%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  3932160/15169091 bytes (25.9%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  4194304/15169091 bytes (27.7%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  4456448/15169091 bytes (29.4%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  4718592/15169091 bytes (31.1%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  4980736/15169091 bytes (32.8%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  5242880/15169091 bytes (34.6%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  5505024/15169091 bytes (36.3%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  5767168/15169091 bytes (38.0%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  6029312/15169091 bytes (39.7%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  6291456/15169091 bytes (41.5%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  6553600/15169091 bytes (43.2%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  6815744/15169091 bytes (44.9%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  7077888/15169091 bytes (46.7%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  7340032/15169091 bytes (48.4%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  7602176/15169091 bytes (50.1%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  7864320/15169091 bytes (51.8%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  8126464/15169091 bytes (53.6%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  8388608/15169091 bytes (55.3%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  8650752/15169091 bytes (57.0%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  8912896/15169091 bytes (58.8%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  9175040/15169091 bytes (60.5%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  9437184/15169091 bytes (62.2%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  9699328/15169091 bytes (63.9%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz  9961472/15169091 bytes (65.7%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 10223616/15169091 bytes (67.4%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 10485760/15169091 bytes (69.1%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 10747904/15169091 bytes (70.9%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 11010048/15169091 bytes (72.6%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 11272192/15169091 bytes (74.3%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 11534336/15169091 bytes (76.0%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 11796480/15169091 bytes (77.8%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 12023363/15169091 bytes (79.3%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 12285507/15169091 bytes (81.0%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 12547651/15169091 bytes (82.7%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 12809795/15169091 bytes (84.4%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 13071939/15169091 bytes (86.2%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 13334083/15169091 bytes (87.9%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 13596227/15169091 bytes (89.6%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 13858371/15169091 bytes (91.4%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 14120515/15169091 bytes (93.1%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 14382659/15169091 bytes (94.8%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 14644803/15169091 bytes (96.5%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 14906947/15169091 bytes (98.3%)\n",
+      "INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 15169091/15169091 bytes (100.0%)\n",
+      "INFO:open_s3:Removing old version (keep_last_n=1): domain-prediction-v2/1\n",
+      "INFO:models:Model domain-prediction-v2 uploaded with version 3\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "3"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "import logging\n",
     "import sys\n",

good_ai/src/__init__.py

@@ -1 +0,0 @@
-

good_ai/src/good_ai/__init__.py

@@ -1,2 +1,3 @@
-from .deploy import process_batch
-from .models import save_model, use_model
+from .good_ai import *
+from .utilities import *
+from .open_s3 import *

good_ai/src/good_ai/good_ai/__init__.py

@@ -0,0 +1,2 @@
+from .deploy import process_batch
+from .models import save_model, use_model

good_ai/src/good_ai/good_ai/deploy/process_batch.py

@@ -1,7 +1,7 @@
 from functools import partial, reduce
 from typing import Any, Callable, Iterable, Optional, Sequence
 
-from sus.parallel_map import parallel_map
+from good_ai.utilities.parallel_map import parallel_map
 
 from ..core import function_registry
 

good_ai/src/good_ai/good_ai/models/load_model.py

@@ -2,7 +2,7 @@ import logging
 from typing import Any, Optional
 
 from joblib import load
-from open_s3 import LargeFile
+from good_ai.open_s3 import LargeFile
 
 logger = logging.getLogger("models")
 

good_ai/src/good_ai/good_ai/models/save_model.py

@@ -3,7 +3,7 @@ from pathlib import Path
 from typing import Optional, Union
 
 from joblib import dump
-from open_s3 import LargeFile
+from good_ai.open_s3 import LargeFile
 
 logger = logging.getLogger("models")
 

good_ai/src/good_ai/utilities/external/negspacy/termsets.py

@@ -40,7 +40,7 @@ preceding = [
     "doubt",
     "negative for",
     "no",
-    "versus",
+    "vergood_ai.utilities",
     "without",
     "doesn't",
     "doesnt",
@@ -104,7 +104,7 @@ pseudo_clinical = pseudo + [
     "not ruled out",
     "not been ruled out",
     "not drain",
-    "no suspicious change",
+    "no good_ai.utilitiespicious change",
     "no interval change",
     "no significant interval change",
 ]

good_ai/src/good_ai/utilities/external/pylatexenc/latexencode/_unicode_to_latex_encoder.py

@@ -342,7 +342,7 @@ class UnicodeToLatexEncoder(object):
 
        Possible protection schemes are:
 
-         - 'braces' (the default):  Any suspicious replacement text (that
+         - 'braces' (the default):  Any good_ai.utilitiespicious replacement text (that
            might look fragile) is placed in curly braces ``{...}``.
 
          - 'braces-all':  All replacement latex escapes are surrounded in

good_ai/src/good_ai/utilities/publication_tei/publication_tei.py

@@ -3,7 +3,7 @@ from typing import Any, List, Optional, Union
 
 from bs4 import BeautifulSoup
 from bs4.element import NavigableString, Tag
-from sus.publication_tei.models.element import Paragraph
+from good_ai.utilities.publication_tei.models.element import Paragraph
 
 from ..clean import clean
 from .models import Affiliation, Author, Element, Meta, PublicationMetadata, Text, Title