diff --git a/.vscode/settings.json b/.vscode/settings.json index 8f43709..6bacb24 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -36,11 +36,13 @@ "organisation's", "Parcoords", "plotly", + "pretrained", "proba", "pydantic", "pymongo", "pyplot", "redoc", + "scibert", "serialise", "sklearn", "starlette", diff --git a/docs/examples/scibert.md b/docs/examples/scibert.md deleted file mode 100644 index 90af250..0000000 --- a/docs/examples/scibert.md +++ /dev/null @@ -1,7 +0,0 @@ -# Summarising scientific publications from a tech-transfer perspective - -This example shows how `great-ai` is used in practice at ScoutinScience. - -
-[:material-test-tube: Check out the code](https://github.com/schmelczer/great-ai/tree/main/docs/examples/scibert){ .md-button .md-button--primary } -
diff --git a/docs/examples/scibert/Dockerfile b/docs/examples/scibert/Dockerfile deleted file mode 100644 index c766fd9..0000000 --- a/docs/examples/scibert/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM schmelczera/great-ai:v0.1.4 - -COPY requirements.txt ./ -RUN pip install --no-cache-dir --requirement requirements.txt - -COPY . ./ - -RUN large-file --backend s3 --secrets s3.ini --cache scibert-highlights - -CMD ["deploy.ipynb"] diff --git a/docs/examples/scibert/additional-files.md b/docs/examples/scibert/additional-files.md new file mode 100644 index 0000000..883c7d8 --- /dev/null +++ b/docs/examples/scibert/additional-files.md @@ -0,0 +1,75 @@ +# Additional files in the repository + +In order to give you a smooth experience while comprehending this example, all non-notebook files are presented on this page in one place. In reality, these files should be in your project's top-level directory. + +## config.ini + +```ini title="config.ini" +ENVIRONMENT = DEVELOPMENT +ENVIRONMENT = ENV:ENVIRONMENT + +MONGO_CONNECTION_STRING=ENV:MONGO_CONNECTION_STRING +MONGO_DATABASE=highlights + +AWS_REGION_NAME = eu-west-2 +AWS_ACCESS_KEY_ID = MY_DEFAULT_AWS_ACCESS_KEY_ID_FOR_DEVELOPMENT +AWS_SECRET_ACCESS_KEY = MY_DEFAULT_AWS_SECRET_ACCESS_KEY_FOR_DEVELOPMENT +LARGE_FILES_BUCKET_NAME = my-orgs-large-files + +AWS_REGION_NAME = ENV:AWS_REGION_NAME +AWS_ACCESS_KEY_ID = ENV:AWS_ACCESS_KEY_ID +AWS_SECRET_ACCESS_KEY = ENV:AWS_SECRET_ACCESS_KEY +LARGE_FILES_BUCKET_NAME = ENV:LARGE_FILES_BUCKET_NAME +``` + +> All necessary configuration which is read by [great_ai.utilities.ConfigFile][]. This will resolve values starting with `ENV:` from your environment variables. + +## requirements.txt + +```requirements.txt title="requirements.txt" +torch==1.12.0 +transformers==4.20.1 +numpy==1.23.0 +``` + +> Usually, it is recommended to pin (freeze) the library versions on which we depend. This file is referenced by the [Dockerfile](#dockerfile). + +## Dockerfile + +```Dockerfile title="Dockerfile" +FROM schmelczera/great-ai:v0.1.6 + +COPY requirements.txt ./ +RUN pip install --no-cache-dir --requirement requirements.txt + +COPY . ./ + +RUN large-file --backend s3 --secrets s3.ini --cache scibert-highlights + +CMD ["deploy.ipynb"] +``` + +> This is used by the CD pipeline to create the production deployment of the service. + +## .dockerignore + +```dockerignore title=".dockerignore" +.cache +.git +data +.gitignore +.env +.vscode +.dockerignore +Dockerfile +.mypy_cache +.gitignore +**/__pycache__ +**/.DS_Store +README.md +``` + +> It is useful not to send, for example, the `.cache` folder used by LargeFile to the docker daemon; this will speed up your local build times substantially. + +!!! Note ".gitignore" + A very similar looking `.gitignore` file should also be present. diff --git a/docs/examples/scibert/analyse.ipynb b/docs/examples/scibert/analyse.ipynb deleted file mode 100644 index ff6fccc..0000000 --- a/docs/examples/scibert/analyse.ipynb +++ /dev/null @@ -1,225 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "data/evaluation-experiment-2-stage #1-sa6a0y.json\n", - "data/evaluation-experiment-2-stage #1-2m6dmb.json\n" - ] - } - ], - "source": [ - "from pathlib import Path\n", - "import json\n", - "\n", - "annotations = []\n", - "for p in Path(\"data\").glob(\"*.json\"):\n", - " with open(p, encoding=\"utf-8\") as f:\n", - " print(p)\n", - " annotations.append(json.load(f))\n", - "\n", - "evaluations = {\n", - " sentence: [\n", - " annotation[sentence] for annotation in annotations if sentence in annotation\n", - " ]\n", - " for sentence in {\n", - " sentence for annotation in annotations for sentence in annotation.keys()\n", - " }\n", - "}\n", - "\n", - "X = [s for s in evaluations.keys()]\n", - "y = [int(sum(e) > 0) for e in evaluations.values()]" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "y1 = [e[0] for e in evaluations.values() if len(e) == 2]\n", - "y2 = [e[1] for e in evaluations.values() if len(e) == 2]" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.3546448712421808" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import sklearn.metrics\n", - "\n", - "sklearn.metrics.cohen_kappa_score(y1, y2)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "X = [s for s in evaluations.keys()]\n", - "y = [int(sum(e) > 0) for e in evaluations.values()]" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Pipeline(steps=[('vectorizer',\n",
-       "                 TfidfVectorizer(max_df=0.3, min_df=3, sublinear_tf=True)),\n",
-       "                ('classifier', LinearSVC())])
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" - ], - "text/plain": [ - "Pipeline(steps=[('vectorizer',\n", - " TfidfVectorizer(max_df=0.3, min_df=3, sublinear_tf=True)),\n", - " ('classifier', LinearSVC())])" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from sklearn.model_selection import train_test_split\n", - "from sklearn.svm import LinearSVC\n", - "from sklearn.pipeline import Pipeline\n", - "from sklearn.feature_extraction.text import TfidfVectorizer\n", - "\n", - "\n", - "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)\n", - "\n", - "model = Pipeline(\n", - " steps=[\n", - " (\"vectorizer\", TfidfVectorizer(sublinear_tf=True, min_df=3, max_df=0.3)),\n", - " (\"classifier\", LinearSVC()),\n", - " ]\n", - ") # baseline model\n", - "\n", - "model.fit(X_train, y_train)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([0.81, 0.68, 0.73, 0.75, 0.81])" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from sklearn.model_selection import cross_val_score\n", - "\n", - "cross_val_score(model, X, y, cv=5)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " precision recall f1-score support\n", - "\n", - " False 0.73 0.96 0.83 45\n", - " True 0.95 0.71 0.81 55\n", - "\n", - " accuracy 0.82 100\n", - " macro avg 0.84 0.83 0.82 100\n", - "weighted avg 0.85 0.82 0.82 100\n", - "\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUUAAAEWCAYAAADxboUEAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAhOklEQVR4nO3deZgV1Z3/8feHppGtAZFFQJBF1BhHEdFgNAbRcYsTdWJios74mzhjFpck/jTbk0TjJDH5TTJqEqND1KiJSVyicUkiIkpQxw0QUXAHXNhXafbu29/fH1UNlwb63qZvd92mP6/nqYdbdatOfa88fD2nTp1zFBGYmVmiQ9YBmJmVEydFM7M8TopmZnmcFM3M8jgpmpnlcVI0M8vTMesAWkKf3hUxdHBl1mFYE7wxq2vWIVgTVbNqeUT0bU4ZJx3XLVaszBV17vRZmyZGxMnNuV8xdsukOHRwJc9PHJx1GNYEJw0clXUI1kSPxb3vNLeM5StzPDdxn6LOrRzwdp/m3q8Yu2VSNLO2IshFXdZBbMNJ0cwyE0Ad5TWqzknRzDITBDVR3DPF1uKkaGaZck3RzCwVQM5J0cxsK9cUzcxSAeTKbPpCJ0Uzy0wQ1LimaGaWCsiVV050UjSz7CTvKZYXTwhhZhkSuSK3okqTKiS9KOnhdH+YpOckvSXpLkmdCpXhpGhmmQmgLorbivQV4NW8/Z8A10bEfsAq4IJCBTgpmlmmSlVTlLQP8Ang5nRfwHjg3vSU24EzCpXjZ4pmlpkAaqJkdbPrgK8DVen+XsDqiKhN998HBhUqxDVFM8tMMqKl6JpiH0nT8rYL68uRdBqwNCKmNzcm1xTNLDOByBVfN1seEWN28t3RwCclnQp0BnoA1wO9JHVMa4v7AAsK3cQ1RTPLVF2oqK0xEfGtiNgnIoYCnwUej4hzgSeAs9LTzgceKBSPk6KZZaaJzedd8Q3gMklvkTxjvKXQBW4+m1lmAlETpU1DETEFmJJ+ngsc2ZTrnRTNLFPNqAW2CCdFM8tMhMiV7pWcknBSNLNM1bmmaGaWSDpaXFM0M0u5+WxmtkUyzK8i6zC24aRoZplp4oiWVuGkaGaZqnPz2cws4Y4WM7M8gcgVGNfc2pwUzSxTda4pmpklIuTeZzOzegF+T9HMLJ87WszMUkHhCWRbm5OimWXKNUUzs1Qyyaw7WszMgKSjpdxGtJRXNGbW7pRijRZJnSU9L+klSbMlfT89fpukeZJmptuoQvG4pmhmmYlQqWqKm4DxEbFWUiXwlKS/pd9dERH3FluQk6KZZaoU7ylGRABr093KdItdKcvNZzPLTJAsR1DMVoikCkkzgaXApIh4Lv3qh5JmSbpW0h6FynFSNLPMBKKmrqKoDegjaVreduE2ZUXkImIUsA9wpKSDgW8BBwJHAL1J1oFulJvPZpapJrynuDwixhQ6KSJWS3oCODkifpoe3iTpN8Dlha53TdHMMlM/oqWYrTGS+krqlX7uAvwj8JqkAekxAWcArxSKyTVFM8tUiaYOGwDcLqmCpLJ3d0Q8LOlxSX0BATOBLxYqyEnRzDITQUkmmY2IWcBhOzg+vqllOSmaWaY8IYSZWcpjn207uRxccvL+7DWghv+8Yx7/fdlg3pjVFQIGDd/E5de9S5duddtd98df9OORP+xFRYfgSz9YwJhx1QC88EQVN313ELk6ccrnVnD2JUtb+ye1C30HbuaK69+lV99aCPjr7/biz7f0bXBW8KX/XMiR49ewcUMHfva1wbz1clcATvj0Ss75yhIAfn99fx67p3cr/4LykIx9bic1RUk54OW8Q2dExPydnLs2Irq3VCzl7M8392XwyE2sX5s8bP7C9xfQrSpJgv9z1UAevLXPdontnTf2YMoDezLhiddYuaSSb549glueehWAG769D9f88W36DKjhklP3Z+xJH7Dv/pta90e1A7laMeHqgbz1cle6dMvxy0feYMbUKt59s/OWc44YX82gYZv4t6MP5MDR67nkmgV85bSRVPWq5bzLlnDJKSOJgF8+8ibPPtqDtR+0xzpKyYb5lUxLRrMhIkblbfNb8F5t0rKFlTw/uQennLNiy7H6hBgBmzZ2YEcv8j8zsSfjTl9Fpz2CvYdsZuDQTbz+Yldef7ErA4duYsC+m6nsFIw7fRXPTOzZWj+nXVm5tHJLrW/Dugree6szfQbUbHPOUSd9wGP37gmI12Z0o1vPHL371XD4uGpmTO1O9eqOrP2gIzOmdmfMcdUZ/IryUKoRLaXSailaUndJkyXNkPSypNN3cM4ASVPT2SxekfSx9PiJkp5Jr71H0m5Rq7zpykH8+3cWogZ/Cz/96mA+e+iHee+tPTj988u2u275okr6Dtz6D7DPgBpWLK5kxeLtjy9fVNli8Vui/z6bGXHwBl6b0XWb4332rmHZwq3//ZcvrGSvvWvS4522Hl/UiT57b5tQ24v63udittbSkkmxS950PfcDG4EzI2I0cBzws/SFynznABPToTqHAjMl9QG+A5yQXjsNuKzhzSRdWD/8Z9mKXAv+rNJ4dlIPevWpZeQhG7b77vLr3uP3L85myMhN/P3BPTOIzorVuWuO7948n5u+N5D1a8urw6AtCERtXUVRW2tpyYcYG9LkBkA6nc+PJB0L1AGDgP7A4rxrXgBuTc/9c0TMlPRx4CDg6TSHdgKeaXiziJgATAAYc2jnXZodozXNeaEbzz7agxcmH8TmTWJ9dQU/uXgI3/jluwBUVMC401dx96/6cdJnV25zbZ8BDWogi5IaCLDd8YZNOiudio7Bd2+ez+P37cnTf+u13ffLG9bcByY1+uWLKznkqLVbjw/YzKxndovGzy5pzaZxMVrzCee5QF/g8DRZLgE6558QEVOBY4EFwG2S/pXkqdqkvGeTB0XEBa0Yd4v4/LcXcef0Odzx/By+deM7HHpMNV//xbssmJc0qyKSZ4eDR2zfSTL2xDVMeWBPNm8Si9/txIJ5e3DAYes5YNR6Fszbg8XvdqJms5jywJ6MPXFNa/+0diK47Gfv8d6bnblvQsNe58Szj/bkhLNWAcGBo9exfk0HVi6tZPqUKg7/+Fq696yle89aDv/4WqZPqWrd8MtEfe9zc4f5lVJrdnf1BJZGRI2k44B9G54gaV/g/Yj4dTrFz2jgh8ANkvaLiLckdQMGRcQbrRh7q4iAn35lCOvXVhABww/awCU/fh+AZyb24I2XunL+1xcz9ICNHPtPq7lw3IFUVAQX/+h9KtLWxUU/fJ9vnzOcupw48bMrGXrAxgx/0e7rw0eu44RPr2LunM78atLrAPzmmgH0G7QZgL/8tg/PT67iiOPX8Jv/fY1N6Ss5ANWrO3Lndf34xV/fBODOa/tTvbo99jwnyq33WcncjC1QcIPXbNJngw8B3UmeC44FTomI+fXnSjofuAKoIZkw8l8jYp6k8cBPgPq50L4TEQ/u7N5jDu0cz08c3CK/y1rGSQNHZR2CNdFjce/0YmataUzvA/vF8bd+qqhz7z36pmbfrxgt9r+nhu8dRsRy4KjGzo2I24Hbd/D94yTzoZnZbqR+ktly0n7r7GaWuQBq68qr+eykaGaZajfD/MzMCqmfZLacOCmaWab8TNHMrF6UX/O5vJ5wmlm7UqqXtyV1lvS8pJckzZb0/fT4MEnPSXpL0l2SOjVaEE6KZpahZOxzh6K2AjYB4yPiUGAUcLKksSTvN18bEfsBq4CCo+GcFM0sUxEqamu8jIiIqB9QXpluAYwH7k2P306yol+jnBTNLFOlmk9RUoWkmcBSYBLwNrA6ImrTU94nmYimUe5oMbPMRNM6WvpImpa3PyGdHSstK3LAqHT95/uBA3clJidFM8tUoaZxnuXFjH2OiNWSniAZVtxLUse0trgPyQxcjXLz2cwyJHJ1HYraGi1F6pvWEJHUBfhH4FXgCeCs9LTzgQcKReSaopllpoSr+Q0AbpdUQVLZuzsiHpY0B/ijpB8ALwK3FCrISdHMshPJc8VmFxMxCzhsB8fnAkc2pSwnRTPLlIf5mZmlgiZ1tLQKJ0Uzy5BnyTEz20ZdnZOimRmQdLK4+WxmlsfNZzOzPC20oOguc1I0s0y5+WxmlvIaLWZm+dzRYmbWgJ8pmplt1WZqipJ+QSM5PCIubZGIzKxdaUu9z9Ma+c7MrNna1NjniLg9f19S14hY3/IhmVm7ERBlNsyv4Mzbko5KJ2p8Ld0/VNKvWjwyM2sfositlRSzHMF1wEnACoCIeAk4tgVjMrN2o7jlTVuziV1U73NEvCdtE1SuZcIxs3anzDpaiqkpvifpo0BIqpR0OcmCMGZmzZO+vN3cmqKkwZKekDRH0mxJX0mPXyVpgaSZ6XZqoZCKqSl+EbieZBHphcBE4KIirjMzK6w0NcVa4P9GxAxJVcB0SZPS766NiJ8WW1DBpBgRy4Fzdy1OM7MCSvC8MCIWAYvSz9WSXiWpyDVZMb3PwyU9JGmZpKWSHpA0fFduZma2nRL3PksaSrKy33PpoYslzZJ0q6Q9C11fzDPF3wN3k6yrOhC4B/hD8SGame1EkNQUi9mgj6RpeduFDYuT1B34E/DViFgD3AiMAEaR1CR/ViikYp4pdo2I3+bt/07SFUVcZ2ZWUBOG+S2PiDE7+1JSJUlCvDMi7kvKjiV53/8aeLjQTRob+9w7/fg3Sd8E/kiS188G/lrMLzAzK6gEHS1K3hm8BXg1Iv477/iA9HkjwJnAK4XKaqymOJ0k3PqnoF/I+y6AbzUlaDOzHVFphvkdDfwL8LKkmemxbwOfkzSKJGfNZ9s8tkONjX0e1twozcwaVaIhfBHxFFsrcPma3KotakSLpIOBg4DOeUHc0dSbmZlta0snStkomBQlXQmMI0mKfwVOAZ4CnBTNrPna4DC/s4DjgcUR8W/AoUDPFo3KzNqPMpslp5jm84aIqJNUK6kHsBQY3MJxmVl7UWY1xWKS4jRJvYBfk/RIrwWeacmgzKydiJL1PpdMMWOfv5x+vEnSI0CPiJjVsmGZWbvRVmqKkkY39l1EzGiZkMzMstNYTbGxMYIBjC9xLGbWDqmt1BQj4rjWDKSU5izqy+irv5R1GNYE1829MesQrIkeK9Xwjrb2nqKZWYtp5ddtiuGkaGaZUl3WEWzLSdHMslVmNcViZt6WpPMkfS/dHyLpyJYPzczahTIb0VLMML9fAUcBn0v3q4EbWiwiM2s3FMVvraWY5vNHImK0pBcBImKVpE4tHJeZtRdtsPe5RlIFaQVWUl+gzB6NmllbVW4dLcU0n38O3A/0k/RDkmnDftSiUZlZ+1FmzxSLGft8p6TpJNOHCTgjIl5t8cjMbPdXoueFkgaTzPHaPymVCRFxfbrW1F3AUJLlCD4TEasaK6uY3uchwHrgIeBBYF16zMys+UpTU6wF/m9EHASMBS6SdBDwTWByRIwEJqf7jSrmmeJf2LqAVWdgGPA68OEirjUza1xp1mhZRLKuMxFRLelVYBBwOsnKAQC3A1OAbzRWVjHN53/I309nz/nyTk43M2uSUr9uI2kocBjwHNA/b4nTxSTN60Y1eURLRMyQ9JGmXmdmtkPFJ8U+kqbl7U+IiAn5J0jqDvwJ+GpErEmWg05vExFS4RRczMJVl+XtdgBGAwsLXWdmVlDTOlqWR8SYnX0pqZIkId4ZEfelh5dIGhARiyQNIFlOpVHFvJJTlbftQfKM8fQirjMzK6wEHS1KqoS3AK9GxH/nffUgcH76+XzggULhNFpTTF/aroqIywsVZGa2S0rzTPFo4F+AlyXNTI99G/gxcLekC4B3gM8UKqix5Qg6RkStpKObH6+Z2fZEaTpaIuKptLgdOb4pZTVWU3ye5PnhTEkPAvcA6/KCuG9nF5qZFa3Mpg4rpve5M7CCZE2W+vcVA3BSNLPmifIb+9xYUuyX9jy/wtZkWK/McruZtVlllk0aS4oVQHd23E4vs59hZm1Vm1nND1gUEVe3WiRm1j61oaRYXjM/mtnup42t5tekbmwzs13RZjpaImJlawZiZu1TW3qmaGbW8pwUzcxSbeyZoplZixLl16PrpGhm2XJN0cxsqzbT+2xm1ipcUzQzS5VoidNSclI0s2w5KZqZbeWaoplZvjJLisUsXGVm1jLSSWaL2QqRdKukpZJeyTt2laQFkmam26mFynFSNLNslWA1v9RtwMk7OH5tRIxKt78WKsTNZzPLTKkWrgKIiKmShja3HNcUzSxbpasp7szFkmalzes9C53spGhmmVJEURvQR9K0vO3CIoq/ERgBjAIWAT8rdIGbz2aWnaat5rc8IsY0qfiIJfWfJf0aeLjQNa4pmlm2WrD5LGlA3u6ZJKuTNso1RTPLVKk6WiT9ARhH0sx+H7gSGCdpFElanQ98oVA5Topmlq3S9T5/bgeHb2lqOU6KZpYdTwhhZtaAk6KZWUKA6sorKzopmlmm3Hy2La78pyf42P7vsHJdFz5z09lbjp99xMt85ojZ1NWJp94awvWPHbXdtR8d8S6Xn/Q0FR2C+1/8ELc9fRgAA3ut4ZpPPUavLht5dVFfvnP/eGrrKlrtN+3OajeJO87ej9zmDtTl4MCTP+DjX1vM/P/tzmPXDKSuRux98AZO+/G7dNjBv6xZf9qTp365NwDHXLyYQz61CoBFL3fhoSuGULupAyPGreHE7y1A5baaU0tpr6v5SdoLmJzu7g3kgGXp/pERsbk14ig3D710AHe9cDBXn/H4lmNjhi5g3AHz+ez/fJqaXAV7dt2w3XUdVMc3TnmKL//uNJas6cbv/v0+/v76vsxb3ptLj3+WO589hEdn78e3T53KGYe9xr3TP9yaP2u3VdEpOO/Ot+nUrY5cDdzxmZGMOHYND14xhHN/+zZ7Dd/E36/dm1l/6s2os1duc+2G1RU8+fO9+fwDb4Dg1k/uz8gT1tClZ46/fXcfPnHNewwctZ4/fn44b/+9iv3GVWf0K1tfua3R0iovb0fEivpZKoCb2HbWis2S2mWNdca7A/lgwx7bHDvr8Nn85unDqMkltbtV67tsd93Bg5by/qoeLFjdg9q6CibOHsG4A+YDwRHDFjJ5znAAHp61P8cdOK+lf0a7IUGnbsm/4LpakasVqoCKymCv4ZsAGHZ0Na890mu7a+dOrWLYMdV06ZWjS88cw46pZu7fq6he2pHNaysYdNh6JDjkzJW8Malna/6s7LX82OcmySwZSboN2AgcBjwtaQ2wNiJ+mn7/CnBaRMyXdB5wKdAJeA74ckTksom8Ze271weMHrKIi8Y/z+baCq6ddBRzFvbb5py+VetY/EH3LftL13Tn4EFL6NVlI2s3diIXyf/rlqzpTt+qda0a/+6uLge3fPIAVr3TiTHnLWfgoeupqxULZ3Vh4CEbeO2RXqxZVLndddVLKukxoGbLftXeNVQvqaR6cSVVezc4vnj763dbUX4dLVkP89sH+GhEXLazEyR9CDgbODqtaeaAc1snvNZX0aGOHl02cf4tZ3LdpLH85FOTKLuHLu1Yhwr4j7+8zqX/O4eFs7qy7I3OnPnz+Tz2g0HcesZIOnXLIT/CbRJFcVtrybrZek8RNb7jgcOBF5Q8fe4CLG14UjpjxoUAld0Lzg5Utpau6c7jrw0DxOyF/akL0avrRlbnNaOXVXdj755rt+z367GWpdXdWL2hM907b6ZCdeSiA/17rGVZdbcMfsXur3OPHPuOXcvcqVWM/Y9l/OvdbwEw98kqVs7bY7vzq/rX8M5zW2v31Ysr2fcja7erGTasObYLZfb//Kxrivltu1q2jadz+qeA2/OeQR4QEVc1LCgiJkTEmIgY07FL200ET7w+lDFDFwIwpPdqKityrF7feZtzZi/ox+DeHzCw1xo6dshx0off5u9vDAXEtPkDOf6guQCcdsgbTHl9aOv+gN3YuhUVbFyTVANrNop5T1Wx1/BNrFue1C1qN4lnburH6HNWbHft8GOrmftkFRs+qGDDBxXMfbKK4cdWU9Wvlk7dcyx4sSsRMOv+3ux/wget+ruyVD/JrGuKOzYfOA1A0mhgWHp8MvCApGsjYqmk3kBVRLyTTZil86N/fozD911Ir64b+dtXf8tNU8bwwIsHctUnp3D3F++iJlfBlQ+MB0Sf7uv43j9N4dI/fIJcdOAnfzuGG879Cx0UPDjzAOYu6w3Azx8byzWfmsRFxz3Pa4v78OcXP5Ttj9yNrF1ayUNXDCFyIgI+dOpqRh6/hsnXDOTNx3sQdXD4uSsY+tGkFr9wVhdm/L4Pp/34Pbr0ynHMxUv4zRn7A/CxS5bQpVfSSDr56vd5+OtDqNnYgREfX8OIdtTzTESylRFFKwck6SpgLXAw8HBE3Jse7wI8AAwi6Uw5Cjgl7Wg5G/gWSU2yBrgoIp7d2T269hsc+5/1tRb9HVZa1339xqxDsCY6bthb05s6v2FDVb32icM+/pWizn3ywa83+37FaPWa4o6avunxDcCJO/nuLuCuFgzLzDJSbu8pllPz2czamwDK7JUcJ0Uzy1Z55UQnRTPLVrlNCJH1Kzlm1t7V90AX2gpIlzBdmo6Gqz/WW9IkSW+mf3qJUzMrbyV8T/E24OQGx74JTI6IkSSv932zUCFOimaWGaVjn4vZComIqcDKBodPB25PP98OnFGoHD9TNLNstewrOf0jYlH6eTHQv9AFTopmlikVP4Ckj6RpefsTImJCsRdHREiFG+JOimaWnabNlbh8F0a0LJE0ICIWSRrADiaTacjPFM0sQ0X2PO/6cOQHgfPTz+eTDCVulGuKZpapUk0yK+kPwDiSZvb7wJXAj4G7JV0AvAN8plA5Topmlp0o3djniPjcTr46vinlOCmaWbbKbOowJ0Uzy1Z55UQnRTPLVhNeyWkVTopmli0nRTOzhCJQzknRzGwr1xTNzPI4KZqZpYKWnhCiyZwUzSxT7n02M8vnpGhmloqAuvJqPzspmlm2yisnOimaWbb8TNHMLJ+ToplZKoASzadYKk6KZpYhd7SYmW3LzWczs5Sbz2Zm+QKiNM1nSfOBaiAH1O7Cyn+Ak6KZZa20zefjImJ5cwpwUjSz7JRh89nrPptZturqitsKC+BRSdMlXbir4bimaGYZatJC930kTcvbnxARE/L2j4mIBZL6AZMkvRYRU5sakZOimWUnaMp7issb6zyJiAXpn0sl3Q8cCTQ5Kbr5bGbZiihua4SkbpKq6j8DJwKv7Eo4rimaWbZK0/vcH7hfEiR57fcR8ciuFOSkaGYZipL0PkfEXODQ5sfjpGhmWQqIXC7rKLbhpGhm2fLYZzOzlJcjMDNrwDVFM7OtwjVFM7NUBOScFM3MtirR1GGl4qRoZpkJIMpslhwnRTPLTpRuktlScVI0s0y5pmhmlq/MaoqKMntHqBQkLQPeyTqOFtIHaNZ069bqdte/s30jom9zCpD0CMl/n2Isj4iTm3O/YuyWSXF3Jmnari7IY9nw31nb4vkUzczyOCmameVxUmx7JhQ+xcqM/87aED9TNDPL45qimVkeJ0UzszxOimYlpMR5kr6X7g+RdGTWcVnxnBTbAEldJX1X0q/T/ZGSTss6LtuhXwFHAZ9L96uBG7ILx5rKSbFt+A2wieQfG8AC4AfZhWON+EhEXARsBIiIVUCnbEOypnBSbBtGRMT/A2oAImI9oGxDsp2okVRBMisWkvoC5TW41xrlpNg2bJbUha3/0EaQ1Byt/PwcuB/oJ+mHwFPAj7INyZrC7ym2AZL+EfgOcBDwKHA08H8iYkqWcdmOSToQOJ6kNj85Il7NOCRrAifFNkLSXsBYkn9oz0bE7jjrSpsnaciOjkfEu60di+0aJ8U2QNLRwMyIWCfpPGA0cH1E7K7To7VZkl4mecwhoDMwDHg9Ij6caWBWND9TbBtuBNZLOhS4DHgbuCPbkGxHIuIfIuKQ9M+RwJHAM1nHZcVzUmwbaiOp0p8O3BARNwBVGcdkRYiIGcBHso7DiuflCNqGaknfAs4DjpXUAajMOCbbAUmX5e12IHnUsTCjcGwXuKbYNpxN8grOBRGxGNgH+K9sQ7KdqMrb9gD+QlLDtzbCHS1mJZK+tP2TiLg861hs17n5XMYkVZO+sN3wKyAiokcrh2Q7IaljRNSmbwpYG+aaolkJSJoREaMl3QgMAu4B1tV/HxH3ZRacNYlrim2IpH4k774BfiG4THUGVgDj2fq+YgBOim2Ek2IbIOmTwM+AgcBSYF/gVcAvBJePfmnP8ytsTYb13BxrQ9z73Db8J8kQvzciYhjJuNpnsw3JGqgAuqdbVd7n+s3aCNcU24aaiFghqYOkDhHxhKTrsg7KtrEoIq7OOghrPifFtmG1pO7AVOBOSUvJe4hvZcHzW+4m3PtcxiQNiYh3JXUDNpA87jgX6AncGRErMg3QtpDUOyJWZh2HNZ+TYhmrf80j/fyniPhU1jGZ7e7c0VLe8ptkwzOLwqwdcVIsb7GTz2bWQtx8LmOSciQdKgK6AOvrv8LD/MxahJOimVkeN5/NzPI4KZqZ5XFSbKck5STNlPSKpHskdW1GWbdJOiv9fLOkgxo5d5ykj+7CPeZL6lPs8QbnrG3iva6S5DkR2yknxfZrQ0SMioiDgc3AF/O/lLRLo50i4t8jYk4jp4wDmpwUzVqLk6IBPAnsl9binpT0IDBHUoWk/5L0gqRZkr4AoMQvJb0u6TGgX31BkqZIGpN+PlnSDEkvSZosaShJ8v1aWkv9mKS+kv6U3uOF+klaJe0l6VFJsyXdTBHD6CT9WdL09JoLG3x3bXp8sqS+6bERkh5Jr3kyXcTe2jmPfW7n0hrhKcAj6aHRwMERMS9NLB9ExBGS9gCelvQocBhwAHAQ0B+YA9zaoNy+wK+BY9OyekfESkk3AWsj4qfpeb8Hro2Ip9KF5CcCHwKuBJ6KiKslfQK4oIif8/n0Hl2AF9JRQCuAbsC0iPiapO+lZV8MTAC+GBFvSvoI8CuSeRCtHXNSbL+6SJqZfn4SuIWkWft8RMxLj58IHFL/vJBkzPVI4FjgDxGRAxZKenwH5Y8FptaX1ci44BOAg6QtFcEe6eQXxwL/nF77F0mrivhNl0o6M/08OI11BVAH3JUe/x1wX3qPjwL35N17jyLuYbs5J8X2a0NEjMo/kCaH/Nl3BFwSERMbnHdqCePoAIyNiI07iKVoksaRJNijImK9pCnkzVLeQKT3Xd3wv4GZnylaYyYCX5JUCSBp/3TGnqnA2ekzxwHAcTu49lmSNaqHpdf2To9Xk0zCWu9R4JL6HUmj0o9TgXPSY6cAexaItSewKk2IB5LUVOt1AOpru+eQNMvXAPMkfTq9hyQdWuAe1g44KVpjbiZ5XjhD0ivA/5C0Lu4H3ky/uwN4puGFEbEMuJCkqfoSW5uvDwFn1ne0AJcCY9KOnDls7QX/PklSnU3SjC60Hs0jQEdJrwI/ZtuZydcBR6a/YTxQPxnsucAFaXyz8frMhof5mZltwzVFM7M8TopmZnmcFM3M8jgpmpnlcVI0M8vjpGhmlsdJ0cwsj5OimVme/w/Z0EIdNZewEQAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "y_predicted = model.predict(X_test)\n", - "print(\n", - " sklearn.metrics.classification_report(\n", - " [y > 0 for y in y_test], [y > 0 for y in y_predicted]\n", - " )\n", - ")\n", - "sklearn.metrics.ConfusionMatrixDisplay.from_predictions(\n", - " [y > 0 for y in y_test],\n", - " [y > 0 for y in y_predicted],\n", - " xticks_rotation=\"vertical\",\n", - " values_format=\".2f\",\n", - ")\n", - "None" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3.10.4 ('.env': venv)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.4" - }, - "orig_nbformat": 4, - "vscode": { - "interpreter": { - "hash": "02dd6d3afbfa9fbbe1037d64ad9014965528a1ccad21929d6e72f466389a68ad" - } - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/docs/examples/scibert/config.ini b/docs/examples/scibert/config.ini deleted file mode 100644 index 86a3fd9..0000000 --- a/docs/examples/scibert/config.ini +++ /dev/null @@ -1,10 +0,0 @@ -ENVIRONMENT = DEVELOPMENT -ENVIRONMENT = ENV:ENVIRONMENT - -MONGO_CONNECTION_STRING=ENV:MONGO_CONNECTION_STRING -MONGO_DATABASE=highlights - -AWS_REGION_NAME = ENV:AWS_REGION_NAME -AWS_ACCESS_KEY_ID = ENV:AWS_ACCESS_KEY_ID -AWS_SECRET_ACCESS_KEY = ENV:AWS_SECRET_ACCESS_KEY -LARGE_FILES_BUCKET_NAME = ENV:LARGE_FILES_BUCKET_NAME diff --git a/docs/examples/scibert/data.ipynb b/docs/examples/scibert/data.ipynb new file mode 100644 index 0000000..97a7fb8 --- /dev/null +++ b/docs/examples/scibert/data.ipynb @@ -0,0 +1,310 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Explore data and feasibility of approach\n", + "\n", + "![screenshot fo the annotator tool](/media/annotator.png)\n", + "\n", + "We had asked our clients and in-house experts to annotate sentences using a rigorous guideline. The aim is to decide on which sentences they would like to see in a summary for a paper.\n", + "\n", + "The results are in JSON format, each annotator has a separate file. Let's load them." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "data/evaluation-experiment-2-stage #1-sa6a0y.json\n", + "data/evaluation-experiment-2-stage #1-2m6dmb.json\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import json\n", + "\n", + "annotations = []\n", + "for p in Path(\"data\").glob(\"*.json\"):\n", + " with open(p, encoding=\"utf-8\") as f:\n", + " print(p)\n", + " annotations.append(json.load(f))\n", + "\n", + "evaluations = {\n", + " sentence: [\n", + " annotation[sentence] for annotation in annotations if sentence in annotation\n", + " ]\n", + " for sentence in {\n", + " sentence for annotation in annotations for sentence in annotation.keys()\n", + " }\n", + "}\n", + "\n", + "X = [s for s in evaluations.keys()]\n", + "y = [int(sum(e) > 0) for e in evaluations.values()]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Save the compiled and processed data for later use using LargeFileS3." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[38;5;39mCopying file for summary-train-dataset-small-0\u001b[0m\n", + "\u001b[38;5;39mCompressing summary-train-dataset-small-0\u001b[0m\n", + "\u001b[38;5;39mUploading summary-train-dataset-small-0 to S3 as summary-train-dataset-small/0\u001b[0m\n", + "\u001b[38;5;39mUploading summary-train-dataset-small-0.tar.gz 0.04/0.04 MB (100.0%)\u001b[0m\n" + ] + } + ], + "source": [ + "from great_ai.large_file import LargeFileS3\n", + "import json\n", + "\n", + "LargeFileS3.configure_credentials_from_file(\"config.ini\")\n", + "\n", + "with LargeFileS3(\"summary-train-dataset-small\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump((X, y), f)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Filter out sentences which don't have enough annotations." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "y1 = [e[0] for e in evaluations.values() if len(e) == 2]\n", + "y2 = [e[1] for e in evaluations.values() if len(e) == 2]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Calculate [Cohen's kappa](https://en.wikipedia.org/wiki/Cohen%27s_kappa).\n", + "\n", + "It's a bit low but the task itself is pretty subjective so it's not all that surprising." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.3546448712421808" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import sklearn.metrics\n", + "\n", + "sklearn.metrics.cohen_kappa_score(y1, y2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Can we train anything on this data?\n", + "\n", + "Let's try with a trivial SVM." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "X = [s for s in evaluations.keys()]\n", + "y = [int(sum(e) > 0) for e in evaluations.values()]" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([0.79, 0.75, 0.77, 0.69, 0.77])" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from sklearn.model_selection import train_test_split\n", + "from sklearn.svm import LinearSVC\n", + "from sklearn.pipeline import Pipeline\n", + "from sklearn.feature_extraction.text import TfidfVectorizer\n", + "from sklearn.model_selection import cross_val_score\n", + "\n", + "\n", + "model = Pipeline(\n", + " steps=[\n", + " (\"vectorizer\", TfidfVectorizer(sublinear_tf=True, min_df=3, max_df=0.3)),\n", + " (\"classifier\", LinearSVC()),\n", + " ]\n", + ") # baseline model\n", + "\n", + "cross_val_score(model, X, y, cv=5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The cross-validation shows promising accuracies. But accuracy isn't everything, therefore, we should investigate the accuracy metrics." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
Pipeline(steps=[('vectorizer',\n",
+       "                 TfidfVectorizer(max_df=0.3, min_df=3, sublinear_tf=True)),\n",
+       "                ('classifier', LinearSVC())])
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" + ], + "text/plain": [ + "Pipeline(steps=[('vectorizer',\n", + " TfidfVectorizer(max_df=0.3, min_df=3, sublinear_tf=True)),\n", + " ('classifier', LinearSVC())])" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)\n", + "model.fit(X_train, y_train)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " precision recall f1-score support\n", + "\n", + " False 0.78 0.78 0.78 51\n", + " True 0.78 0.78 0.78 49\n", + "\n", + " accuracy 0.78 100\n", + " macro avg 0.78 0.78 0.78 100\n", + "weighted avg 0.78 0.78 0.78 100\n", + "\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUUAAAEaCAYAAACGrEV/AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAfQElEQVR4nO3deZxcVZn/8c+3O52NkI0OIYSQhEUWGZIwkVUxgGyKg7ihoIIDooLiiI7KojgI/FxYdJRlAiiogGwqgkqCSIblxxYghEDYSQIkIQtZydZd/cwf93a6snVXJ1V1q7q+79frvqi7P0VTD+fcc885igjMzCxRl3UAZmaVxEnRzCyPk6KZWR4nRTOzPE6KZmZ5nBTNzPI4KZpZlyGpXtLTku5O10dKekzSK5JukdS9o2s4KZpZV/INYHre+k+AyyNiF2ARcEpHF3BSNLMuQdIOwEeAa9N1AYcCt6eH3AB8rKPrOCmaWVfxc+A7QEu6vg2wOCKa0/U3gaEdXaRbSULLWOPA+hgxrCHrMKwTXpraO+sQrJOWsWhBRAzakmscechWsfCdXEHHPjl19XPAqrxN4yNiPICkY4B5EfGkpHFbElOXTIojhjXw+IRhWYdhnXDk9qOzDsE66R9x+8wtvcaCd3I8NmGHgo5tGPLqqogYu4ndBwH/JunDQE+gL/ALoL+kbmlpcQfgrY7u4+qzmWUoyEVLQUu7V4k4OyJ2iIgRwGeAf0bEicD9wCfTw04C7uwoIidFM8tMAC1EQctm+i5wlqRXSJ4xXtfRCV2y+mxm1SEImqKwZ4oFXzNiEjAp/fwasG9nzndSNLNMbUEpsCScFM0sMwHknBTNzNq4pGhmlgogV2FTojgpmllmgqDJJUUzs1RArrJyopOimWUneU+xsjgpmlmGRA5lHcQ6nBTNLDMBtLj6bGbWxiVFM7NUAE1RWUMwOCmaWWaSHi0uKZqZARCIXIUN1uWkaGaZagmXFM3MAFefzczWEYimqKw0VFnRmFnNcUnRzCwVIXJ+JcfMrE2LS4pmZomkocUlRTOzlKvPZmZrJd386rMOYx1OimaWGfdoMTNbT4urz2ZmCTe0mJnlCUTOfZ/NzNq0uKRoZpaIkFufzcxaBfg9RTOzfMVoaJHUE3gA6EGS126PiPMlXQ98EFiSHnpyRExp71pOimaWmUDFGmR2NXBoRCyX1AA8JOnv6b7/jIjbC72Qk6KZZaoYJcWICGB5utqQLps1eWplVebNrKYkg8zWF7R0RFK9pCnAPODeiHgs3XWRpKmSLpfUo6PrOCmaWWaCpEdLIQvQKGly3nLaOteKyEXEaGAHYF9JewFnA7sD7wMGAt/tKCZXn80sU50YeXtBRIzt6KCIWCzpfuCoiLgk3bxa0m+Ab3d0vkuKZpaZCHWmpLhJkgZJ6p9+7gUcDrwgaUi6TcDHgGkdxeSSopllqkjvKQ4BbpBUT1LYuzUi7pb0T0mDAAFTgK90dCEnRTPLTFCc6QgiYiowZiPbD+3stZwUzSwzgWhqcTc/M7O1PHSYmVmqiD1aisZJ0cwy5aHDzMxSEXiQWTOzfK4+m5mlWvs+V5LKqszXoFwOTj/8PXz/CyMBmDurO2d+ZFdOPnAPLvrycJrWbPz/on/45bacfOAenPL+3Zk8aeu125+4f2tOef/unHzgHtzyy23L8h1qxVmXzeKWqc/xP/98ce22DxyzmPH3v8Df33yGXfdesclzx45byrUPvsBvHp7Op7/29trtg4et5hd3v8xvHp7OOVfPoFtDS0m/Q6VJ+j6roKVcSpYUJeUkTclbRrRz7PJN7evq/nztIIbtunrt+rUXDeHjX5rP9f9/On3657jn5oEbnDPzpR5MunMA4+9/gYtueo1fnb0DuVySYK84ZwcuvPE1rpn0AvffOYCZL3U4KIgVaOItAzn3xJHrbJvxQk8uOHUEzz661SbPq6sLzrj4Lc47cSRfGrcbhxy7mB13XQXAqefO4Y/XNPLFg/Zg+eJuHPXZd0r6HSpPcbr5FVMp77QyIkbnLTNKeK+qNH92A4/f15ejT1gIJA+dn3loaz5wzGIADv/UOzxyT78NzntkQj/GHbuI7j2C7XZcw/YjVvPi07158enebD9iNUOGr6GhezDu2EU8MmHD823zTHusD8sWrfvE6Y1XevLmqz3bPW+3MSuYPaM7c2f1oLmpjkl39ueAI5cAwaj3L+fBu/sDcO9tAzjgqCXtXqsrakEFLeVStvQrqY+k+yQ9JelZScdu5Jghkh5IS5bTJH0g3X6EpEfSc2+T1KdccZfS1ecP5dTzZqP0r7D0nXq26pejPv3dNQ5pYsHchg3OWzCngUHbN61dbxzSxMK5DSycu+H2BXM2PN/Ka5vtmpg/u/va9QVzGmgc0kTfgTneXVJPS05t27drzirMTLS2PheylEspk2KvvKrzn4BVwHERsQ9wCHBpOnJFvhOACemYaKOAKZIagfOAD6XnTgbOWv9mkk5rHWdt/sJcCb9WcTx6b1/6Nzaz694rsw7FLDOBaG6pL2gpl1K2Pq9MkxsA6bwJF0s6GGgBhgKDgbl55zwB/Do99s8RMUXSB4E9gYfTHNodeGT9m0XEeGA8wNhRPTdrGPJyev6JrXh0Yl+euG9P1qwWK5bVc9UPhvLuknpyzVDfrbXk0LTBuY1Dmpg/u60EuGBOA9ukx62/vXHIhudbeSUl+DVr11tL8K01g7r6oCWntGZQey+ElLNqXIhytj6fCAwC/jVNlm8D6zyMiYgHgIOBt4DrJX2BZMife/OeTe4ZEaeUMe6S+Pdz5nDjk8/z28ef5+yrZjLq/cv43hWzGHVQ/jOmgemzp3Xtf8RSJt05gDWrxdxZ3Xnr9R7sNmYFu41ewVuv92DurO40rRGT7hzA/kcsLfM3s/W9OKU3Q0euYfCw1XRraGHcsYt5dGI/QDzzcJ+8Z8i19wy4Elufy/m/pX7AvIhoknQIMHz9AyQNB96MiGvSuRT2AS4CrpC0S0S8ImkrYGhEvFTG2MvmlHNnc/FXh3P9T4ewy14rOTJtjXxkQl9eeqY3J31nLiN2W8XBH13MaeN2p74++NrFb1Kf1i7OuOhNzjlhJ1py4ojPvMOI3VZl+G26lu9dOZO9D1hOv4HN/H7y8/zu0sEsW9SN0y98i37bNPOj373Oq8/15NwTdmbg4Ca+eckbfP/zyd/iinOHcvFNr1FXDxP/MJCZLyXlgesuGsI5V83k5O/M5ZVpvZiwkbcNurpytiwXQskkWCW4sLQ8IvrkrTcCdwF9SJ4L7g8cHREzWo+VdBLwn0ATycxcX4iI1yUdCvyEZE5XgPMi4i+buvfYUT3j8QnDSvK9rDSO3H501iFYJ/0jbn+ykOkB2jNw923jsF9/oqBjbz/o6i2+XyFKVlLMT4jp+gLggPaOjYgbgBs2sv+fJBPPmFkXUqxBZoup9p7qmlnFCKC5pbKqz06KZpYpDwhhZpbyILNmZuvxM0Uzs1bh6rOZ2VqtL29XEidFM8tM0vfZrc9mZmuFS4pmZm3c0GJmlgo3tJiZravSqs+V9YTTzGqMyLXUFbS0exWpp6THJT0j6TlJ/5VuHynpMUmvSLpFUvd2L4STopllqIjjKa4GDo2IUcBo4ChJ+5OMrnV5ROwCLAI6HIvVSdHMshPJc8VClnYvk2idFbQhXQI4FLg93X4D8LGOQnJSNLNMFWs2P0n1kqYA84B7gVeBxRHROhvYmyTToLTLDS1mlpmgUw0tjZIm562PT+dmSq4VkQNGS+oP/AnYfXNiclI0swx1apScBYWMvB0RiyXdTzKodX9J3dLS4g4k8z+1y9VnM8tUS4sKWtojaVBaQkRSL+BwYDpwP/DJ9LCTgDs7isclRTPLTNKIUpT3FIcAN0iqJyns3RoRd0t6HviDpAuBp4HrOrqQk6KZZaoYPVoiYiowZiPbXwP27cy1nBTNLFMlmlB0szkpmlmmKq2bn5OimWXGc7SYmeUrXkNL0Tgpmlm2/EzRzKxN1ZQUJf2SdnJ4RJxZkojMrKZUU+vz5Hb2mZltsU72fS6LTSbFiLghf11S74hYUfqQzKxmBEQHXfjKrcO+z5IOSLvKvJCuj5J0ZckjM7PaEAUuZVLIgBA/B44EFgJExDPAwSWMycxqhogobCmXglqfI+INaZ2gcqUJx8xqThU1tLR6Q9KBQEhqAL5BMiSPmdmWqcCXtwupPn8FOINkGO/ZJJPCnFHCmMysllTYM8UOS4oRsQA4sQyxmFktqraSoqSdJN0lab6keZLulLRTOYIzsxpQYSXFQqrPNwG3koxsuz1wG3BzKYMysxoRJCXFQpYyKSQp9o6I30VEc7r8HuhZ6sDMrDYUY97nYmqv7/PA9OPfJX0P+ANJXj8e+FsZYjOzWlBFr+Q8SRJua7n1y3n7Aji7VEGZWe1QhXXza6/v88hyBmJmNajMjSiFKKhHi6S9gD3Je5YYEb8tVVBmVivK24hSiA6ToqTzgXEkSfFvwNHAQ4CTopltuQorKRbS+vxJ4DBgbkR8ERgF9CtpVGZWOyrsPcVCqs8rI6JFUrOkvsA8YFiJ4zKzWlFhJcVCkuJkSf2Ba0hapJcDj5QyKDOrEVFFrc+tIuL09OPVku4B+kbE1NKGZWY1o1pKipL2aW9fRDxVmpDMzLLTXknx0nb2BXBokWMxsxqkaikpRsQh5QykmF6a2psjtx+ddRjWCZfN8GPqarP38CJdqAjvKUoaRvKa4GCSQtv4iPiFpB8CXwLmp4eeExHtdlMu6OVtM7OSKN7rNs3AtyLiKUlbA09Kujfdd3lEXFLohZwUzSxTatnya0TEHGBO+nmZpOkkswV0WiEvb5uZlU6RX96WNAIYAzyWbvqapKmSfi1pQEfnFzLytiR9TtIP0vUdJe1beIhmZu0oPCk2Spqct5y2/qUk9QHuAP4jIpYCVwE7k8wtNYf2G5CBwqrPVwItJK3NFwDL0pu+r4Bzzcw2SdGp1ucFETF2k9dKZhu9A7gxIv4IEBFv5+2/Bri7o5sUUn3eLyLOAFalN1kEdC/gPDOzjhVhOgIlE9NfB0yPiMvytg/JO+w4YFpH4RRSUmySVE9agJU0iKTkaGa2xYrR0AIcBHweeFbSlHTbOcBnJY0myV8zWHew7I0qJCn+N/AnYFtJF5GMmnNep0M2M9uYIrySExEP0TZLQL5OT51SSN/nGyU9STJ8mICPRcT0zt7IzGwDnXumWBaFDDK7I7ACuCt/W0TMKmVgZlYjqi0pAn+lbQKrnsBI4EXgvSWMy8xqRbUlxYj4l/z1dPSc0zdxuJlZp1Rd9Xl9ad/C/UoRjJnVoGpLipLOylutA/YBZpcsIjOrHdXY0AJsnfe5meQZ4x2lCcfMak41JcX0pe2tI+LbZYrHzGpNtSRFSd0iolnSQeUMyMxqh6iu6vPjJM8Pp0j6C3Ab8G7rztYO12ZmW6SKkmKrnsBCklFyWt9XDMBJ0cy2TBSt73PRtJcUt01bnqfRlgxbVVhuN7OqVWHZpL2kWA/0YeOdrCvsa5hZtaqmZ4pzIuKCskViZrWpipLils87aGbWnuLN5lc07SXFw8oWhZnVrKppaImId8oZiJnVpmp6pmhmVnpOimZmqSp7pmhmVlKi8lp0nRTNLFsuKZqZtama1mczs7JwSdHMLFWlI2+bmZWOk6KZWRuXFM3M8jkpmpmlKnCQ2bqsAzCzGhcFLu2QNEzS/ZKel/ScpG+k2wdKulfSy+k/B3QUjpOimWWmdeKqQpYONAPfiog9gf2BMyTtCXwPuC8idgXuS9fb5aRoZtkqQkkxIuZExFPp52XAdGAocCxwQ3rYDcDHOgrHzxTNLFOK4ra0SBoBjAEeAwZHxJx011xgcEfnOymaWXY619DSKGly3vr4iBiff4CkPsAdwH9ExFKpbbiJiAip44q4k6KZZavwguKCiBi7qZ2SGkgS4o1589K/LWlIRMyRNASY19FN/EzRzDJVjIYWJUXC64DpEXFZ3q6/ACeln08C7uwoHpcUzSxbxXmkeBDweeBZSVPSbecAPwZulXQKMBP4dEcXclI0s+wUaUCIiHiITY9X26lJ+JwUzSxb7uZnZpYQoJbKyopOimaWKY+SY2udddks9vvQMhYv6MaXD90NgA8cs5jPf2suw3ZdzZkf3pWXp/be6Lljxy3lKz+aTX1d8PebB3Lrr5J3UgcPW805V82i74BmXn62Fz/9+o40N/klg2JoWiV+dfxeNK8WLTkx6uiFHHXWm7z0cF/uung40SJ6bJXjM5e8yqARqzY4/x9XbM9jtw6mrj447vzX2f2DSwCYPqk/f75gBC05sf/xb3PY6bPL/dWyU4Gz+ZXl1yJpG0lT0mWupLfy1ruXI4ZKNPGWgZx74sh1ts14oScXnDqCZx/dapPn1dUFZ1z8FuedOJIvjduNQ45dzI67Jj/CU8+dwx+vaeSLB+3B8sXdOOqz75T0O9SSbj2C0296jv+8Zyrf/ttUXvjf/sx4qg93nLcTn/vFK3z771PZ59gF/OOXQzc4d+7LvXj6rka+O3EKp90wnTu+vxMtOWjJwR9/MJLTrp/Od++dwlN/aWTuy70y+HbZUUthS7mUJSlGxMKIGB0Ro4Grgctb1yNijaSaLLFOe6wPyxat+9XfeKUnb77as93zdhuzgtkzujN3Vg+am+qYdGd/DjhyCRCMev9yHry7PwD33jaAA45aUqLoa48EPbZKfp25ZpFrFkrn6Fy1rB6AlUvr6Tt4zQbnTps4gDEfXUC3HsE2w1bTOHwVs6b0YdaUPjQOX8U2O66mW/dgzEcXMG1ihwO5dC1F6PtcTJklI0nXA6tI+ig+LGkpsDwiLkn3TwOOiYgZkj4HnAl0J+nPeHpE5LKJPHvbbNfE/NltBewFcxrYfZ8V9B2Y490l9bTktHZ743bNWYXZJbXk4LJj9mbBzJ4c9Pm5DB+znON//CrXfHF3Gnq20LNPjm/8adoG5y15uwfDxyxbu95vyBqWvJ38Dftvv3rt9v5D1jBzytal/yKVIiqvoSXrh007AAdGxFmbOkDSHsDxwEFpSTMHnFie8MzWVVcP3/77VM5/5ElmPdOHOS/24n+vG8KXfvMC5z/6FO/71HzuvHB41mFWlSINHVY0WVdbbyugxHcY8K/AE2nn7l5spP+ipNOA0wB6svHGia5i4dwGBm3fVkVrHNLEgjkNLH2nnq365airD1pySrbPzfpP3DX16pdjlwOWMn3SAGZP34rhY5YDMOaYBYw/aY8Nju83eDWL80r3S+Z0p19azV48u8fa7YvndKff4NUbnN+lVVZBMfOS4rt5n5tZN57WB2sCbsh7BrlbRPxw/QtFxPiIGBsRYxvosf7uLuXFKb0ZOnINg4etpltDC+OOXcyjE/sB4pmH+/CBYxYDcPinFvHIhH6ZxtqVLF/YjZVLkmeHa1bV8dJD/Ri8ywpWLatn3mvJf64vPtSfbXdZucG5ex2+iKfvaqR5tVj4Rg/mz+jJjqOXM2zUcubP6MnCN3rQvEY8fVcjex2+qKzfK0tFHGS2aCqpGDEDOAZA0j5Aa7PsfcCdki6PiHmSBgJbR8TMbMIsnu9dOZO9D1hOv4HN/H7y8/zu0sEsW9SN0y98i37bNPOj373Oq8/15NwTdmbg4Ca+eckbfP/zO9GSE1ecO5SLb3qNunqY+IeBzHwp+VFed9EQzrlqJid/Zy6vTOvFhJsHZvwtu46l87pz87d2oaUFokWM+shC3nvYYj71/17j+q/uhhT07tfMZ372KgDT7h3AG8/24eiz3mC796xk9DEL+cnho6nrFnzigtepS/IrH7/gdcZ/YQ9acmLfT89ju/dsmFS7rIhkqSCKMgck6YfAcmAv4O6IuD3d3otkBIuhJI0pBwBHpw0txwNnk5Qkm4AzIuLRTd2jrwbGfupUd0fL2GUzHsk6BOukvYe/9WR7Q3kVYuv+O8SYD36joGMf/Mt3tvh+hSh7SXFjVd90+0rgiE3suwW4pYRhmVlGKm02v0qqPptZrQmgwl7JcVI0s2xVVk50UjSzbHlACDOzfBXW+uykaGaZcknRzCylCuz77KRoZtnyKzlmZm3kZ4pmZqkKHHnbSdHMMlR5fZ+dFM0sU25oMTNrFe77bGa2LlefzczyVFZOdFI0s2z5lRwzs3wVlhSznqPFzGqYIlCusKXDa0m/ljQvnR65ddsPJb0laUq6fLij6zgpmlm2Wudp6Wjp2PXAURvZfnnexHd/6+girj6bWbaKVH2OiAckjdjS67ikaGbZCZIBIQpZNt/XJE1Nq9cDOjrYSdHMMqWIghagUdLkvOW0Ai5/FbAzMBqYA1za0QmuPptZtgqvPi/o7BSnEfF262dJ1wB3d3SOk6KZZScCWkrXz0/SkIiYk64eB0xr73hwUjSzrBUpJ0q6GRhHUs1+EzgfGCdpNMnTyxnAlzu6jpOimWWqWD1aIuKzG9l8XWev46RoZtmqsB4tTopmlp0APJ6imVmr0ja0bA4nRTPLlqvPZmYpV5/NzPIFhKvPZmZtXH02M0u5+mxmth63PpuZtSp4ANmycVI0s+wELimama3DJUUzszxOimZmrcKtz2ZmawVELpd1FOtwUjSzbLn6bGaWKvF0BJvDSdHMsuWSoplZm3BJ0cwsFQE5J0UzszYeOszMLBFA+D1FM7NUeJBZM7N1uKRoZpavwkqKigp7R6gYJM0HZmYdR4k0AguyDsI6pav+zYZHxKAtuYCke0j+/RRiQUQctSX3K0SXTIpdmaTJETE26ziscP6bVZe6rAMwM6skTopmZnmcFKvP+KwDsE7z36yK+JmimVkelxTNzPI4KZqZ5XFSNCsiJT4n6Qfp+o6S9s06Liuck2IVkNRb0vclXZOu7yrpmKzjso26EjgA+Gy6vgy4IrtwrLOcFKvDb4DVJD82gLeAC7MLx9qxX0ScAawCiIhFQPdsQ7LOcFKsDjtHxE+BJoCIWAEo25BsE5ok1ZOMioWkQUBlde61djkpVoc1knrR9kPbmaTkaJXnv4E/AdtKugh4CLg425CsM/yeYhWQdDhwHrAnMBE4CDg5IiZlGZdtnKTdgcNISvP3RcT0jEOyTnBSrBKStgH2J/mhPRoRXXHUlaonaceNbY+IWeWOxTaPk2IVkHQQMCUi3pX0OWAf4BcR0VWHR6takp4lecwhoCcwEngxIt6baWBWMD9TrA5XASskjQLOAl4FfpttSLYxEfEvEbF3+s9dgX2BR7KOywrnpFgdmiMp0h8LXBERVwBbZxyTFSAingL2yzoOK5ynI6gOyySdDXwOOFhSHdCQcUy2EZLOylutI3nUMTujcGwzuKRYHY4neQXnlIiYC+wA/CzbkGwTts5begB/JSnhW5VwQ4tZkaQvbf8kIr6ddSy2+Vx9rmCSlpG+sL3+LiAiom+ZQ7JNkNQtIprTNwWsirmkaFYEkp6KiH0kXQUMBW4D3m3dHxF/zCw46xSXFKuIpG1J3n0D/EJwheoJLAQOpe19xQCcFKuEk2IVkPRvwKXA9sA8YDgwHfALwZVj27TleRptybCVq2NVxK3P1eFHJF38XoqIkST9ah/NNiRbTz3QJ122zvvculiVcEmxOjRFxEJJdZLqIuJ+ST/POihbx5yIuCDrIGzLOSlWh8WS+gAPADdKmkfeQ3yrCB7fsotw63MFk7RjRMyStBWwkuRxx4lAP+DGiFiYaYC2lqSBEfFO1nHYlnNSrGCtr3mkn++IiE9kHZNZV+eGlsqWXyXbKbMozGqIk2Jli018NrMScfW5gknKkTSoCOgFrGjdhbv5mZWEk6KZWR5Xn83M8jgpmpnlcVKsUZJykqZImibpNkm9t+Ba10v6ZPr5Wkl7tnPsOEkHbsY9ZkhqLHT7escs7+S9fijJYyLWKCfF2rUyIkZHxF7AGuAr+TslbVZvp4g4NSKeb+eQcUCnk6JZuTgpGsCDwC5pKe5BSX8BnpdUL+lnkp6QNFXSlwGU+JWkFyX9A9i29UKSJkkam34+StJTkp6RdJ+kESTJ95tpKfUDkgZJuiO9xxOtg7RK2kbSREnPSbqWArrRSfqzpCfTc05bb9/l6fb7JA1Kt+0s6Z70nAfTSeytxrnvc41LS4RHA/ekm/YB9oqI19PEsiQi3iepB/CwpInAGGA3YE9gMPA88Ov1rjsIuAY4OL3WwIh4R9LVwPKIuCQ97ibg8oh4KJ1IfgKwB3A+8FBEXCDpI8ApBXydf0/v0Qt4Iu0FtBDYCpgcEd+U9IP02l8DxgNfiYiXJe0HXEkyDqLVMCfF2tVL0pT084PAdSTV2scj4vV0+xHA3q3PC0n6XO8KHAzcHBE5YLakf27k+vsDD7Req51+wR8C9pTWFgT7poNfHAx8PD33r5IWFfCdzpR0XPp5WBrrQqAFuCXd/nvgj+k9DgRuy7t3jwLuYV2ck2LtWhkRo/M3pMkhf/QdAV+PiAnrHffhIsZRB+wfEas2EkvBJI0jSbAHRMQKSZPIG6V8PZHed/H6/w7M/EzR2jMB+KqkBgBJ70lH7HkAOD595jgEOGQj5z5KMkf1yPTcgen2ZSSDsLaaCHy9dUXS6PTjA8AJ6bajgQEdxNoPWJQmxN1JSqqt6oDW0u4JJNXypcDrkj6V3kOSRnVwD6sBTorWnmtJnhc+JWka8D8ktYs/AS+n+34LPLL+iRExHziNpKr6DG3V17uA41obWoAzgbFpQ87ztLWC/xdJUn2OpBrd0Xw09wDdJE0Hfsy6I5O/C+ybfodDgdbBYE8ETknjew7Pz2y4m5+Z2TpcUjQzy+OkaGaWx0nRzCyPk6KZWR4nRTOzPE6KZmZ5nBTNzPI4KZqZ5fk/zhlvHEqJskYAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "y_predicted = model.predict(X_test)\n", + "print(\n", + " sklearn.metrics.classification_report(\n", + " [y > 0 for y in y_test], [y > 0 for y in y_predicted]\n", + " )\n", + ")\n", + "sklearn.metrics.ConfusionMatrixDisplay.from_predictions(\n", + " [y > 0 for y in y_test],\n", + " [y > 0 for y in y_predicted],\n", + " xticks_rotation=\"vertical\",\n", + " values_format=\".2f\",\n", + ")\n", + "None" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We get an F1-score of 0.78 without any hyperparameter-optimisation; this task may be feasible to solve with AI.\n", + "\n", + "Next: [Part 2](/examples/scibert/train)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.10.4 ('.env': venv)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.4" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "02dd6d3afbfa9fbbe1037d64ad9014965528a1ccad21929d6e72f466389a68ad" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/examples/scibert/deploy.ipynb b/docs/examples/scibert/deploy.ipynb index 8531de7..905f369 100644 --- a/docs/examples/scibert/deploy.ipynb +++ b/docs/examples/scibert/deploy.ipynb @@ -1,5 +1,94 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Create an inference function\n", + "\n", + "Everything is ready to wrap the previously trained model and deploy it. \n", + "\n", + "First, we need to configure the LargeFileBackend, the TracingDatabase and GreatAI." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[38;5;226mThe value of `ENVIRONMENT` contains the \"ENV` prefix but `ENVIRONMENT` is not defined as an environment variable, using the default value defined above (`DEVELOPMENT`)\u001b[0m\n", + "\u001b[38;5;226mEnvironment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n", + "\u001b[38;5;39mMongoDbDriver has been already configured: skipping initialisation\u001b[0m\n", + "\u001b[38;5;39mLargeFileS3 has been already configured: skipping initialisation\u001b[0m\n", + "\u001b[38;5;39mGreatAI (v0.1.6): configured ✅\u001b[0m\n", + "\u001b[38;5;39m 🔩 tracing_database: MongoDbDriver\u001b[0m\n", + "\u001b[38;5;39m 🔩 large_file_implementation: LargeFileS3\u001b[0m\n", + "\u001b[38;5;39m 🔩 is_production: False\u001b[0m\n", + "\u001b[38;5;39m 🔩 should_log_exception_stack: True\u001b[0m\n", + "\u001b[38;5;39m 🔩 prediction_cache_size: 4096\u001b[0m\n", + "\u001b[38;5;39m 🔩 dashboard_table_size: 100\u001b[0m\n", + "\u001b[38;5;226mYou still need to check whether you follow all best practices before trusting your deployment.\u001b[0m\n", + "\u001b[38;5;226m> Find out more at https://se-ml.github.io/practices\u001b[0m\n" + ] + } + ], + "source": [ + "from great_ai.utilities import ConfigFile\n", + "from great_ai.large_file import LargeFileS3\n", + "from great_ai import configure, MongoDbDriver\n", + "\n", + "configuration = ConfigFile(\"config.ini\")\n", + "\n", + "LargeFileS3.configure_credentials_from_file(configuration)\n", + "MongoDbDriver.configure_credentials_from_file(configuration)\n", + "\n", + "configure(\n", + " dashboard_table_size=100, # traces are small, we can show many\n", + " prediction_cache_size=4096, # predictions are expensive, cache them\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For a pleasant developer experience, we create some typed models that will show up in the automatically generated OpenAPI schema specification and will also provide runtime type validation." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from typing import List\n", + "from pydantic import BaseModel\n", + "\n", + "\n", + "class Attention(BaseModel):\n", + " weight: float\n", + " token: str\n", + "\n", + "\n", + "class EvaluatedSentence(BaseModel):\n", + " score: float\n", + " text: str\n", + " explanation: List[Attention]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Even though `@use_model` caches the remote files locally and it also handles deserialising objects, we only use it to store a directory. In this case, it gives back a path, the path to that directory. So, we need to load the files from that folder ourselves. In order to only load it once per process, we create a small model loader helper function.\n", + "\n", + "> This is usually not needed, however, when we can outsmart `dill` so for optimisation purposes, we do it." + ] + }, { "cell_type": "code", "execution_count": 3, @@ -9,86 +98,96 @@ "name": "stderr", "output_type": "stream", "text": [ - "\u001b[38;5;226m2022-07-01 14:28:43,377 | WARNING | Environment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n", - "\u001b[38;5;39m2022-07-01 14:28:43,378 | INFO | Found credentials file (/data/projects/scoutinscience/platform/projects/highlights-service2/mongo.ini), initialising MongoDbDriver\u001b[0m\n", - "\u001b[38;5;39m2022-07-01 14:28:43,379 | INFO | Found credentials file (/data/projects/scoutinscience/platform/projects/highlights-service2/s3.ini), initialising LargeFileS3\u001b[0m\n", - "\u001b[38;5;39m2022-07-01 14:28:43,380 | INFO | Settings: configured ✅\u001b[0m\n", - "\u001b[38;5;39m2022-07-01 14:28:43,380 | INFO | 🔩 tracing_database: MongoDbDriver\u001b[0m\n", - "\u001b[38;5;39m2022-07-01 14:28:43,381 | INFO | 🔩 large_file_implementation: LargeFileS3\u001b[0m\n", - "\u001b[38;5;39m2022-07-01 14:28:43,381 | INFO | 🔩 is_production: False\u001b[0m\n", - "\u001b[38;5;39m2022-07-01 14:28:43,382 | INFO | 🔩 should_log_exception_stack: True\u001b[0m\n", - "\u001b[38;5;39m2022-07-01 14:28:43,382 | INFO | 🔩 prediction_cache_size: 512\u001b[0m\n", - "\u001b[38;5;39m2022-07-01 14:28:43,383 | INFO | 🔩 dashboard_table_size: 50\u001b[0m\n", - "\u001b[38;5;226m2022-07-01 14:28:43,384 | WARNING | You still need to check whether you follow all best practices before trusting your deployment.\u001b[0m\n", - "\u001b[38;5;226m2022-07-01 14:28:43,385 | WARNING | > Find out more at https://se-ml.github.io/practices/\u001b[0m\n", - "\u001b[38;5;39m2022-07-01 14:28:44,082 | INFO | Latest version of scibert-highlights is 0 (from versions: 0)\u001b[0m\n", - "\u001b[38;5;39m2022-07-01 14:28:44,083 | INFO | File scibert-highlights-0 found in cache\u001b[0m\n", - "\u001b[38;5;39m2022-07-01 14:28:44,084 | INFO | File scibert-highlights-0 found in cache\u001b[0m\n" + "\u001b[38;5;39mLatest version of scibert-highlights is 0 (from versions: 0)\u001b[0m\n", + "\u001b[38;5;39mFile scibert-highlights-0 found in cache\u001b[0m\n" ] } ], "source": [ - "from great_ai import GreatAI, use_model, MongoDbDriver, configure\n", - "from great_ai.utilities import clean\n", + "from great_ai import use_model\n", "from pathlib import Path\n", + "from typing import Tuple\n", + "from transformers import (\n", + " PreTrainedModel,\n", + " PreTrainedTokenizer,\n", + ")\n", "from transformers import (\n", " AutoConfig,\n", " AutoModelForSequenceClassification,\n", " AutoTokenizer,\n", ")\n", + "\n", + "_tokenizer: PreTrainedTokenizer = None\n", + "_loaded_model: PreTrainedModel = None\n", + "\n", + "\n", + "@use_model(\"scibert-highlights\", version=\"latest\", model_kwarg_name=\"model_path\")\n", + "def get_tokenizer_and_model(\n", + " model_path: Path, original_model: str = \"allenai/scibert_scivocab_uncased\"\n", + ") -> Tuple[PreTrainedTokenizer, PreTrainedModel]:\n", + " global _tokenizer, _loaded_model\n", + "\n", + " if _tokenizer is None:\n", + " _tokenizer = AutoTokenizer.from_pretrained(original_model)\n", + "\n", + " if _loaded_model is None:\n", + " config = AutoConfig.from_pretrained(\n", + " model_path, output_hidden_states=True, output_attentions=True\n", + " )\n", + " _loaded_model = AutoModelForSequenceClassification.from_pretrained(\n", + " model_path, config=config\n", + " )\n", + "\n", + " return _tokenizer, _loaded_model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, implement the inference function." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from great_ai import GreatAI\n", + "from great_ai.utilities import clean\n", + "\n", "import re\n", "import numpy as np\n", "import torch\n", "from transformers.modeling_outputs import SequenceClassifierOutput\n", - "from transformers import (\n", - " PreTrainedModel,\n", - " PreTrainedTokenizer,\n", - ")\n", - "from views import EvaluatedSentence, Match\n", - "from great_ai.large_file import LargeFileS3\n", - "\n", - "LargeFileS3.configure_credentials_from_file(\"config.ini\")\n", - "MongoDbDriver.configure_credentials_from_file(\"config.ini\")\n", - "configure(dashboard_table_size=100)\n", - "\n", - "ORIGINAL_MODEL = \"allenai/scibert_scivocab_uncased\"\n", - "\n", - "loaded_model: PreTrainedModel = None\n", - "tokenizer: PreTrainedTokenizer = None\n", "\n", "\n", "@GreatAI.create\n", - "@use_model(\"scibert-highlights\", version=\"latest\")\n", - "def find_highlights(sentence: str, model: Path) -> EvaluatedSentence:\n", - " global loaded_model, tokenizer\n", + "def find_highlights(sentence: str) -> EvaluatedSentence:\n", + " \"\"\"Get the interestingness prediction of the input sentence using SciBERT.\n", "\n", - " if loaded_model is None:\n", - " config = AutoConfig.from_pretrained(\n", - " model, output_hidden_states=True, output_attentions=True\n", - " )\n", - " loaded_model = AutoModelForSequenceClassification.from_pretrained(\n", - " model, config=config\n", - " )\n", - " if tokenizer is None:\n", - " tokenizer = AutoTokenizer.from_pretrained(ORIGINAL_MODEL)\n", + " Run the SciBERT model in inference mode and evaluate the sentence.\n", + " Additionally, provide explanation in the form of the last layer's sum attention\n", + " between `[CLS]` and the other tokens.\n", + " \"\"\"\n", "\n", + " tokenizer, loaded_model = get_tokenizer_and_model()\n", " sentence = clean(sentence, convert_to_ascii=True, remove_brackets=True)\n", "\n", - " return evaluate_sentence(sentence=sentence)\n", - "\n", - "\n", - "def evaluate_sentence(sentence: str) -> EvaluatedSentence:\n", " tensors = tokenizer(sentence, return_tensors=\"pt\", truncation=True, max_length=512)\n", "\n", - " with torch.no_grad():\n", + " with torch.inference_mode():\n", " result: SequenceClassifierOutput = loaded_model(**tensors)\n", " positive_likelihood = torch.nn.Softmax(dim=1)(result.logits)[0][1]\n", " tokens = tensors[\"input_ids\"][0]\n", "\n", " attentions = np.sum(result.attentions[-1].numpy()[0], axis=0)[0][\n", " 1:-1\n", - " ] # Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.\n", - " matches = []\n", + " ] # Tuple of `torch.FloatTensor` (one for each layer) of shape\n", + " # `(batch_size, num_heads, sequence_length, sequence_length)`.\n", + "\n", + " explanation = []\n", "\n", " token_attentions = list(zip(attentions, tokens[1:-1]))\n", " for token in re.split(r\"([ .,])\", sentence):\n", @@ -99,24 +198,101 @@ " token, return_tensors=\"pt\", truncation=True, max_length=512\n", " )[\"input_ids\"][0][\n", " 1:-1\n", - " ] # truncation=True needed to fix RuntimeError: Already borrowed\n", - " score = 0\n", + " ] # truncation=True needed to fix `RuntimeError: Already borrowed`\n", + " weight = 0\n", " for t1 in bert_tokens:\n", " if not token_attentions:\n", " break\n", " a, t2 = token_attentions.pop(0)\n", " assert t1 == t2, sentence\n", - " score += a\n", - " matches.append(\n", - " Match(phrase=token if token in \".,\" else \" \" + token, score=round(score, 4))\n", + " weight += a\n", + " explanation.append(\n", + " Attention(\n", + " token=token if token in \".,\" else \" \" + token, weight=round(weight, 4)\n", + " )\n", " )\n", " if not token_attentions:\n", " break\n", "\n", " return EvaluatedSentence(\n", - " score=positive_likelihood, text=sentence, explanation=matches\n", + " score=positive_likelihood, text=sentence, explanation=explanation\n", " )" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "A simple test to see everything works. Note that the models list is filled by the `@use_model` call even though it's not on the main inference function." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(Trace[EvaluatedSentence]({'created': '2022-07-16T18:47:29.581701',\n", + " 'exception': None,\n", + " 'feedback': None,\n", + " 'logged_values': { 'arg:sentence:length': 51,\n", + " 'arg:sentence:value': 'Our solution has outperformed the '\n", + " 'state-of-the-art.'},\n", + " 'models': [{'key': 'scibert-highlights', 'version': 0}],\n", + " 'original_execution_time_ms': 7127.2063,\n", + " 'output': { 'explanation': [ {'token': ' Our', 'weight': 0.3993},\n", + " {'token': ' solution', 'weight': 0.3481},\n", + " {'token': ' has', 'weight': 0.2945},\n", + " {'token': ' outperformed', 'weight': 0.4011},\n", + " {'token': ' the', 'weight': 0.1484},\n", + " {'token': ' state-of-the-art', 'weight': 0.5727},\n", + " {'token': '.', 'weight': 7.775}],\n", + " 'score': 0.9991180300712585,\n", + " 'text': 'Our solution has outperformed the state-of-the-art.'},\n", + " 'tags': ['find_highlights', 'online', 'development'],\n", + " 'trace_id': '56e20e94-79df-4793-ae61-d20820ebe2d3'}),\n", + " Trace[EvaluatedSentence]({'created': '2022-07-16T18:47:37.020275',\n", + " 'exception': None,\n", + " 'feedback': None,\n", + " 'logged_values': { 'arg:sentence:length': 36,\n", + " 'arg:sentence:value': 'Their solution did not perform '\n", + " 'well.'},\n", + " 'models': [{'key': 'scibert-highlights', 'version': 0}],\n", + " 'original_execution_time_ms': 170.7057,\n", + " 'output': { 'explanation': [ {'token': ' Their', 'weight': 1.1475},\n", + " {'token': ' solution', 'weight': 0.8205},\n", + " {'token': ' did', 'weight': 0.3254},\n", + " {'token': ' not', 'weight': 0.2921},\n", + " {'token': ' perform', 'weight': 0.4293},\n", + " {'token': ' well', 'weight': 0.2772},\n", + " {'token': '.', 'weight': 4.4723}],\n", + " 'score': 0.12305451184511185,\n", + " 'text': 'Their solution did not perform well.'},\n", + " 'tags': ['find_highlights', 'online', 'development'],\n", + " 'trace_id': '7fcf8271-1738-4025-8305-d5a1e5100aea'}))" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "if __name__ == \"__main__\":\n", + " find_highlights(\n", + " \"Our solution has outperformed the state-of-the-art.\"\n", + " ), find_highlights(\"Their solution did not perform well.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this case, the service is built as a docker image, pushed to our image registry and subsequent rolling update is performed in the production cluster.\n", + "To check out the Dockerimage, go to [the additional files page](/examples/scibert/additional-files)." + ] } ], "metadata": { diff --git a/docs/examples/scibert/index.md b/docs/examples/scibert/index.md new file mode 100644 index 0000000..5400d54 --- /dev/null +++ b/docs/examples/scibert/index.md @@ -0,0 +1,50 @@ +# Summarising scientific publications from a tech-transfer perspective + +This is a simplified example illustrating how `great-ai` is used in practice at [ScoutinScience](https://www.scoutinscience.com/){ target=_blank }. The subpages show `great-ai` in action by going over the life-cycle of finetuning and deploying a BERT-based software service. + +??? note "Propriety data" + The purpose of this example is to show you different ways in which `great-ai` can assist you. The exact NLP task being solved is not central. Stemming from this, and from the difficult nature of obtaining appropriate training data, the propriety dataset used for the experiments is not shared. + +## Objectives + +1. You will see how the [great_ai.utilities](/reference/utilities) can integrate into your Data Science workflow. +2. You will see how [great_ai.large_file](/reference/large-file) can be used to version and store your trained model. +3. You will see how [GreatAI][great_ai.GreatAI] should be used prepare your model for a robust and responsible deployment. +4. You will see multiple ways of customising your deployment. + +## Overview + +One of the core features of the ScoutinScience platform is summarising research papers form a tech-transfer perspective. In short, extractive summarisation is preferred using a binary classifier trained on clients' judgement of sentence interestingness. Thus, documents are sentences and the expected output is a binary label showing whether a sentence is "worthy" of being in the tech-transfer summary. Providing an explanation for each decision is imperative since ScoutinScience embraces applying only explainable AI (XAI) methods wherever feasible. + +!!! success + You are ready to start the tutorial. Feel free to come back to the [summary](#summary) section once you're finished. + +
+[:material-database: Examine data](data.ipynb){ .md-button .md-button--primary } + +[:fontawesome-solid-chart-simple: Train model](train.ipynb){ .md-button .md-button--primary } + +[:material-cloud-tags: Deploy service](deploy.ipynb){ .md-button .md-button--primary } +
+ +## Summary + +### [Data notebook](data.ipynb) + +We load and analyse the data by calculating inter-rater reliability and checking the feasibility of using an AI-based approach by testing the accuracy of a trivial baseline method. + +### [Training notebook](train.ipynb) + +We simply fine-tune SciBERT. + +After training and evaluating a model, it is exported using [great_ai.save_model][]. For more info, checkout [the configuration how-to page](/how-to-guides/configure-service). + +### [Deployment notebook](deploy.ipynb) + +We customise the GreatAI configuration, create custom cahcing for the model and implement an inference function that can be hardened by wrapping it in a [GreatAI][great_ai.GreatAI] instance. We also extract the attention weights as a quasi-explanation. + +Finally, we test the model's inference function through the GreatAI dashboard. [The only thing left is to deploy the hardened service properly.](/how-to-guides/use-service) + +#### [Additional files](additional-files.md) + +There are some other files required for deploying the notebook. For example, the config file for S3 and MongoDB or a Dockerfile for building a custom image. These are compiled and shown on a separate page. diff --git a/docs/examples/scibert/requirements.txt b/docs/examples/scibert/requirements.txt deleted file mode 100644 index 1dd0570..0000000 --- a/docs/examples/scibert/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -torch -transformers -numpy diff --git a/docs/examples/scibert/train.ipynb b/docs/examples/scibert/train.ipynb index cfc0645..ef3a462 100644 --- a/docs/examples/scibert/train.ipynb +++ b/docs/examples/scibert/train.ipynb @@ -1,210 +1,90 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Fine-tune SciBERT\n", + "\n", + "We are planning to do a simple classification task on scientific text. For that, [SciBERT](https://github.com/allenai/scibert) is an ideal model to fine-tune since it has been pretrained of academic publications.\n", + "\n", + "This notebook was updated so that it can run in [Google Colab](https://colab.research.google.com/).\n", + "\n", + "First, we need to install the dependencies." + ] + }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 1, "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, "executionInfo": { "elapsed": 2529, "status": "ok", "timestamp": 1656596749103, - "user": { - "displayName": "Schmelczer András", - "userId": "08401926777942666437" - }, "user_tz": -120 }, "id": "j7l0nD9hDQbB", "outputId": "88a9931b-396a-4cf1-c659-8a7b098b3cdd" }, + "outputs": [], + "source": [ + "!pip install transformers datasets great-ai > /dev/null" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Load the training data from S3. (We have uploaded this to S3 in the `data` notebook.)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, "outputs": [ { - "name": "stdout", + "name": "stderr", "output_type": "stream", "text": [ - "evaluation-experiment-2-stage #1-2m6dmb.json\n", - "evaluation-experiment-2-stage #1-sa6a0y.json\n", - "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n", - "Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.20.1)\n", - "Requirement already satisfied: datasets in /usr/local/lib/python3.7/dist-packages (2.3.2)\n", - "\u001b[31mERROR: Could not find a version that satisfies the requirement great-ai==0.0.6 (from versions: none)\u001b[0m\n", - "\u001b[31mERROR: No matching distribution found for great-ai==0.0.6\u001b[0m\n" + "\u001b[38;5;39mLatest version of summary-train-dataset-small is 0 (from versions: 0)\u001b[0m\n", + "\u001b[38;5;39mFile summary-train-dataset-small-0 found in cache\u001b[0m\n" ] } ], "source": [ - "from pathlib import Path\n", + "from great_ai.large_file import LargeFileS3\n", "import json\n", "\n", - "annotations = []\n", - "for p in Path(\".\").glob(\"*.json\"):\n", - " with open(p, encoding=\"utf-8\") as f:\n", - " print(p)\n", - " annotations.append(json.load(f))\n", + "LargeFileS3.configure_credentials_from_file(\"config.ini\")\n", "\n", - "evaluations = {\n", - " sentence: [\n", - " annotation[sentence] for annotation in annotations if sentence in annotation\n", - " ]\n", - " for sentence in {\n", - " sentence for annotation in annotations for sentence in annotation.keys()\n", - " }\n", - "}\n", - "\n", - "X = [s for s in evaluations.keys()]\n", - "y = [int(sum(e) > 0) for e in evaluations.values()]\n", - "\n", - "# !pip install transformers datasets great-ai==0.0.6" + "with LargeFileS3(\"summary-train-dataset-small\", encoding=\"utf-8\") as f:\n", + " # splitting training and test data is done later by `datasets`\n", + " X, y = json.load(f)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finetune SciBERT, for more info about this step, check out [HuggingFace](https://huggingface.co/docs/transformers/training).\n", + "If you're only here for `great-ai`, feel free to skip the next cell." ] }, { "cell_type": "code", "execution_count": 22, "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 1000, - "referenced_widgets": [ - "9c57de70e68a41ecbde5093bd671715a", - "9185277b1e5945a9a6a3ad75811bc86b", - "4be5b3c59dc04aa2b92f51654e815589", - "8e0ede9d1dd84c149a7e282211c7071b", - "9bd5d2fb87bd428796155cc67d06b333", - "bf773a86ec0a4899bb3636035f7ab35e", - "22119b79eb514ad684cc3f00f519fb4a", - "cb7ec6240337466d8833c70083e1c3cb", - "34631de39509438aad98cbd3fc64c999", - "32adc54185894f0598c2d9ad438c76e2", - "981e11fb9d4f4a2ba28c011741a1eaba" - ] - }, "executionInfo": { "elapsed": 118131, "status": "ok", "timestamp": 1656593941974, - "user": { - "displayName": "Schmelczer András", - "userId": "08401926777942666437" - }, "user_tz": -120 }, "id": "AL3etUQ3LtKN", "outputId": "fe00589f-64dd-4b70-e612-3873b504c00a" }, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Could not locate the tokenizer configuration file, will try to use the model config instead.\n", - "loading configuration file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/858852fd2471ce39075378592ddc87f5a6551e64c6825d1b92c8dab9318e0fc3.03ff9e9f998b9a9d40647a2148a202e3fb3d568dc0f170dda9dda194bab4d5dd\n", - "Model config BertConfig {\n", - " \"_name_or_path\": \"allenai/scibert_scivocab_uncased\",\n", - " \"attention_probs_dropout_prob\": 0.1,\n", - " \"classifier_dropout\": null,\n", - " \"hidden_act\": \"gelu\",\n", - " \"hidden_dropout_prob\": 0.1,\n", - " \"hidden_size\": 768,\n", - " \"initializer_range\": 0.02,\n", - " \"intermediate_size\": 3072,\n", - " \"layer_norm_eps\": 1e-12,\n", - " \"max_position_embeddings\": 512,\n", - " \"model_type\": \"bert\",\n", - " \"num_attention_heads\": 12,\n", - " \"num_hidden_layers\": 12,\n", - " \"pad_token_id\": 0,\n", - " \"position_embedding_type\": \"absolute\",\n", - " \"transformers_version\": \"4.20.1\",\n", - " \"type_vocab_size\": 2,\n", - " \"use_cache\": true,\n", - " \"vocab_size\": 31090\n", - "}\n", - "\n", - "loading file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/vocab.txt from cache at /root/.cache/huggingface/transformers/33593020f507d72099bd84ea6cd2296feb424fecd62d4a8edcc2a02899af6e29.38339d84e6e392addd730fd85fae32652c4cc7c5423633d6fa73e5f7937bbc38\n", - "loading file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/tokenizer.json from cache at None\n", - "loading file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/added_tokens.json from cache at None\n", - "loading file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/special_tokens_map.json from cache at None\n", - "loading file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/tokenizer_config.json from cache at None\n", - "loading configuration file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/858852fd2471ce39075378592ddc87f5a6551e64c6825d1b92c8dab9318e0fc3.03ff9e9f998b9a9d40647a2148a202e3fb3d568dc0f170dda9dda194bab4d5dd\n", - "Model config BertConfig {\n", - " \"_name_or_path\": \"allenai/scibert_scivocab_uncased\",\n", - " \"attention_probs_dropout_prob\": 0.1,\n", - " \"classifier_dropout\": null,\n", - " \"hidden_act\": \"gelu\",\n", - " \"hidden_dropout_prob\": 0.1,\n", - " \"hidden_size\": 768,\n", - " \"initializer_range\": 0.02,\n", - " \"intermediate_size\": 3072,\n", - " \"layer_norm_eps\": 1e-12,\n", - " \"max_position_embeddings\": 512,\n", - " \"model_type\": \"bert\",\n", - " \"num_attention_heads\": 12,\n", - " \"num_hidden_layers\": 12,\n", - " \"pad_token_id\": 0,\n", - " \"position_embedding_type\": \"absolute\",\n", - " \"transformers_version\": \"4.20.1\",\n", - " \"type_vocab_size\": 2,\n", - " \"use_cache\": true,\n", - " \"vocab_size\": 31090\n", - "}\n", - "\n", - "loading configuration file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/858852fd2471ce39075378592ddc87f5a6551e64c6825d1b92c8dab9318e0fc3.03ff9e9f998b9a9d40647a2148a202e3fb3d568dc0f170dda9dda194bab4d5dd\n", - "Model config BertConfig {\n", - " \"_name_or_path\": \"allenai/scibert_scivocab_uncased\",\n", - " \"attention_probs_dropout_prob\": 0.1,\n", - " \"classifier_dropout\": null,\n", - " \"hidden_act\": \"gelu\",\n", - " \"hidden_dropout_prob\": 0.1,\n", - " \"hidden_size\": 768,\n", - " \"initializer_range\": 0.02,\n", - " \"intermediate_size\": 3072,\n", - " \"layer_norm_eps\": 1e-12,\n", - " \"max_position_embeddings\": 512,\n", - " \"model_type\": \"bert\",\n", - " \"num_attention_heads\": 12,\n", - " \"num_hidden_layers\": 12,\n", - " \"pad_token_id\": 0,\n", - " \"position_embedding_type\": \"absolute\",\n", - " \"transformers_version\": \"4.20.1\",\n", - " \"type_vocab_size\": 2,\n", - " \"use_cache\": true,\n", - " \"vocab_size\": 31090\n", - "}\n", - "\n", - "loading configuration file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/858852fd2471ce39075378592ddc87f5a6551e64c6825d1b92c8dab9318e0fc3.03ff9e9f998b9a9d40647a2148a202e3fb3d568dc0f170dda9dda194bab4d5dd\n", - "Model config BertConfig {\n", - " \"_name_or_path\": \"allenai/scibert_scivocab_uncased\",\n", - " \"attention_probs_dropout_prob\": 0.1,\n", - " \"classifier_dropout\": null,\n", - " \"hidden_act\": \"gelu\",\n", - " \"hidden_dropout_prob\": 0.1,\n", - " \"hidden_size\": 768,\n", - " \"initializer_range\": 0.02,\n", - " \"intermediate_size\": 3072,\n", - " \"layer_norm_eps\": 1e-12,\n", - " \"max_position_embeddings\": 512,\n", - " \"model_type\": \"bert\",\n", - " \"num_attention_heads\": 12,\n", - " \"num_hidden_layers\": 12,\n", - " \"pad_token_id\": 0,\n", - " \"position_embedding_type\": \"absolute\",\n", - " \"transformers_version\": \"4.20.1\",\n", - " \"type_vocab_size\": 2,\n", - " \"use_cache\": true,\n", - " \"vocab_size\": 31090\n", - "}\n", - "\n", - "loading weights file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/pytorch_model.bin from cache at /root/.cache/huggingface/transformers/de14937a851e8180a2bc5660c0041d385f8a0c62b1b2ccafa46df31043a2390c.74830bb01a0ffcdeaed8be9916312726d0c4cd364ac6fc15b375f789eaff4cbb\n", - "Some weights of the model checkpoint at allenai/scibert_scivocab_uncased were not used when initializing BertForSequenceClassification: ['cls.seq_relationship.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.bias', 'cls.seq_relationship.weight', 'cls.predictions.decoder.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight']\n", - "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", - "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", - "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at allenai/scibert_scivocab_uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n", - "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n" - ] - }, { "data": { "application/vnd.jupyter.widget-view+json": { @@ -219,24 +99,6 @@ "metadata": {}, "output_type": "display_data" }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no truncation.\n", - "PyTorch: setting up devices\n", - "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n", - "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", - " FutureWarning,\n", - "***** Running training *****\n", - " Num examples = 400\n", - " Num Epochs = 50\n", - " Instantaneous batch size per device = 32\n", - " Total train batch size (w. parallel, distributed & accumulation) = 32\n", - " Gradient Accumulation steps = 1\n", - " Total optimization steps = 650\n" - ] - }, { "data": { "text/html": [ @@ -330,61 +192,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "***** Running Evaluation *****\n", - " Num examples = 100\n", - " Batch size = 32\n", - "Saving model checkpoint to models/checkpoint-13\n", - "Configuration saved in models/checkpoint-13/config.json\n", - "Model weights saved in models/checkpoint-13/pytorch_model.bin\n", - "Deleting older checkpoint [models/checkpoint-91] due to args.save_total_limit\n", - "***** Running Evaluation *****\n", - " Num examples = 100\n", - " Batch size = 32\n", - "Saving model checkpoint to models/checkpoint-26\n", - "Configuration saved in models/checkpoint-26/config.json\n", - "Model weights saved in models/checkpoint-26/pytorch_model.bin\n", - "Deleting older checkpoint [models/checkpoint-117] due to args.save_total_limit\n", - "***** Running Evaluation *****\n", - " Num examples = 100\n", - " Batch size = 32\n", - "Saving model checkpoint to models/checkpoint-39\n", - "Configuration saved in models/checkpoint-39/config.json\n", - "Model weights saved in models/checkpoint-39/pytorch_model.bin\n", - "Deleting older checkpoint [models/checkpoint-130] due to args.save_total_limit\n", - "***** Running Evaluation *****\n", - " Num examples = 100\n", - " Batch size = 32\n", - "Saving model checkpoint to models/checkpoint-52\n", - "Configuration saved in models/checkpoint-52/config.json\n", - "Model weights saved in models/checkpoint-52/pytorch_model.bin\n", - "Deleting older checkpoint [models/checkpoint-143] due to args.save_total_limit\n", - "***** Running Evaluation *****\n", - " Num examples = 100\n", - " Batch size = 32\n", - "Saving model checkpoint to models/checkpoint-65\n", - "Configuration saved in models/checkpoint-65/config.json\n", - "Model weights saved in models/checkpoint-65/pytorch_model.bin\n", - "Deleting older checkpoint [models/checkpoint-156] due to args.save_total_limit\n", - "***** Running Evaluation *****\n", - " Num examples = 100\n", - " Batch size = 32\n", - "Saving model checkpoint to models/checkpoint-78\n", - "Configuration saved in models/checkpoint-78/config.json\n", - "Model weights saved in models/checkpoint-78/pytorch_model.bin\n", - "Deleting older checkpoint [models/checkpoint-13] due to args.save_total_limit\n", - "***** Running Evaluation *****\n", - " Num examples = 100\n", - " Batch size = 32\n", - "Saving model checkpoint to models/checkpoint-91\n", - "Configuration saved in models/checkpoint-91/config.json\n", - "Model weights saved in models/checkpoint-91/pytorch_model.bin\n", - "Deleting older checkpoint [models/checkpoint-26] due to args.save_total_limit\n", - "***** Running Evaluation *****\n", - " Num examples = 100\n", - " Batch size = 32\n", - "Saving model checkpoint to models/checkpoint-104\n", - "Configuration saved in models/checkpoint-104/config.json\n", - "Model weights saved in models/checkpoint-104/pytorch_model.bin\n", + "...\n", "Deleting older checkpoint [models/checkpoint-39] due to args.save_total_limit\n", "***** Running Evaluation *****\n", " Num examples = 100\n", @@ -418,6 +226,7 @@ " TrainingArguments,\n", " EarlyStoppingCallback,\n", ")\n", + "from pathlib import Path\n", "import numpy as np\n", "from datasets import Dataset, load_metric\n", "\n", @@ -474,13 +283,17 @@ ").train()" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The best macro F1-score on the test set is **0.89** which is (not surprisingly) substantially more than the SVM achieved. We have a great model, it's time to deploy it. But first, we have to store it in a secure place." + ] + }, { "cell_type": "code", "execution_count": 44, "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, "executionInfo": { "elapsed": 25368, "status": "ok", @@ -514,387 +327,46 @@ } ], "source": [ - "from great_ai.large_file import LargeFileS3\n", - "\n", - "LargeFileS3.configure_credentials_from_file(\"config.ini\")\n", + "from great_ai import save_model\n", "\n", + "# save Torch model to local disk\n", "model.save_pretrained(\"pretrained\")\n", - "LargeFileS3(\"scibert-highlights\").push(\"pretrained\")" + "\n", + "# upload model from local disk to S3\n", + "# (because the S3 credentials have been already set, `save_model` will use LargeFileS3)\n", + "save_model(\"pretrained\", key=\"scibert-highlights\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next: [Part 3](/examples/scibert/deploy)" ] } ], "metadata": { - "accelerator": "GPU", - "colab": { - "collapsed_sections": [], - "name": "Copy of Forms snippets", - "provenance": [ - { - "file_id": "/v2/external/notebooks/snippets/forms.ipynb", - "timestamp": 1656585404621 - } - ] - }, - "gpuClass": "standard", "kernelspec": { "display_name": "Python 3.10.4 ('.env': venv)", "language": "python", "name": "python3" }, "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", "version": "3.10.4" }, "vscode": { "interpreter": { "hash": "02dd6d3afbfa9fbbe1037d64ad9014965528a1ccad21929d6e72f466389a68ad" } - }, - "widgets": { - "application/vnd.jupyter.widget-state+json": { - "22119b79eb514ad684cc3f00f519fb4a": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "32adc54185894f0598c2d9ad438c76e2": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "34631de39509438aad98cbd3fc64c999": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "4be5b3c59dc04aa2b92f51654e815589": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_cb7ec6240337466d8833c70083e1c3cb", - "max": 1, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_34631de39509438aad98cbd3fc64c999", - "value": 1 - } - }, - "8e0ede9d1dd84c149a7e282211c7071b": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_32adc54185894f0598c2d9ad438c76e2", - "placeholder": "​", - "style": "IPY_MODEL_981e11fb9d4f4a2ba28c011741a1eaba", - "value": " 1/1 [00:00<00:00, 5.38ba/s]" - } - }, - "9185277b1e5945a9a6a3ad75811bc86b": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_bf773a86ec0a4899bb3636035f7ab35e", - "placeholder": "​", - "style": "IPY_MODEL_22119b79eb514ad684cc3f00f519fb4a", - "value": "100%" - } - }, - "981e11fb9d4f4a2ba28c011741a1eaba": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "9bd5d2fb87bd428796155cc67d06b333": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "9c57de70e68a41ecbde5093bd671715a": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_9185277b1e5945a9a6a3ad75811bc86b", - "IPY_MODEL_4be5b3c59dc04aa2b92f51654e815589", - "IPY_MODEL_8e0ede9d1dd84c149a7e282211c7071b" - ], - "layout": "IPY_MODEL_9bd5d2fb87bd428796155cc67d06b333" - } - }, - "bf773a86ec0a4899bb3636035f7ab35e": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "cb7ec6240337466d8833c70083e1c3cb": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - } - } } }, "nbformat": 4, diff --git a/docs/examples/scibert/views/__init__.py b/docs/examples/scibert/views/__init__.py deleted file mode 100644 index cf9b8b1..0000000 --- a/docs/examples/scibert/views/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .evaluated_sentence import EvaluatedSentence -from .match import Match diff --git a/docs/examples/scibert/views/evaluated_sentence.py b/docs/examples/scibert/views/evaluated_sentence.py deleted file mode 100644 index e31ed30..0000000 --- a/docs/examples/scibert/views/evaluated_sentence.py +++ /dev/null @@ -1,11 +0,0 @@ -from typing import List - -from pydantic import BaseModel - -from .match import Match - - -class EvaluatedSentence(BaseModel): - score: float - text: str - explanation: List[Match] diff --git a/docs/examples/scibert/views/match.py b/docs/examples/scibert/views/match.py deleted file mode 100644 index 8485809..0000000 --- a/docs/examples/scibert/views/match.py +++ /dev/null @@ -1,6 +0,0 @@ -from pydantic import BaseModel - - -class Match(BaseModel): - phrase: str - score: float diff --git a/docs/media/annotator.png b/docs/media/annotator.png new file mode 100644 index 0000000..9bad80c Binary files /dev/null and b/docs/media/annotator.png differ diff --git a/mkdocs.yaml b/mkdocs.yaml index 1f25e09..1014fc7 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -63,7 +63,6 @@ plugins: - search - mkdocs-jupyter: include_source: true - ignore: ["docs/examples/scibert/*.ipynb"] markdown_extensions: - pymdownx.highlight: @@ -111,5 +110,10 @@ nav: - examples/simple/data.ipynb - examples/simple/train.ipynb - examples/simple/deploy.ipynb - - Explainable SciBERT: examples/scibert.md + - Explainable SciBERT: + - examples/scibert/index.md + - examples/scibert/data.ipynb + - examples/scibert/train.ipynb + - examples/scibert/deploy.ipynb + - examples/scibert/additional-files.md - explanation.md