Add documentation for SciBERT example

.vscode/settings.json

@@ -36,11 +36,13 @@
     "organisation's",
     "Parcoords",
     "plotly",
+    "pretrained",
     "proba",
     "pydantic",
     "pymongo",
     "pyplot",
     "redoc",
+    "scibert",
     "serialise",
     "sklearn",
     "starlette",

docs/examples/scibert.md

@@ -1,7 +0,0 @@
-# Summarising scientific publications from a tech-transfer perspective
-
-This example shows how `great-ai` is used in practice at ScoutinScience. 
-
-<div style="display: flex; justify-content: center;" markdown>
-[:material-test-tube: Check out the code](https://github.com/schmelczer/great-ai/tree/main/docs/examples/scibert){ .md-button .md-button--primary }
-</div>

docs/examples/scibert/Dockerfile

@@ -1,10 +0,0 @@
-FROM schmelczera/great-ai:v0.1.4
-
-COPY requirements.txt ./
-RUN pip install --no-cache-dir --requirement requirements.txt
-
-COPY . ./
-
-RUN large-file --backend s3 --secrets s3.ini --cache scibert-highlights
-
-CMD ["deploy.ipynb"]

docs/examples/scibert/additional-files.md

@@ -0,0 +1,75 @@
+# Additional files in the repository
+
+In order to give you a smooth experience while comprehending this example, all non-notebook files are presented on this page in one place. In reality, these files should be in your project's top-level directory.
+
+## config.ini
+
+```ini title="config.ini"
+ENVIRONMENT = DEVELOPMENT
+ENVIRONMENT = ENV:ENVIRONMENT
+
+MONGO_CONNECTION_STRING=ENV:MONGO_CONNECTION_STRING
+MONGO_DATABASE=highlights
+
+AWS_REGION_NAME = eu-west-2
+AWS_ACCESS_KEY_ID = MY_DEFAULT_AWS_ACCESS_KEY_ID_FOR_DEVELOPMENT
+AWS_SECRET_ACCESS_KEY = MY_DEFAULT_AWS_SECRET_ACCESS_KEY_FOR_DEVELOPMENT
+LARGE_FILES_BUCKET_NAME = my-orgs-large-files
+
+AWS_REGION_NAME = ENV:AWS_REGION_NAME
+AWS_ACCESS_KEY_ID = ENV:AWS_ACCESS_KEY_ID
+AWS_SECRET_ACCESS_KEY = ENV:AWS_SECRET_ACCESS_KEY
+LARGE_FILES_BUCKET_NAME = ENV:LARGE_FILES_BUCKET_NAME
+```
+
+> All necessary configuration which is read by [great_ai.utilities.ConfigFile][]. This will resolve values starting with `ENV:` from your environment variables.
+
+## requirements.txt
+
+```requirements.txt title="requirements.txt"
+torch==1.12.0
+transformers==4.20.1
+numpy==1.23.0
+```
+
+> Usually, it is recommended to pin (freeze) the library versions on which we depend. This file is referenced by the [Dockerfile](#dockerfile).
+
+## Dockerfile
+
+```Dockerfile title="Dockerfile"
+FROM schmelczera/great-ai:v0.1.6
+
+COPY requirements.txt ./
+RUN pip install --no-cache-dir --requirement requirements.txt
+
+COPY . ./
+
+RUN large-file --backend s3 --secrets s3.ini --cache scibert-highlights
+
+CMD ["deploy.ipynb"]
+```
+
+> This is used by the CD pipeline to create the production deployment of the service.
+
+## .dockerignore
+
+```dockerignore title=".dockerignore"
+.cache
+.git
+data
+.gitignore
+.env
+.vscode
+.dockerignore
+Dockerfile
+.mypy_cache
+.gitignore
+**/__pycache__
+**/.DS_Store
+README.md
+```
+
+> It is useful not to send, for example, the `.cache` folder used by LargeFile to the docker daemon; this will speed up your local build times substantially.
+
+!!! Note ".gitignore"
+    A very similar looking `.gitignore` file should also be present.

docs/examples/scibert/config.ini

@@ -1,10 +0,0 @@
-ENVIRONMENT = DEVELOPMENT
-ENVIRONMENT = ENV:ENVIRONMENT
-
-MONGO_CONNECTION_STRING=ENV:MONGO_CONNECTION_STRING
-MONGO_DATABASE=highlights
-
-AWS_REGION_NAME = ENV:AWS_REGION_NAME
-AWS_ACCESS_KEY_ID = ENV:AWS_ACCESS_KEY_ID
-AWS_SECRET_ACCESS_KEY = ENV:AWS_SECRET_ACCESS_KEY
-LARGE_FILES_BUCKET_NAME = ENV:LARGE_FILES_BUCKET_NAME

docs/examples/scibert/deploy.ipynb

@@ -1,5 +1,94 @@
 {
  "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Create an inference function\n",
+    "\n",
+    "Everything is ready to wrap the previously trained model and deploy it. \n",
+    "\n",
+    "First, we need to configure the LargeFileBackend, the TracingDatabase and GreatAI."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\u001b[38;5;226mThe value of `ENVIRONMENT` contains the \"ENV` prefix but `ENVIRONMENT` is not defined as an environment variable, using the default value defined above (`DEVELOPMENT`)\u001b[0m\n",
+      "\u001b[38;5;226mEnvironment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
+      "\u001b[38;5;39mMongoDbDriver has been already configured: skipping initialisation\u001b[0m\n",
+      "\u001b[38;5;39mLargeFileS3 has been already configured: skipping initialisation\u001b[0m\n",
+      "\u001b[38;5;39mGreatAI (v0.1.6): configured ✅\u001b[0m\n",
+      "\u001b[38;5;39m  🔩 tracing_database: MongoDbDriver\u001b[0m\n",
+      "\u001b[38;5;39m  🔩 large_file_implementation: LargeFileS3\u001b[0m\n",
+      "\u001b[38;5;39m  🔩 is_production: False\u001b[0m\n",
+      "\u001b[38;5;39m  🔩 should_log_exception_stack: True\u001b[0m\n",
+      "\u001b[38;5;39m  🔩 prediction_cache_size: 4096\u001b[0m\n",
+      "\u001b[38;5;39m  🔩 dashboard_table_size: 100\u001b[0m\n",
+      "\u001b[38;5;226mYou still need to check whether you follow all best practices before trusting your deployment.\u001b[0m\n",
+      "\u001b[38;5;226m> Find out more at https://se-ml.github.io/practices\u001b[0m\n"
+     ]
+    }
+   ],
+   "source": [
+    "from great_ai.utilities import ConfigFile\n",
+    "from great_ai.large_file import LargeFileS3\n",
+    "from great_ai import configure, MongoDbDriver\n",
+    "\n",
+    "configuration = ConfigFile(\"config.ini\")\n",
+    "\n",
+    "LargeFileS3.configure_credentials_from_file(configuration)\n",
+    "MongoDbDriver.configure_credentials_from_file(configuration)\n",
+    "\n",
+    "configure(\n",
+    "    dashboard_table_size=100,  # traces are small, we can show many\n",
+    "    prediction_cache_size=4096,  # predictions are expensive, cache them\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "For a pleasant developer experience, we create some typed models that will show up in the automatically generated OpenAPI schema specification and will also provide runtime type validation."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from typing import List\n",
+    "from pydantic import BaseModel\n",
+    "\n",
+    "\n",
+    "class Attention(BaseModel):\n",
+    "    weight: float\n",
+    "    token: str\n",
+    "\n",
+    "\n",
+    "class EvaluatedSentence(BaseModel):\n",
+    "    score: float\n",
+    "    text: str\n",
+    "    explanation: List[Attention]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Even though `@use_model` caches the remote files locally and it also handles deserialising objects, we only use it to store a directory. In this case, it gives back a path, the path to that directory. So, we need to load the files from that folder ourselves. In order to only load it once per process, we create a small model loader helper function.\n",
+    "\n",
+    "> This is usually not needed, however, when we can outsmart `dill` so for optimisation purposes, we do it."
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 3,
@@ -9,86 +98,96 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "\u001b[38;5;226m2022-07-01 14:28:43,377 |  WARNING | Environment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
+      "\u001b[38;5;39mLatest version of scibert-highlights is 0 (from versions: 0)\u001b[0m\n",
-      "\u001b[38;5;39m2022-07-01 14:28:43,378 |     INFO | Found credentials file (/data/projects/scoutinscience/platform/projects/highlights-service2/mongo.ini), initialising MongoDbDriver\u001b[0m\n",
+      "\u001b[38;5;39mFile scibert-highlights-0 found in cache\u001b[0m\n"
-      "\u001b[38;5;39m2022-07-01 14:28:43,379 |     INFO | Found credentials file (/data/projects/scoutinscience/platform/projects/highlights-service2/s3.ini), initialising LargeFileS3\u001b[0m\n",
-      "\u001b[38;5;39m2022-07-01 14:28:43,380 |     INFO | Settings: configured ✅\u001b[0m\n",
-      "\u001b[38;5;39m2022-07-01 14:28:43,380 |     INFO | 🔩 tracing_database: MongoDbDriver\u001b[0m\n",
-      "\u001b[38;5;39m2022-07-01 14:28:43,381 |     INFO | 🔩 large_file_implementation: LargeFileS3\u001b[0m\n",
-      "\u001b[38;5;39m2022-07-01 14:28:43,381 |     INFO | 🔩 is_production: False\u001b[0m\n",
-      "\u001b[38;5;39m2022-07-01 14:28:43,382 |     INFO | 🔩 should_log_exception_stack: True\u001b[0m\n",
-      "\u001b[38;5;39m2022-07-01 14:28:43,382 |     INFO | 🔩 prediction_cache_size: 512\u001b[0m\n",
-      "\u001b[38;5;39m2022-07-01 14:28:43,383 |     INFO | 🔩 dashboard_table_size: 50\u001b[0m\n",
-      "\u001b[38;5;226m2022-07-01 14:28:43,384 |  WARNING | You still need to check whether you follow all best practices before trusting your deployment.\u001b[0m\n",
-      "\u001b[38;5;226m2022-07-01 14:28:43,385 |  WARNING | > Find out more at https://se-ml.github.io/practices/\u001b[0m\n",
-      "\u001b[38;5;39m2022-07-01 14:28:44,082 |     INFO | Latest version of scibert-highlights is 0 (from versions: 0)\u001b[0m\n",
-      "\u001b[38;5;39m2022-07-01 14:28:44,083 |     INFO | File scibert-highlights-0 found in cache\u001b[0m\n",
-      "\u001b[38;5;39m2022-07-01 14:28:44,084 |     INFO | File scibert-highlights-0 found in cache\u001b[0m\n"
      ]
     }
    ],
    "source": [
-    "from great_ai import GreatAI, use_model, MongoDbDriver, configure\n",
+    "from great_ai import use_model\n",
-    "from great_ai.utilities import clean\n",
     "from pathlib import Path\n",
+    "from typing import Tuple\n",
+    "from transformers import (\n",
+    "    PreTrainedModel,\n",
+    "    PreTrainedTokenizer,\n",
+    ")\n",
     "from transformers import (\n",
     "    AutoConfig,\n",
     "    AutoModelForSequenceClassification,\n",
     "    AutoTokenizer,\n",
     ")\n",
+    "\n",
+    "_tokenizer: PreTrainedTokenizer = None\n",
+    "_loaded_model: PreTrainedModel = None\n",
+    "\n",
+    "\n",
+    "@use_model(\"scibert-highlights\", version=\"latest\", model_kwarg_name=\"model_path\")\n",
+    "def get_tokenizer_and_model(\n",
+    "    model_path: Path, original_model: str = \"allenai/scibert_scivocab_uncased\"\n",
+    ") -> Tuple[PreTrainedTokenizer, PreTrainedModel]:\n",
+    "    global _tokenizer, _loaded_model\n",
+    "\n",
+    "    if _tokenizer is None:\n",
+    "        _tokenizer = AutoTokenizer.from_pretrained(original_model)\n",
+    "\n",
+    "    if _loaded_model is None:\n",
+    "        config = AutoConfig.from_pretrained(\n",
+    "            model_path, output_hidden_states=True, output_attentions=True\n",
+    "        )\n",
+    "        _loaded_model = AutoModelForSequenceClassification.from_pretrained(\n",
+    "            model_path, config=config\n",
+    "        )\n",
+    "\n",
+    "    return _tokenizer, _loaded_model"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Finally, implement the inference function."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from great_ai import GreatAI\n",
+    "from great_ai.utilities import clean\n",
+    "\n",
     "import re\n",
     "import numpy as np\n",
     "import torch\n",
     "from transformers.modeling_outputs import SequenceClassifierOutput\n",
-    "from transformers import (\n",
-    "    PreTrainedModel,\n",
-    "    PreTrainedTokenizer,\n",
-    ")\n",
-    "from views import EvaluatedSentence, Match\n",
-    "from great_ai.large_file import LargeFileS3\n",
-    "\n",
-    "LargeFileS3.configure_credentials_from_file(\"config.ini\")\n",
-    "MongoDbDriver.configure_credentials_from_file(\"config.ini\")\n",
-    "configure(dashboard_table_size=100)\n",
-    "\n",
-    "ORIGINAL_MODEL = \"allenai/scibert_scivocab_uncased\"\n",
-    "\n",
-    "loaded_model: PreTrainedModel = None\n",
-    "tokenizer: PreTrainedTokenizer = None\n",
     "\n",
     "\n",
     "@GreatAI.create\n",
-    "@use_model(\"scibert-highlights\", version=\"latest\")\n",
+    "def find_highlights(sentence: str) -> EvaluatedSentence:\n",
-    "def find_highlights(sentence: str, model: Path) -> EvaluatedSentence:\n",
+    "    \"\"\"Get the interestingness prediction of the input sentence using SciBERT.\n",
-    "    global loaded_model, tokenizer\n",
     "\n",
-    "    if loaded_model is None:\n",
+    "    Run the SciBERT model in inference mode and evaluate the sentence.\n",
-    "        config = AutoConfig.from_pretrained(\n",
+    "    Additionally, provide explanation in the form of the last layer's sum attention\n",
-    "            model, output_hidden_states=True, output_attentions=True\n",
+    "    between `[CLS]` and the other tokens.\n",
-    "        )\n",
+    "    \"\"\"\n",
-    "        loaded_model = AutoModelForSequenceClassification.from_pretrained(\n",
-    "            model, config=config\n",
-    "        )\n",
-    "    if tokenizer is None:\n",
-    "        tokenizer = AutoTokenizer.from_pretrained(ORIGINAL_MODEL)\n",
     "\n",
+    "    tokenizer, loaded_model = get_tokenizer_and_model()\n",
     "    sentence = clean(sentence, convert_to_ascii=True, remove_brackets=True)\n",
     "\n",
-    "    return evaluate_sentence(sentence=sentence)\n",
-    "\n",
-    "\n",
-    "def evaluate_sentence(sentence: str) -> EvaluatedSentence:\n",
     "    tensors = tokenizer(sentence, return_tensors=\"pt\", truncation=True, max_length=512)\n",
     "\n",
-    "    with torch.no_grad():\n",
+    "    with torch.inference_mode():\n",
     "        result: SequenceClassifierOutput = loaded_model(**tensors)\n",
     "        positive_likelihood = torch.nn.Softmax(dim=1)(result.logits)[0][1]\n",
     "    tokens = tensors[\"input_ids\"][0]\n",
     "\n",
     "    attentions = np.sum(result.attentions[-1].numpy()[0], axis=0)[0][\n",
     "        1:-1\n",
-    "    ]  # Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.\n",
+    "    ]  # Tuple of `torch.FloatTensor` (one for each layer) of shape\n",
-    "    matches = []\n",
+    "    # `(batch_size, num_heads, sequence_length, sequence_length)`.\n",
+    "\n",
+    "    explanation = []\n",
     "\n",
     "    token_attentions = list(zip(attentions, tokens[1:-1]))\n",
     "    for token in re.split(r\"([ .,])\", sentence):\n",
@@ -99,24 +198,101 @@
     "            token, return_tensors=\"pt\", truncation=True, max_length=512\n",
     "        )[\"input_ids\"][0][\n",
     "            1:-1\n",
-    "        ]  # truncation=True needed to fix RuntimeError: Already borrowed\n",
+    "        ]  # truncation=True needed to fix `RuntimeError: Already borrowed`\n",
-    "        score = 0\n",
+    "        weight = 0\n",
     "        for t1 in bert_tokens:\n",
     "            if not token_attentions:\n",
     "                break\n",
     "            a, t2 = token_attentions.pop(0)\n",
     "            assert t1 == t2, sentence\n",
-    "            score += a\n",
+    "            weight += a\n",
-    "        matches.append(\n",
+    "        explanation.append(\n",
-    "            Match(phrase=token if token in \".,\" else \" \" + token, score=round(score, 4))\n",
+    "            Attention(\n",
+    "                token=token if token in \".,\" else \" \" + token, weight=round(weight, 4)\n",
+    "            )\n",
     "        )\n",
     "        if not token_attentions:\n",
     "            break\n",
     "\n",
     "    return EvaluatedSentence(\n",
-    "        score=positive_likelihood, text=sentence, explanation=matches\n",
+    "        score=positive_likelihood, text=sentence, explanation=explanation\n",
     "    )"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "A simple test to see everything works. Note that the models list is filled by the `@use_model` call even though it's not on the main inference function."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(Trace[EvaluatedSentence]({'created': '2022-07-16T18:47:29.581701',\n",
+       "   'exception': None,\n",
+       "   'feedback': None,\n",
+       "   'logged_values': { 'arg:sentence:length': 51,\n",
+       "                      'arg:sentence:value': 'Our solution has outperformed the '\n",
+       "                                            'state-of-the-art.'},\n",
+       "   'models': [{'key': 'scibert-highlights', 'version': 0}],\n",
+       "   'original_execution_time_ms': 7127.2063,\n",
+       "   'output': { 'explanation': [ {'token': ' Our', 'weight': 0.3993},\n",
+       "                                {'token': ' solution', 'weight': 0.3481},\n",
+       "                                {'token': ' has', 'weight': 0.2945},\n",
+       "                                {'token': ' outperformed', 'weight': 0.4011},\n",
+       "                                {'token': ' the', 'weight': 0.1484},\n",
+       "                                {'token': ' state-of-the-art', 'weight': 0.5727},\n",
+       "                                {'token': '.', 'weight': 7.775}],\n",
+       "               'score': 0.9991180300712585,\n",
+       "               'text': 'Our solution has outperformed the state-of-the-art.'},\n",
+       "   'tags': ['find_highlights', 'online', 'development'],\n",
+       "   'trace_id': '56e20e94-79df-4793-ae61-d20820ebe2d3'}),\n",
+       " Trace[EvaluatedSentence]({'created': '2022-07-16T18:47:37.020275',\n",
+       "   'exception': None,\n",
+       "   'feedback': None,\n",
+       "   'logged_values': { 'arg:sentence:length': 36,\n",
+       "                      'arg:sentence:value': 'Their solution did not perform '\n",
+       "                                            'well.'},\n",
+       "   'models': [{'key': 'scibert-highlights', 'version': 0}],\n",
+       "   'original_execution_time_ms': 170.7057,\n",
+       "   'output': { 'explanation': [ {'token': ' Their', 'weight': 1.1475},\n",
+       "                                {'token': ' solution', 'weight': 0.8205},\n",
+       "                                {'token': ' did', 'weight': 0.3254},\n",
+       "                                {'token': ' not', 'weight': 0.2921},\n",
+       "                                {'token': ' perform', 'weight': 0.4293},\n",
+       "                                {'token': ' well', 'weight': 0.2772},\n",
+       "                                {'token': '.', 'weight': 4.4723}],\n",
+       "               'score': 0.12305451184511185,\n",
+       "               'text': 'Their solution did not perform well.'},\n",
+       "   'tags': ['find_highlights', 'online', 'development'],\n",
+       "   'trace_id': '7fcf8271-1738-4025-8305-d5a1e5100aea'}))"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "if __name__ == \"__main__\":\n",
+    "    find_highlights(\n",
+    "        \"Our solution has outperformed the state-of-the-art.\"\n",
+    "    ), find_highlights(\"Their solution did not perform well.\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "In this case, the service is built as a docker image, pushed to our image registry and subsequent rolling update is performed in the production cluster.\n",
+    "To check out the Dockerimage, go to [the additional files page](/examples/scibert/additional-files)."
+   ]
   }
  ],
  "metadata": {

docs/examples/scibert/index.md

@@ -0,0 +1,50 @@
+# Summarising scientific publications from a tech-transfer perspective
+
+This is a simplified example illustrating how `great-ai` is used in practice at [ScoutinScience](https://www.scoutinscience.com/){ target=_blank }. The subpages show `great-ai` in action by going over the life-cycle of finetuning and deploying a BERT-based software service.
+
+??? note "Propriety data"
+    The purpose of this example is to show you different ways in which `great-ai` can assist you. The exact NLP task being solved is not central. Stemming from this, and from the difficult nature of obtaining appropriate training data, the propriety dataset used for the experiments is not shared.
+
+## Objectives
+
+1. You will see how the [great_ai.utilities](/reference/utilities) can integrate into your Data Science workflow.
+2. You will see how [great_ai.large_file](/reference/large-file) can be used to version and store your trained model.
+3. You will see how [GreatAI][great_ai.GreatAI] should be used prepare your model for a robust and responsible deployment.
+4. You will see multiple ways of customising your deployment.
+
+## Overview
+
+One of the core features of the ScoutinScience platform is summarising research papers form a tech-transfer perspective. In short, extractive summarisation is preferred using a binary classifier trained on clients' judgement of sentence interestingness. Thus, documents are sentences and the expected output is a binary label showing whether a sentence is "worthy" of being in the tech-transfer summary. Providing an explanation for each decision is imperative since ScoutinScience embraces applying only explainable AI (XAI) methods wherever feasible.
+
+!!! success
+    You are ready to start the tutorial. Feel free to come back to the [summary](#summary) section once you're finished.
+
+<div style="display: flex; justify-content: space-evenly;" markdown>
+[:material-database: Examine data](data.ipynb){ .md-button .md-button--primary }
+
+[:fontawesome-solid-chart-simple: Train model](train.ipynb){ .md-button .md-button--primary }
+
+[:material-cloud-tags: Deploy service](deploy.ipynb){ .md-button .md-button--primary }
+</div>
+
+## Summary
+
+### [Data notebook](data.ipynb)
+
+We load and analyse the data by calculating inter-rater reliability and checking the feasibility of using an AI-based approach by testing the accuracy of a trivial baseline method.
+
+### [Training notebook](train.ipynb)
+
+We simply fine-tune SciBERT.
+
+After training and evaluating a model, it is exported using [great_ai.save_model][]. For more info, checkout [the configuration how-to page](/how-to-guides/configure-service).
+
+### [Deployment notebook](deploy.ipynb)
+
+We customise the GreatAI configuration, create custom cahcing for the model and implement an inference function that can be hardened by wrapping it in a [GreatAI][great_ai.GreatAI] instance. We also extract the attention weights as a quasi-explanation.
+
+Finally, we test the model's inference function through the GreatAI dashboard. [The only thing left is to deploy the hardened service properly.](/how-to-guides/use-service)
+
+#### [Additional files](additional-files.md)
+
+There are some other files required for deploying the notebook. For example, the config file for S3 and MongoDB or a Dockerfile for building a custom image. These are compiled and shown on a separate page.

docs/examples/scibert/requirements.txt

@@ -1,3 +0,0 @@
-torch
-transformers
-numpy

docs/examples/scibert/train.ipynb

@@ -1,210 +1,90 @@
 {
  "cells": [
   {
-   "cell_type": "code",
+   "cell_type": "markdown",
-   "execution_count": 48,
+   "metadata": {},
-   "metadata": {
+   "source": [
-    "colab": {
+    "# Fine-tune SciBERT\n",
-     "base_uri": "https://localhost:8080/"
+    "\n",
+    "We are planning to do a simple classification task on scientific text. For that, [SciBERT](https://github.com/allenai/scibert) is an ideal model to fine-tune since it has been pretrained of academic publications.\n",
+    "\n",
+    "This notebook was updated so that it can run in [Google Colab](https://colab.research.google.com/).\n",
+    "\n",
+    "First, we need to install the dependencies."
+   ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
     "executionInfo": {
      "elapsed": 2529,
      "status": "ok",
      "timestamp": 1656596749103,
-     "user": {
-      "displayName": "Schmelczer András",
-      "userId": "08401926777942666437"
-     },
      "user_tz": -120
     },
     "id": "j7l0nD9hDQbB",
     "outputId": "88a9931b-396a-4cf1-c659-8a7b098b3cdd"
    },
+   "outputs": [],
+   "source": [
+    "!pip install transformers datasets great-ai > /dev/null"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Load the training data from S3. (We have uploaded this to S3 in the `data` notebook.)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
    "outputs": [
     {
-     "name": "stdout",
+     "name": "stderr",
      "output_type": "stream",
      "text": [
-      "evaluation-experiment-2-stage #1-2m6dmb.json\n",
+      "\u001b[38;5;39mLatest version of summary-train-dataset-small is 0 (from versions: 0)\u001b[0m\n",
-      "evaluation-experiment-2-stage #1-sa6a0y.json\n",
+      "\u001b[38;5;39mFile summary-train-dataset-small-0 found in cache\u001b[0m\n"
-      "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
-      "Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.20.1)\n",
-      "Requirement already satisfied: datasets in /usr/local/lib/python3.7/dist-packages (2.3.2)\n",
-      "\u001b[31mERROR: Could not find a version that satisfies the requirement great-ai==0.0.6 (from versions: none)\u001b[0m\n",
-      "\u001b[31mERROR: No matching distribution found for great-ai==0.0.6\u001b[0m\n"
      ]
     }
    ],
    "source": [
-    "from pathlib import Path\n",
+    "from great_ai.large_file import LargeFileS3\n",
     "import json\n",
     "\n",
-    "annotations = []\n",
+    "LargeFileS3.configure_credentials_from_file(\"config.ini\")\n",
-    "for p in Path(\".\").glob(\"*.json\"):\n",
-    "    with open(p, encoding=\"utf-8\") as f:\n",
-    "        print(p)\n",
-    "        annotations.append(json.load(f))\n",
     "\n",
-    "evaluations = {\n",
+    "with LargeFileS3(\"summary-train-dataset-small\", encoding=\"utf-8\") as f:\n",
-    "    sentence: [\n",
+    "    # splitting training and test data is done later by `datasets`\n",
-    "        annotation[sentence] for annotation in annotations if sentence in annotation\n",
+    "    X, y = json.load(f)"
-    "    ]\n",
+   ]
-    "    for sentence in {\n",
+  },
-    "        sentence for annotation in annotations for sentence in annotation.keys()\n",
+  {
-    "    }\n",
+   "cell_type": "markdown",
-    "}\n",
+   "metadata": {},
-    "\n",
+   "source": [
-    "X = [s for s in evaluations.keys()]\n",
+    "Finetune SciBERT, for more info about this step, check out [HuggingFace](https://huggingface.co/docs/transformers/training).\n",
-    "y = [int(sum(e) > 0) for e in evaluations.values()]\n",
+    "If you're only here for `great-ai`, feel free to skip the next cell."
-    "\n",
-    "# !pip install transformers datasets great-ai==0.0.6"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 22,
    "metadata": {
-    "colab": {
-     "base_uri": "https://localhost:8080/",
-     "height": 1000,
-     "referenced_widgets": [
-      "9c57de70e68a41ecbde5093bd671715a",
-      "9185277b1e5945a9a6a3ad75811bc86b",
-      "4be5b3c59dc04aa2b92f51654e815589",
-      "8e0ede9d1dd84c149a7e282211c7071b",
-      "9bd5d2fb87bd428796155cc67d06b333",
-      "bf773a86ec0a4899bb3636035f7ab35e",
-      "22119b79eb514ad684cc3f00f519fb4a",
-      "cb7ec6240337466d8833c70083e1c3cb",
-      "34631de39509438aad98cbd3fc64c999",
-      "32adc54185894f0598c2d9ad438c76e2",
-      "981e11fb9d4f4a2ba28c011741a1eaba"
-     ]
-    },
     "executionInfo": {
      "elapsed": 118131,
      "status": "ok",
      "timestamp": 1656593941974,
-     "user": {
-      "displayName": "Schmelczer András",
-      "userId": "08401926777942666437"
-     },
      "user_tz": -120
     },
     "id": "AL3etUQ3LtKN",
     "outputId": "fe00589f-64dd-4b70-e612-3873b504c00a"
    },
    "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Could not locate the tokenizer configuration file, will try to use the model config instead.\n",
-      "loading configuration file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/858852fd2471ce39075378592ddc87f5a6551e64c6825d1b92c8dab9318e0fc3.03ff9e9f998b9a9d40647a2148a202e3fb3d568dc0f170dda9dda194bab4d5dd\n",
-      "Model config BertConfig {\n",
-      "  \"_name_or_path\": \"allenai/scibert_scivocab_uncased\",\n",
-      "  \"attention_probs_dropout_prob\": 0.1,\n",
-      "  \"classifier_dropout\": null,\n",
-      "  \"hidden_act\": \"gelu\",\n",
-      "  \"hidden_dropout_prob\": 0.1,\n",
-      "  \"hidden_size\": 768,\n",
-      "  \"initializer_range\": 0.02,\n",
-      "  \"intermediate_size\": 3072,\n",
-      "  \"layer_norm_eps\": 1e-12,\n",
-      "  \"max_position_embeddings\": 512,\n",
-      "  \"model_type\": \"bert\",\n",
-      "  \"num_attention_heads\": 12,\n",
-      "  \"num_hidden_layers\": 12,\n",
-      "  \"pad_token_id\": 0,\n",
-      "  \"position_embedding_type\": \"absolute\",\n",
-      "  \"transformers_version\": \"4.20.1\",\n",
-      "  \"type_vocab_size\": 2,\n",
-      "  \"use_cache\": true,\n",
-      "  \"vocab_size\": 31090\n",
-      "}\n",
-      "\n",
-      "loading file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/vocab.txt from cache at /root/.cache/huggingface/transformers/33593020f507d72099bd84ea6cd2296feb424fecd62d4a8edcc2a02899af6e29.38339d84e6e392addd730fd85fae32652c4cc7c5423633d6fa73e5f7937bbc38\n",
-      "loading file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/tokenizer.json from cache at None\n",
-      "loading file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/added_tokens.json from cache at None\n",
-      "loading file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/special_tokens_map.json from cache at None\n",
-      "loading file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/tokenizer_config.json from cache at None\n",
-      "loading configuration file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/858852fd2471ce39075378592ddc87f5a6551e64c6825d1b92c8dab9318e0fc3.03ff9e9f998b9a9d40647a2148a202e3fb3d568dc0f170dda9dda194bab4d5dd\n",
-      "Model config BertConfig {\n",
-      "  \"_name_or_path\": \"allenai/scibert_scivocab_uncased\",\n",
-      "  \"attention_probs_dropout_prob\": 0.1,\n",
-      "  \"classifier_dropout\": null,\n",
-      "  \"hidden_act\": \"gelu\",\n",
-      "  \"hidden_dropout_prob\": 0.1,\n",
-      "  \"hidden_size\": 768,\n",
-      "  \"initializer_range\": 0.02,\n",
-      "  \"intermediate_size\": 3072,\n",
-      "  \"layer_norm_eps\": 1e-12,\n",
-      "  \"max_position_embeddings\": 512,\n",
-      "  \"model_type\": \"bert\",\n",
-      "  \"num_attention_heads\": 12,\n",
-      "  \"num_hidden_layers\": 12,\n",
-      "  \"pad_token_id\": 0,\n",
-      "  \"position_embedding_type\": \"absolute\",\n",
-      "  \"transformers_version\": \"4.20.1\",\n",
-      "  \"type_vocab_size\": 2,\n",
-      "  \"use_cache\": true,\n",
-      "  \"vocab_size\": 31090\n",
-      "}\n",
-      "\n",
-      "loading configuration file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/858852fd2471ce39075378592ddc87f5a6551e64c6825d1b92c8dab9318e0fc3.03ff9e9f998b9a9d40647a2148a202e3fb3d568dc0f170dda9dda194bab4d5dd\n",
-      "Model config BertConfig {\n",
-      "  \"_name_or_path\": \"allenai/scibert_scivocab_uncased\",\n",
-      "  \"attention_probs_dropout_prob\": 0.1,\n",
-      "  \"classifier_dropout\": null,\n",
-      "  \"hidden_act\": \"gelu\",\n",
-      "  \"hidden_dropout_prob\": 0.1,\n",
-      "  \"hidden_size\": 768,\n",
-      "  \"initializer_range\": 0.02,\n",
-      "  \"intermediate_size\": 3072,\n",
-      "  \"layer_norm_eps\": 1e-12,\n",
-      "  \"max_position_embeddings\": 512,\n",
-      "  \"model_type\": \"bert\",\n",
-      "  \"num_attention_heads\": 12,\n",
-      "  \"num_hidden_layers\": 12,\n",
-      "  \"pad_token_id\": 0,\n",
-      "  \"position_embedding_type\": \"absolute\",\n",
-      "  \"transformers_version\": \"4.20.1\",\n",
-      "  \"type_vocab_size\": 2,\n",
-      "  \"use_cache\": true,\n",
-      "  \"vocab_size\": 31090\n",
-      "}\n",
-      "\n",
-      "loading configuration file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/858852fd2471ce39075378592ddc87f5a6551e64c6825d1b92c8dab9318e0fc3.03ff9e9f998b9a9d40647a2148a202e3fb3d568dc0f170dda9dda194bab4d5dd\n",
-      "Model config BertConfig {\n",
-      "  \"_name_or_path\": \"allenai/scibert_scivocab_uncased\",\n",
-      "  \"attention_probs_dropout_prob\": 0.1,\n",
-      "  \"classifier_dropout\": null,\n",
-      "  \"hidden_act\": \"gelu\",\n",
-      "  \"hidden_dropout_prob\": 0.1,\n",
-      "  \"hidden_size\": 768,\n",
-      "  \"initializer_range\": 0.02,\n",
-      "  \"intermediate_size\": 3072,\n",
-      "  \"layer_norm_eps\": 1e-12,\n",
-      "  \"max_position_embeddings\": 512,\n",
-      "  \"model_type\": \"bert\",\n",
-      "  \"num_attention_heads\": 12,\n",
-      "  \"num_hidden_layers\": 12,\n",
-      "  \"pad_token_id\": 0,\n",
-      "  \"position_embedding_type\": \"absolute\",\n",
-      "  \"transformers_version\": \"4.20.1\",\n",
-      "  \"type_vocab_size\": 2,\n",
-      "  \"use_cache\": true,\n",
-      "  \"vocab_size\": 31090\n",
-      "}\n",
-      "\n",
-      "loading weights file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/pytorch_model.bin from cache at /root/.cache/huggingface/transformers/de14937a851e8180a2bc5660c0041d385f8a0c62b1b2ccafa46df31043a2390c.74830bb01a0ffcdeaed8be9916312726d0c4cd364ac6fc15b375f789eaff4cbb\n",
-      "Some weights of the model checkpoint at allenai/scibert_scivocab_uncased were not used when initializing BertForSequenceClassification: ['cls.seq_relationship.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.bias', 'cls.seq_relationship.weight', 'cls.predictions.decoder.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight']\n",
-      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
-      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
-      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at allenai/scibert_scivocab_uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
-      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
-     ]
-    },
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
@@ -219,24 +99,6 @@
      "metadata": {},
      "output_type": "display_data"
     },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no truncation.\n",
-      "PyTorch: setting up devices\n",
-      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
-      "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
-      "  FutureWarning,\n",
-      "***** Running training *****\n",
-      "  Num examples = 400\n",
-      "  Num Epochs = 50\n",
-      "  Instantaneous batch size per device = 32\n",
-      "  Total train batch size (w. parallel, distributed & accumulation) = 32\n",
-      "  Gradient Accumulation steps = 1\n",
-      "  Total optimization steps = 650\n"
-     ]
-    },
     {
      "data": {
       "text/html": [
@@ -330,61 +192,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "***** Running Evaluation *****\n",
+      "...\n",
-      "  Num examples = 100\n",
-      "  Batch size = 32\n",
-      "Saving model checkpoint to models/checkpoint-13\n",
-      "Configuration saved in models/checkpoint-13/config.json\n",
-      "Model weights saved in models/checkpoint-13/pytorch_model.bin\n",
-      "Deleting older checkpoint [models/checkpoint-91] due to args.save_total_limit\n",
-      "***** Running Evaluation *****\n",
-      "  Num examples = 100\n",
-      "  Batch size = 32\n",
-      "Saving model checkpoint to models/checkpoint-26\n",
-      "Configuration saved in models/checkpoint-26/config.json\n",
-      "Model weights saved in models/checkpoint-26/pytorch_model.bin\n",
-      "Deleting older checkpoint [models/checkpoint-117] due to args.save_total_limit\n",
-      "***** Running Evaluation *****\n",
-      "  Num examples = 100\n",
-      "  Batch size = 32\n",
-      "Saving model checkpoint to models/checkpoint-39\n",
-      "Configuration saved in models/checkpoint-39/config.json\n",
-      "Model weights saved in models/checkpoint-39/pytorch_model.bin\n",
-      "Deleting older checkpoint [models/checkpoint-130] due to args.save_total_limit\n",
-      "***** Running Evaluation *****\n",
-      "  Num examples = 100\n",
-      "  Batch size = 32\n",
-      "Saving model checkpoint to models/checkpoint-52\n",
-      "Configuration saved in models/checkpoint-52/config.json\n",
-      "Model weights saved in models/checkpoint-52/pytorch_model.bin\n",
-      "Deleting older checkpoint [models/checkpoint-143] due to args.save_total_limit\n",
-      "***** Running Evaluation *****\n",
-      "  Num examples = 100\n",
-      "  Batch size = 32\n",
-      "Saving model checkpoint to models/checkpoint-65\n",
-      "Configuration saved in models/checkpoint-65/config.json\n",
-      "Model weights saved in models/checkpoint-65/pytorch_model.bin\n",
-      "Deleting older checkpoint [models/checkpoint-156] due to args.save_total_limit\n",
-      "***** Running Evaluation *****\n",
-      "  Num examples = 100\n",
-      "  Batch size = 32\n",
-      "Saving model checkpoint to models/checkpoint-78\n",
-      "Configuration saved in models/checkpoint-78/config.json\n",
-      "Model weights saved in models/checkpoint-78/pytorch_model.bin\n",
-      "Deleting older checkpoint [models/checkpoint-13] due to args.save_total_limit\n",
-      "***** Running Evaluation *****\n",
-      "  Num examples = 100\n",
-      "  Batch size = 32\n",
-      "Saving model checkpoint to models/checkpoint-91\n",
-      "Configuration saved in models/checkpoint-91/config.json\n",
-      "Model weights saved in models/checkpoint-91/pytorch_model.bin\n",
-      "Deleting older checkpoint [models/checkpoint-26] due to args.save_total_limit\n",
-      "***** Running Evaluation *****\n",
-      "  Num examples = 100\n",
-      "  Batch size = 32\n",
-      "Saving model checkpoint to models/checkpoint-104\n",
-      "Configuration saved in models/checkpoint-104/config.json\n",
-      "Model weights saved in models/checkpoint-104/pytorch_model.bin\n",
       "Deleting older checkpoint [models/checkpoint-39] due to args.save_total_limit\n",
       "***** Running Evaluation *****\n",
       "  Num examples = 100\n",
@@ -418,6 +226,7 @@
     "    TrainingArguments,\n",
     "    EarlyStoppingCallback,\n",
     ")\n",
+    "from pathlib import Path\n",
     "import numpy as np\n",
     "from datasets import Dataset, load_metric\n",
     "\n",
@@ -474,13 +283,17 @@
     ").train()"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The best macro F1-score on the test set is **0.89** which is (not surprisingly) substantially more than the SVM achieved. We have a great model, it's time to deploy it. But first, we have to store it in a secure place."
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 44,
    "metadata": {
-    "colab": {
-     "base_uri": "https://localhost:8080/"
-    },
     "executionInfo": {
      "elapsed": 25368,
      "status": "ok",
@@ -514,387 +327,46 @@
     }
    ],
    "source": [
-    "from great_ai.large_file import LargeFileS3\n",
+    "from great_ai import save_model\n",
-    "\n",
-    "LargeFileS3.configure_credentials_from_file(\"config.ini\")\n",
     "\n",
+    "# save Torch model to local disk\n",
     "model.save_pretrained(\"pretrained\")\n",
-    "LargeFileS3(\"scibert-highlights\").push(\"pretrained\")"
+    "\n",
+    "# upload model from local disk to S3\n",
+    "# (because the S3 credentials have been already set, `save_model` will use LargeFileS3)\n",
+    "save_model(\"pretrained\", key=\"scibert-highlights\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Next: [Part 3](/examples/scibert/deploy)"
    ]
   }
  ],
  "metadata": {
-  "accelerator": "GPU",
-  "colab": {
-   "collapsed_sections": [],
-   "name": "Copy of Forms snippets",
-   "provenance": [
-    {
-     "file_id": "/v2/external/notebooks/snippets/forms.ipynb",
-     "timestamp": 1656585404621
-    }
-   ]
-  },
-  "gpuClass": "standard",
   "kernelspec": {
    "display_name": "Python 3.10.4 ('.env': venv)",
    "language": "python",
    "name": "python3"
   },
   "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
    "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
    "version": "3.10.4"
   },
   "vscode": {
    "interpreter": {
     "hash": "02dd6d3afbfa9fbbe1037d64ad9014965528a1ccad21929d6e72f466389a68ad"
    }
-  },
-  "widgets": {
-   "application/vnd.jupyter.widget-state+json": {
-    "22119b79eb514ad684cc3f00f519fb4a": {
-     "model_module": "@jupyter-widgets/controls",
-     "model_module_version": "1.5.0",
-     "model_name": "DescriptionStyleModel",
-     "state": {
-      "_model_module": "@jupyter-widgets/controls",
-      "_model_module_version": "1.5.0",
-      "_model_name": "DescriptionStyleModel",
-      "_view_count": null,
-      "_view_module": "@jupyter-widgets/base",
-      "_view_module_version": "1.2.0",
-      "_view_name": "StyleView",
-      "description_width": ""
-     }
-    },
-    "32adc54185894f0598c2d9ad438c76e2": {
-     "model_module": "@jupyter-widgets/base",
-     "model_module_version": "1.2.0",
-     "model_name": "LayoutModel",
-     "state": {
-      "_model_module": "@jupyter-widgets/base",
-      "_model_module_version": "1.2.0",
-      "_model_name": "LayoutModel",
-      "_view_count": null,
-      "_view_module": "@jupyter-widgets/base",
-      "_view_module_version": "1.2.0",
-      "_view_name": "LayoutView",
-      "align_content": null,
-      "align_items": null,
-      "align_self": null,
-      "border": null,
-      "bottom": null,
-      "display": null,
-      "flex": null,
-      "flex_flow": null,
-      "grid_area": null,
-      "grid_auto_columns": null,
-      "grid_auto_flow": null,
-      "grid_auto_rows": null,
-      "grid_column": null,
-      "grid_gap": null,
-      "grid_row": null,
-      "grid_template_areas": null,
-      "grid_template_columns": null,
-      "grid_template_rows": null,
-      "height": null,
-      "justify_content": null,
-      "justify_items": null,
-      "left": null,
-      "margin": null,
-      "max_height": null,
-      "max_width": null,
-      "min_height": null,
-      "min_width": null,
-      "object_fit": null,
-      "object_position": null,
-      "order": null,
-      "overflow": null,
-      "overflow_x": null,
-      "overflow_y": null,
-      "padding": null,
-      "right": null,
-      "top": null,
-      "visibility": null,
-      "width": null
-     }
-    },
-    "34631de39509438aad98cbd3fc64c999": {
-     "model_module": "@jupyter-widgets/controls",
-     "model_module_version": "1.5.0",
-     "model_name": "ProgressStyleModel",
-     "state": {
-      "_model_module": "@jupyter-widgets/controls",
-      "_model_module_version": "1.5.0",
-      "_model_name": "ProgressStyleModel",
-      "_view_count": null,
-      "_view_module": "@jupyter-widgets/base",
-      "_view_module_version": "1.2.0",
-      "_view_name": "StyleView",
-      "bar_color": null,
-      "description_width": ""
-     }
-    },
-    "4be5b3c59dc04aa2b92f51654e815589": {
-     "model_module": "@jupyter-widgets/controls",
-     "model_module_version": "1.5.0",
-     "model_name": "FloatProgressModel",
-     "state": {
-      "_dom_classes": [],
-      "_model_module": "@jupyter-widgets/controls",
-      "_model_module_version": "1.5.0",
-      "_model_name": "FloatProgressModel",
-      "_view_count": null,
-      "_view_module": "@jupyter-widgets/controls",
-      "_view_module_version": "1.5.0",
-      "_view_name": "ProgressView",
-      "bar_style": "success",
-      "description": "",
-      "description_tooltip": null,
-      "layout": "IPY_MODEL_cb7ec6240337466d8833c70083e1c3cb",
-      "max": 1,
-      "min": 0,
-      "orientation": "horizontal",
-      "style": "IPY_MODEL_34631de39509438aad98cbd3fc64c999",
-      "value": 1
-     }
-    },
-    "8e0ede9d1dd84c149a7e282211c7071b": {
-     "model_module": "@jupyter-widgets/controls",
-     "model_module_version": "1.5.0",
-     "model_name": "HTMLModel",
-     "state": {
-      "_dom_classes": [],
-      "_model_module": "@jupyter-widgets/controls",
-      "_model_module_version": "1.5.0",
-      "_model_name": "HTMLModel",
-      "_view_count": null,
-      "_view_module": "@jupyter-widgets/controls",
-      "_view_module_version": "1.5.0",
-      "_view_name": "HTMLView",
-      "description": "",
-      "description_tooltip": null,
-      "layout": "IPY_MODEL_32adc54185894f0598c2d9ad438c76e2",
-      "placeholder": "​",
-      "style": "IPY_MODEL_981e11fb9d4f4a2ba28c011741a1eaba",
-      "value": " 1/1 [00:00<00:00,  5.38ba/s]"
-     }
-    },
-    "9185277b1e5945a9a6a3ad75811bc86b": {
-     "model_module": "@jupyter-widgets/controls",
-     "model_module_version": "1.5.0",
-     "model_name": "HTMLModel",
-     "state": {
-      "_dom_classes": [],
-      "_model_module": "@jupyter-widgets/controls",
-      "_model_module_version": "1.5.0",
-      "_model_name": "HTMLModel",
-      "_view_count": null,
-      "_view_module": "@jupyter-widgets/controls",
-      "_view_module_version": "1.5.0",
-      "_view_name": "HTMLView",
-      "description": "",
-      "description_tooltip": null,
-      "layout": "IPY_MODEL_bf773a86ec0a4899bb3636035f7ab35e",
-      "placeholder": "​",
-      "style": "IPY_MODEL_22119b79eb514ad684cc3f00f519fb4a",
-      "value": "100%"
-     }
-    },
-    "981e11fb9d4f4a2ba28c011741a1eaba": {
-     "model_module": "@jupyter-widgets/controls",
-     "model_module_version": "1.5.0",
-     "model_name": "DescriptionStyleModel",
-     "state": {
-      "_model_module": "@jupyter-widgets/controls",
-      "_model_module_version": "1.5.0",
-      "_model_name": "DescriptionStyleModel",
-      "_view_count": null,
-      "_view_module": "@jupyter-widgets/base",
-      "_view_module_version": "1.2.0",
-      "_view_name": "StyleView",
-      "description_width": ""
-     }
-    },
-    "9bd5d2fb87bd428796155cc67d06b333": {
-     "model_module": "@jupyter-widgets/base",
-     "model_module_version": "1.2.0",
-     "model_name": "LayoutModel",
-     "state": {
-      "_model_module": "@jupyter-widgets/base",
-      "_model_module_version": "1.2.0",
-      "_model_name": "LayoutModel",
-      "_view_count": null,
-      "_view_module": "@jupyter-widgets/base",
-      "_view_module_version": "1.2.0",
-      "_view_name": "LayoutView",
-      "align_content": null,
-      "align_items": null,
-      "align_self": null,
-      "border": null,
-      "bottom": null,
-      "display": null,
-      "flex": null,
-      "flex_flow": null,
-      "grid_area": null,
-      "grid_auto_columns": null,
-      "grid_auto_flow": null,
-      "grid_auto_rows": null,
-      "grid_column": null,
-      "grid_gap": null,
-      "grid_row": null,
-      "grid_template_areas": null,
-      "grid_template_columns": null,
-      "grid_template_rows": null,
-      "height": null,
-      "justify_content": null,
-      "justify_items": null,
-      "left": null,
-      "margin": null,
-      "max_height": null,
-      "max_width": null,
-      "min_height": null,
-      "min_width": null,
-      "object_fit": null,
-      "object_position": null,
-      "order": null,
-      "overflow": null,
-      "overflow_x": null,
-      "overflow_y": null,
-      "padding": null,
-      "right": null,
-      "top": null,
-      "visibility": null,
-      "width": null
-     }
-    },
-    "9c57de70e68a41ecbde5093bd671715a": {
-     "model_module": "@jupyter-widgets/controls",
-     "model_module_version": "1.5.0",
-     "model_name": "HBoxModel",
-     "state": {
-      "_dom_classes": [],
-      "_model_module": "@jupyter-widgets/controls",
-      "_model_module_version": "1.5.0",
-      "_model_name": "HBoxModel",
-      "_view_count": null,
-      "_view_module": "@jupyter-widgets/controls",
-      "_view_module_version": "1.5.0",
-      "_view_name": "HBoxView",
-      "box_style": "",
-      "children": [
-       "IPY_MODEL_9185277b1e5945a9a6a3ad75811bc86b",
-       "IPY_MODEL_4be5b3c59dc04aa2b92f51654e815589",
-       "IPY_MODEL_8e0ede9d1dd84c149a7e282211c7071b"
-      ],
-      "layout": "IPY_MODEL_9bd5d2fb87bd428796155cc67d06b333"
-     }
-    },
-    "bf773a86ec0a4899bb3636035f7ab35e": {
-     "model_module": "@jupyter-widgets/base",
-     "model_module_version": "1.2.0",
-     "model_name": "LayoutModel",
-     "state": {
-      "_model_module": "@jupyter-widgets/base",
-      "_model_module_version": "1.2.0",
-      "_model_name": "LayoutModel",
-      "_view_count": null,
-      "_view_module": "@jupyter-widgets/base",
-      "_view_module_version": "1.2.0",
-      "_view_name": "LayoutView",
-      "align_content": null,
-      "align_items": null,
-      "align_self": null,
-      "border": null,
-      "bottom": null,
-      "display": null,
-      "flex": null,
-      "flex_flow": null,
-      "grid_area": null,
-      "grid_auto_columns": null,
-      "grid_auto_flow": null,
-      "grid_auto_rows": null,
-      "grid_column": null,
-      "grid_gap": null,
-      "grid_row": null,
-      "grid_template_areas": null,
-      "grid_template_columns": null,
-      "grid_template_rows": null,
-      "height": null,
-      "justify_content": null,
-      "justify_items": null,
-      "left": null,
-      "margin": null,
-      "max_height": null,
-      "max_width": null,
-      "min_height": null,
-      "min_width": null,
-      "object_fit": null,
-      "object_position": null,
-      "order": null,
-      "overflow": null,
-      "overflow_x": null,
-      "overflow_y": null,
-      "padding": null,
-      "right": null,
-      "top": null,
-      "visibility": null,
-      "width": null
-     }
-    },
-    "cb7ec6240337466d8833c70083e1c3cb": {
-     "model_module": "@jupyter-widgets/base",
-     "model_module_version": "1.2.0",
-     "model_name": "LayoutModel",
-     "state": {
-      "_model_module": "@jupyter-widgets/base",
-      "_model_module_version": "1.2.0",
-      "_model_name": "LayoutModel",
-      "_view_count": null,
-      "_view_module": "@jupyter-widgets/base",
-      "_view_module_version": "1.2.0",
-      "_view_name": "LayoutView",
-      "align_content": null,
-      "align_items": null,
-      "align_self": null,
-      "border": null,
-      "bottom": null,
-      "display": null,
-      "flex": null,
-      "flex_flow": null,
-      "grid_area": null,
-      "grid_auto_columns": null,
-      "grid_auto_flow": null,
-      "grid_auto_rows": null,
-      "grid_column": null,
-      "grid_gap": null,
-      "grid_row": null,
-      "grid_template_areas": null,
-      "grid_template_columns": null,
-      "grid_template_rows": null,
-      "height": null,
-      "justify_content": null,
-      "justify_items": null,
-      "left": null,
-      "margin": null,
-      "max_height": null,
-      "max_width": null,
-      "min_height": null,
-      "min_width": null,
-      "object_fit": null,
-      "object_position": null,
-      "order": null,
-      "overflow": null,
-      "overflow_x": null,
-      "overflow_y": null,
-      "padding": null,
-      "right": null,
-      "top": null,
-      "visibility": null,
-      "width": null
-     }
-    }
-   }
   }
  },
  "nbformat": 4,

docs/examples/scibert/views/__init__.py

@@ -1,2 +0,0 @@
-from .evaluated_sentence import EvaluatedSentence
-from .match import Match

docs/examples/scibert/views/evaluated_sentence.py

@@ -1,11 +0,0 @@
-from typing import List
-
-from pydantic import BaseModel
-
-from .match import Match
-
-
-class EvaluatedSentence(BaseModel):
-    score: float
-    text: str
-    explanation: List[Match]

docs/examples/scibert/views/match.py

@@ -1,6 +0,0 @@
-from pydantic import BaseModel
-
-
-class Match(BaseModel):
-    phrase: str
-    score: float

mkdocs.yaml

@@ -63,7 +63,6 @@ plugins:
   - search
   - mkdocs-jupyter:
       include_source: true
-      ignore: ["docs/examples/scibert/*.ipynb"]
 
 markdown_extensions:
   - pymdownx.highlight:
@@ -111,5 +110,10 @@ nav:
           - examples/simple/data.ipynb
           - examples/simple/train.ipynb
           - examples/simple/deploy.ipynb
-      - Explainable SciBERT: examples/scibert.md
+      - Explainable SciBERT:
+          - examples/scibert/index.md
+          - examples/scibert/data.ipynb
+          - examples/scibert/train.ipynb
+          - examples/scibert/deploy.ipynb
+          - examples/scibert/additional-files.md
   - explanation.md