{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train a domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)\n",
    "\n",
    "## Part 1: obtain clean data\n",
    "\n",
    "This can be achieved by downloading a public dataset (such as in this case), or by having a Data Engineer setup and give us access to the organisations's data.\n",
    "\n",
    "1. **Extract**: Download the semantic scholar dataset from a public S3 bucket\n",
    "2. **Transform**: Project it to only keep the necessary components (text and labels), clean the textual content using `great_ai.utilities.clean`\n",
    "3. **Load**: Upload the dataset (or a part of it) to a shared infrastructure using `great_ai.LargeFile` with its S3 backend\n",
    "\n",
    "> We will speed up the processing using `great_ai.utilities.parallel_map`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[38;5;39m2022-05-27 11:02:29,131 |     INFO | Environment variable ENVIRONMENT is not set, defaulting to development mode\u001b[0m\n",
      "\u001b[38;5;226m2022-05-27 11:02:29,132 |  WARNING | Running in development mode ‼️\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:02:29,133 |     INFO | Options: configured ✅\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "from typing import List, Tuple\n",
    "import urllib.request\n",
    "from itertools import chain\n",
    "import gzip\n",
    "from great_ai.utilities.clean import clean\n",
    "from great_ai.utilities.parallel_map import parallel_map\n",
    "from great_ai.utilities.language import is_english, predict_language\n",
    "from great_ai import configure, LargeFile\n",
    "\n",
    "\n",
    "DATASET_KEY = \"semantic-scholar-dataset-small\"\n",
    "MAX_FILE_COUNT = 12\n",
    "\n",
    "configure()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Download the list of available chunks\n",
    "\n",
    "manifest = (\n",
    "    urllib.request.urlopen(\n",
    "        \"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/manifest.txt\"\n",
    "    )\n",
    "    .read()\n",
    "    .decode()\n",
    ")\n",
    "chunks = manifest.split()[:MAX_FILE_COUNT]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[38;5;39m2022-05-27 11:02:30,364 |     INFO | Starting parallel map (concurrency: 12, chunk size: 1)\u001b[0m\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f6ee8c2708e24e53a5a00428fd40e7da",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[38;5;39m2022-05-27 11:04:30,870 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,177)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:36,961 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,2108)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:44,270 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1302)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:44,300 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1408)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:44,324 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1497)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:44,347 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1553)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:45,265 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,470)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:45,302 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,606)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:45,331 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,881)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:45,355 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,894)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:45,379 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,953)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:45,396 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,966)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:45,416 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1046)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:45,431 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1056)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:45,448 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1121)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:45,464 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1132)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:45,483 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1286)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:45,507 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1297)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:45,526 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1359)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:45,544 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1369)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:45,560 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1414)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:45,576 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1428)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:45,592 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1484)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:45,608 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1498)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:45,632 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1593)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:45,654 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1607)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:46,484 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,939)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:47,579 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,388)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:47,606 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,713)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:47,919 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,871)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:49,879 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1177)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:53,969 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected closing math mode: '\\]' @(1,1233)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:58,136 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,7084)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:04:58,520 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,29)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:05,993 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,752)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:06,079 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,605)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:06,698 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,311)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:08,997 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(11,19)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:10,844 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,981)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:13,325 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,430)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:16,625 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,65)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:16,660 |     INFO | Ignoring parse error (tolerant parsing mode): Bad \\end macro: expected {environmentname} @(1,1059)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:17,496 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1569)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:19,161 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,319)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:24,219 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,277)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:24,477 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(30,17)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:26,147 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1619)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:29,802 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,846)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:29,982 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,241)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:31,986 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,577)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:32,095 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,392)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:32,516 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1746)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:32,961 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,211)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:41,376 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,459)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:41,402 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,838)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:43,892 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,51)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:46,151 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,221)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:46,187 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,232)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:46,211 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,268)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:46,787 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,233)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:48,708 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1728)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:51,076 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1474)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:52,706 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,955)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:52,731 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1003)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:52,751 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,120)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:55,564 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,105)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:57,188 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1398)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:05:58,994 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1079)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:00,466 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,276)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:00,497 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,483)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:02,578 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,666)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:04,618 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,476)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:04,620 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,554)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:05,738 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,215)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:08,360 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,436)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:08,383 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,452)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:08,416 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,595)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:08,440 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,716)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:08,468 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,889)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:08,664 |     INFO | Ignoring parse error (tolerant parsing mode): End of input while parsing arguments of macro \"\"\" @(1,116)\n",
      "Open LaTeX blocks:\n",
      "          @(1,116)  arguments of macro \"\"\"\n",
      "\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:13,458 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1194)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:15,817 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,994)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:17,295 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,41)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:17,323 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,84)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:17,340 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,131)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:17,360 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,202)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:17,383 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,216)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:17,407 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,219)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:17,427 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,225)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:17,451 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,272)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:17,468 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,319)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:17,487 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,348)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:17,511 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,349)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:17,533 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,410)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:17,548 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,414)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:17,571 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,437)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:17,595 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,465)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:17,619 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,480)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:17,639 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,501)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:17,663 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,507)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:17,687 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,537)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:17,711 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,552)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:19,274 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,978)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:23,079 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,577)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:24,409 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1879)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:27,015 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,133)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:27,360 |     INFO | Ignoring parse error (tolerant parsing mode): Bad \\end macro: expected {environmentname} @(1,713)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:33,252 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,169)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:33,294 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,2020)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:35,078 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,5231)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:35,912 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,573)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:37,633 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,365)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:39,974 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,83)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:41,592 |     INFO | Ignoring parse error (tolerant parsing mode): End of input while parsing arguments of macro \"\"\" @(1,216)\n",
      "Open LaTeX blocks:\n",
      "          @(1,216)  arguments of macro \"\"\"\n",
      "\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:42,432 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,551)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:43,077 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,597)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:43,203 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,58)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:44,178 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,782)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:46,846 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,84)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:46,875 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,95)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:46,891 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,106)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:46,915 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,117)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:46,935 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,128)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:46,959 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,138)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:47,528 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,2778)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:48,175 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,197)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:51,821 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,271)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:51,844 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,380)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:51,867 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,395)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:51,891 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,410)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:51,916 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,480)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:51,939 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,497)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:51,965 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,711)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:51,987 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,744)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:06:56,684 |     INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,708)\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "# Extract, Transform\n",
    "\n",
    "\n",
    "def process_chunk(chunk_key: str) -> List[Tuple[str, List[str]]]:\n",
    "    response = urllib.request.urlopen(\n",
    "        \"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/\"\n",
    "        + chunk_key\n",
    "    )  # returns a gzipped JSON Lines file\n",
    "    decompressed = gzip.decompress(response.read())\n",
    "    decoded = decompressed.decode()\n",
    "    chunk = [json.loads(line) for line in decoded.split(\"\\n\") if line]\n",
    "\n",
    "    all = [\n",
    "        # Create pairs of `(text, [domains])`\n",
    "        # The text is cleaned to remove PDF extraction, web scraping, and other common artifacts\n",
    "        (\n",
    "            clean(\n",
    "                f'{c[\"title\"]} {c[\"paperAbstract\"]} {c[\"journalName\"]} {c[\"venue\"]}',\n",
    "                convert_to_ascii=True,\n",
    "            ),\n",
    "            c[\"fieldsOfStudy\"],\n",
    "        )\n",
    "        for c in chunk\n",
    "    ]\n",
    "\n",
    "    return [\n",
    "        (content, domains)\n",
    "        for content, domains in all\n",
    "        if (domains and content and is_english(predict_language(content)))\n",
    "    ]\n",
    "\n",
    "\n",
    "clean_chunks = parallel_map(process_chunk, chunks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[38;5;39m2022-05-27 11:01:09,266 |     INFO | Fetching online versions of semantic-scholar-dataset-small\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:01:09,581 |     INFO | Found versions: [0]\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:01:09,890 |     INFO | Copying file for semantic-scholar-dataset-small-1\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:01:09,916 |     INFO | Compressing semantic-scholar-dataset-small-1\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:01:11,197 |     INFO | Uploading semantic-scholar-dataset-small-1 to S3 from /tmp/large-file-s55zn39v\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:01:13,095 |     INFO | Uploading semantic-scholar-dataset-small-1.tar.gz 0.79/7.46 MB (10.5%)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:01:14,729 |     INFO | Uploading semantic-scholar-dataset-small-1.tar.gz 1.57/7.46 MB (21.1%)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:01:16,608 |     INFO | Uploading semantic-scholar-dataset-small-1.tar.gz 2.36/7.46 MB (31.6%)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:01:18,209 |     INFO | Uploading semantic-scholar-dataset-small-1.tar.gz 3.15/7.46 MB (42.2%)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:01:19,471 |     INFO | Uploading semantic-scholar-dataset-small-1.tar.gz 3.93/7.46 MB (52.7%)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:01:20,607 |     INFO | Uploading semantic-scholar-dataset-small-1.tar.gz 4.72/7.46 MB (63.3%)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:01:22,727 |     INFO | Uploading semantic-scholar-dataset-small-1.tar.gz 5.51/7.46 MB (73.8%)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:01:24,953 |     INFO | Uploading semantic-scholar-dataset-small-1.tar.gz 6.29/7.46 MB (84.4%)\u001b[0m\n",
      "\u001b[38;5;39m2022-05-27 11:01:27,016 |     INFO | Uploading semantic-scholar-dataset-small-1.tar.gz 7.08/7.46 MB (94.9%)\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "# Load\n",
    "# Using this GreatAI utility, the data will be made available on S3 (and in our local cache)\n",
    "\n",
    "with LargeFile(DATASET_KEY, \"w\", keep_last_n=1) as f:\n",
    "    json.dump(list(chain(clean_chunks)), f)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "9583b8c15346c7ad861da481b60c8e8605a71a48eda767f1640baa8f25efebcb"
  },
  "kernelspec": {
   "display_name": "Python 3.9.2 ('.venv': venv)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}