{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Train a domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)\n", "\n", "## Part 1: obtain clean data\n", "\n", "This can be achieved by downloading a public dataset (such as in this case), or by having a Data Engineer setup and give us access to the organisations's data.\n", "\n", "1. **Extract**: Download the semantic scholar dataset from a public S3 bucket\n", "2. **Transform**: Project it to only keep the necessary components (text and labels), clean the textual content using `great_ai.utilities.clean`\n", "3. **Load**: Upload the dataset (or a part of it) to a shared infrastructure using `great_ai.LargeFile` with its S3 backend\n", "\n", "> We will speed up the processing using `great_ai.utilities.parallel_map`" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "\u001b[38;5;39m2022-05-27 12:12:27,324 | INFO | Environment variable ENVIRONMENT is not set, defaulting to development mode\u001b[0m\n", "\u001b[38;5;226m2022-05-27 12:12:27,324 | WARNING | Running in development mode ‼️\u001b[0m\n", "\u001b[38;5;39m2022-05-27 12:12:27,325 | INFO | Options: configured ✅\u001b[0m\n" ] } ], "source": [ "import json\n", "from typing import List, Tuple\n", "import urllib.request\n", "from itertools import chain\n", "import gzip\n", "from great_ai.utilities.clean import clean\n", "from great_ai.utilities.parallel_map import parallel_map\n", "from great_ai.utilities.language import is_english, predict_language\n", "from great_ai import configure, LargeFile\n", "\n", "\n", "DATASET_KEY = \"semantic-scholar-dataset-small\"\n", "MAX_FILE_COUNT = 12\n", "configure()" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "# Download the list of available chunks\n", "\n", "manifest = (\n", " urllib.request.urlopen(\n", " \"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/manifest.txt\"\n", " )\n", " .read()\n", " .decode()\n", ")\n", "chunks = manifest.split()[:MAX_FILE_COUNT]" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "\u001b[38;5;39m2022-05-27 12:12:28,639 | INFO | Starting parallel map (concurrency: 12, chunk size: 1)\u001b[0m\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "7a381a677ed240e08c1605cd0917616a", "version_major": 2, "version_minor": 0 }, "text/plain": [ " 0%| | 0/12 [00:00 List[Tuple[str, List[str]]]:\n", " response = urllib.request.urlopen(\n", " \"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/\"\n", " + chunk_key\n", " ) # returns a gzipped JSON Lines file\n", " decompressed = gzip.decompress(response.read())\n", " decoded = decompressed.decode()\n", " chunk = [json.loads(line) for line in decoded.split(\"\\n\") if line]\n", "\n", " all = [\n", " # Create pairs of `(text, [domains])`\n", " # The text is cleaned to remove PDF extraction, web scraping, and other common artifacts\n", " (\n", " clean(\n", " f'{c[\"title\"]} {c[\"paperAbstract\"]} {c[\"journalName\"]} {c[\"venue\"]}',\n", " convert_to_ascii=True,\n", " ),\n", " c[\"fieldsOfStudy\"],\n", " )\n", " for c in chunk\n", " ]\n", "\n", " return [\n", " (content, domains)\n", " for content, domains in all\n", " if (domains and content and is_english(predict_language(content)))\n", " ]\n", "\n", "\n", "clean_chunks = parallel_map(process_chunk, chunks)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "\u001b[38;5;39m2022-05-27 12:22:35,833 | INFO | Fetching online versions of semantic-scholar-dataset-small\u001b[0m\n", "\u001b[38;5;39m2022-05-27 12:22:36,133 | INFO | Found versions: [2, 3]\u001b[0m\n", "\u001b[38;5;39m2022-05-27 12:22:39,542 | INFO | Copying file for semantic-scholar-dataset-small-4\u001b[0m\n", "\u001b[38;5;39m2022-05-27 12:22:39,843 | INFO | Compressing semantic-scholar-dataset-small-4\u001b[0m\n", "\u001b[38;5;39m2022-05-27 12:22:54,990 | INFO | Uploading semantic-scholar-dataset-small-4 to S3 from /tmp/large-file-l9z1mo61\u001b[0m\n", "\u001b[38;5;39m2022-05-27 12:23:19,988 | INFO | Uploading semantic-scholar-dataset-small-4.tar.gz 8.91/88.72 MB (10.0%)\u001b[0m\n", "\u001b[38;5;39m2022-05-27 12:23:41,914 | INFO | Uploading semantic-scholar-dataset-small-4.tar.gz 17.83/88.72 MB (20.1%)\u001b[0m\n", "\u001b[38;5;39m2022-05-27 12:24:06,996 | INFO | Uploading semantic-scholar-dataset-small-4.tar.gz 26.74/88.72 MB (30.1%)\u001b[0m\n", "\u001b[38;5;39m2022-05-27 12:24:30,563 | INFO | Uploading semantic-scholar-dataset-small-4.tar.gz 35.65/88.72 MB (40.2%)\u001b[0m\n", "\u001b[38;5;39m2022-05-27 12:24:59,623 | INFO | Uploading semantic-scholar-dataset-small-4.tar.gz 44.56/88.72 MB (50.2%)\u001b[0m\n", "\u001b[38;5;39m2022-05-27 12:25:28,237 | INFO | Uploading semantic-scholar-dataset-small-4.tar.gz 53.48/88.72 MB (60.3%)\u001b[0m\n", "\u001b[38;5;39m2022-05-27 12:25:59,432 | INFO | Uploading semantic-scholar-dataset-small-4.tar.gz 62.13/88.72 MB (70.0%)\u001b[0m\n", "\u001b[38;5;39m2022-05-27 12:26:25,016 | INFO | Uploading semantic-scholar-dataset-small-4.tar.gz 71.04/88.72 MB (80.1%)\u001b[0m\n", "\u001b[38;5;39m2022-05-27 12:26:55,491 | INFO | Uploading semantic-scholar-dataset-small-4.tar.gz 79.95/88.72 MB (90.1%)\u001b[0m\n", "\u001b[38;5;39m2022-05-27 12:27:26,801 | INFO | Uploading semantic-scholar-dataset-small-4.tar.gz 88.72/88.72 MB (100.0%)\u001b[0m\n", "\u001b[38;5;39m2022-05-27 12:27:28,540 | INFO | Removing old version (keep_last_n=1): semantic-scholar-dataset-small/2\u001b[0m\n" ] } ], "source": [ "# Load\n", "# Using the LargeFile GreatAI utility, the data will be made available on S3 (and in our local cache)\n", "\n", "with LargeFile(DATASET_KEY, \"w\", keep_last_n=1) as f:\n", " json.dump(list(chain(*clean_chunks)), f)" ] } ], "metadata": { "interpreter": { "hash": "9583b8c15346c7ad861da481b60c8e8605a71a48eda767f1640baa8f25efebcb" }, "kernelspec": { "display_name": "Python 3.9.2 ('.venv': venv)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.2" }, "orig_nbformat": 4 }, "nbformat": 4, "nbformat_minor": 2 }