Add data engineer script
This commit is contained in:
parent
68fb486ba3
commit
f0ea6d82bd
1 changed files with 341 additions and 0 deletions
341
examples/simple/data.ipynb
Normal file
341
examples/simple/data.ipynb
Normal file
|
|
@ -0,0 +1,341 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Train a domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)\n",
|
||||
"\n",
|
||||
"## Part 1: obtain clean data\n",
|
||||
"\n",
|
||||
"This can be achieved by downloading a public dataset (such as in this case), or by having a Data Engineer setup and give us access to the organisations's data.\n",
|
||||
"\n",
|
||||
"1. **Extract**: Download the semantic scholar dataset from a public S3 bucket\n",
|
||||
"2. **Transform**: Project it to only keep the necessary components (text and labels), clean the textual content using `great_ai.utilities.clean`\n",
|
||||
"3. **Load**: Upload the dataset (or a part of it) to a shared infrastructure using `great_ai.LargeFile` with its S3 backend\n",
|
||||
"\n",
|
||||
"> We will speed up the processing using `great_ai.utilities.parallel_map`"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\u001b[38;5;39m2022-05-27 11:02:29,131 | INFO | Environment variable ENVIRONMENT is not set, defaulting to development mode\u001b[0m\n",
|
||||
"\u001b[38;5;226m2022-05-27 11:02:29,132 | WARNING | Running in development mode ‼️\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:02:29,133 | INFO | Options: configured ✅\u001b[0m\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import json\n",
|
||||
"from typing import List, Tuple\n",
|
||||
"import urllib.request\n",
|
||||
"from itertools import chain\n",
|
||||
"import gzip\n",
|
||||
"from great_ai.utilities.clean import clean\n",
|
||||
"from great_ai.utilities.parallel_map import parallel_map\n",
|
||||
"from great_ai.utilities.language import is_english, predict_language\n",
|
||||
"from great_ai import configure, LargeFile\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"DATASET_KEY = \"semantic-scholar-dataset-small\"\n",
|
||||
"MAX_FILE_COUNT = 12\n",
|
||||
"\n",
|
||||
"configure()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Download the list of available chunks\n",
|
||||
"\n",
|
||||
"manifest = (\n",
|
||||
" urllib.request.urlopen(\n",
|
||||
" \"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/manifest.txt\"\n",
|
||||
" )\n",
|
||||
" .read()\n",
|
||||
" .decode()\n",
|
||||
")\n",
|
||||
"chunks = manifest.split()[:MAX_FILE_COUNT]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\u001b[38;5;39m2022-05-27 11:02:30,364 | INFO | Starting parallel map (concurrency: 12, chunk size: 1)\u001b[0m\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "f6ee8c2708e24e53a5a00428fd40e7da",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/12 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\u001b[38;5;39m2022-05-27 11:04:30,870 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,177)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:36,961 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,2108)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:44,270 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1302)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:44,300 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1408)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:44,324 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1497)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:44,347 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1553)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:45,265 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,470)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:45,302 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,606)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:45,331 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,881)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:45,355 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,894)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:45,379 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,953)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:45,396 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,966)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:45,416 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1046)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:45,431 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1056)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:45,448 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1121)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:45,464 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1132)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:45,483 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1286)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:45,507 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1297)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:45,526 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1359)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:45,544 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1369)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:45,560 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1414)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:45,576 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1428)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:45,592 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1484)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:45,608 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1498)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:45,632 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1593)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:45,654 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1607)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:46,484 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,939)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:47,579 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,388)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:47,606 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,713)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:47,919 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,871)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:49,879 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1177)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:53,969 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected closing math mode: '\\]' @(1,1233)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:58,136 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,7084)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:04:58,520 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,29)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:05,993 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,752)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:06,079 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,605)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:06,698 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,311)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:08,997 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(11,19)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:10,844 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,981)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:13,325 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,430)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:16,625 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,65)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:16,660 | INFO | Ignoring parse error (tolerant parsing mode): Bad \\end macro: expected {environmentname} @(1,1059)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:17,496 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1569)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:19,161 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,319)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:24,219 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,277)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:24,477 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(30,17)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:26,147 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1619)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:29,802 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,846)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:29,982 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,241)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:31,986 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,577)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:32,095 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,392)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:32,516 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1746)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:32,961 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,211)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:41,376 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,459)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:41,402 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,838)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:43,892 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,51)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:46,151 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,221)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:46,187 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,232)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:46,211 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,268)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:46,787 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,233)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:48,708 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1728)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:51,076 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1474)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:52,706 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,955)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:52,731 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1003)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:52,751 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,120)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:55,564 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,105)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:57,188 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1398)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:05:58,994 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1079)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:00,466 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,276)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:00,497 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,483)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:02,578 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,666)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:04,618 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,476)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:04,620 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,554)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:05,738 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,215)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:08,360 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,436)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:08,383 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,452)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:08,416 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,595)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:08,440 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,716)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:08,468 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,889)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:08,664 | INFO | Ignoring parse error (tolerant parsing mode): End of input while parsing arguments of macro \"\"\" @(1,116)\n",
|
||||
"Open LaTeX blocks:\n",
|
||||
" @(1,116) arguments of macro \"\"\"\n",
|
||||
"\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:13,458 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1194)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:15,817 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,994)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:17,295 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,41)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:17,323 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,84)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:17,340 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,131)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:17,360 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,202)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:17,383 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,216)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:17,407 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,219)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:17,427 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,225)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:17,451 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,272)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:17,468 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,319)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:17,487 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,348)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:17,511 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,349)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:17,533 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,410)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:17,548 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,414)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:17,571 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,437)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:17,595 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,465)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:17,619 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,480)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:17,639 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,501)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:17,663 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,507)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:17,687 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,537)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:17,711 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,552)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:19,274 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,978)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:23,079 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,577)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:24,409 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1879)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:27,015 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,133)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:27,360 | INFO | Ignoring parse error (tolerant parsing mode): Bad \\end macro: expected {environmentname} @(1,713)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:33,252 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,169)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:33,294 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,2020)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:35,078 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,5231)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:35,912 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,573)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:37,633 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,365)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:39,974 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,83)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:41,592 | INFO | Ignoring parse error (tolerant parsing mode): End of input while parsing arguments of macro \"\"\" @(1,216)\n",
|
||||
"Open LaTeX blocks:\n",
|
||||
" @(1,216) arguments of macro \"\"\"\n",
|
||||
"\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:42,432 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,551)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:43,077 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,597)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:43,203 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,58)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:44,178 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,782)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:46,846 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,84)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:46,875 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,95)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:46,891 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,106)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:46,915 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,117)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:46,935 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,128)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:46,959 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,138)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:47,528 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,2778)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:48,175 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,197)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:51,821 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,271)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:51,844 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,380)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:51,867 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,395)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:51,891 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,410)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:51,916 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,480)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:51,939 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,497)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:51,965 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,711)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:51,987 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,744)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:06:56,684 | INFO | Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,708)\u001b[0m\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Extract, Transform\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def process_chunk(chunk_key: str) -> List[Tuple[str, List[str]]]:\n",
|
||||
" response = urllib.request.urlopen(\n",
|
||||
" \"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/\"\n",
|
||||
" + chunk_key\n",
|
||||
" ) # returns a gzipped JSON Lines file\n",
|
||||
" decompressed = gzip.decompress(response.read())\n",
|
||||
" decoded = decompressed.decode()\n",
|
||||
" chunk = [json.loads(line) for line in decoded.split(\"\\n\") if line]\n",
|
||||
"\n",
|
||||
" all = [\n",
|
||||
" # Create pairs of `(text, [domains])`\n",
|
||||
" # The text is cleaned to remove PDF extraction, web scraping, and other common artifacts\n",
|
||||
" (\n",
|
||||
" clean(\n",
|
||||
" f'{c[\"title\"]} {c[\"paperAbstract\"]} {c[\"journalName\"]} {c[\"venue\"]}',\n",
|
||||
" convert_to_ascii=True,\n",
|
||||
" ),\n",
|
||||
" c[\"fieldsOfStudy\"],\n",
|
||||
" )\n",
|
||||
" for c in chunk\n",
|
||||
" ]\n",
|
||||
"\n",
|
||||
" return [\n",
|
||||
" (content, domains)\n",
|
||||
" for content, domains in all\n",
|
||||
" if (domains and content and is_english(predict_language(content)))\n",
|
||||
" ]\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"clean_chunks = parallel_map(process_chunk, chunks)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\u001b[38;5;39m2022-05-27 11:01:09,266 | INFO | Fetching online versions of semantic-scholar-dataset-small\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:01:09,581 | INFO | Found versions: [0]\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:01:09,890 | INFO | Copying file for semantic-scholar-dataset-small-1\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:01:09,916 | INFO | Compressing semantic-scholar-dataset-small-1\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:01:11,197 | INFO | Uploading semantic-scholar-dataset-small-1 to S3 from /tmp/large-file-s55zn39v\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:01:13,095 | INFO | Uploading semantic-scholar-dataset-small-1.tar.gz 0.79/7.46 MB (10.5%)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:01:14,729 | INFO | Uploading semantic-scholar-dataset-small-1.tar.gz 1.57/7.46 MB (21.1%)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:01:16,608 | INFO | Uploading semantic-scholar-dataset-small-1.tar.gz 2.36/7.46 MB (31.6%)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:01:18,209 | INFO | Uploading semantic-scholar-dataset-small-1.tar.gz 3.15/7.46 MB (42.2%)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:01:19,471 | INFO | Uploading semantic-scholar-dataset-small-1.tar.gz 3.93/7.46 MB (52.7%)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:01:20,607 | INFO | Uploading semantic-scholar-dataset-small-1.tar.gz 4.72/7.46 MB (63.3%)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:01:22,727 | INFO | Uploading semantic-scholar-dataset-small-1.tar.gz 5.51/7.46 MB (73.8%)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:01:24,953 | INFO | Uploading semantic-scholar-dataset-small-1.tar.gz 6.29/7.46 MB (84.4%)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 11:01:27,016 | INFO | Uploading semantic-scholar-dataset-small-1.tar.gz 7.08/7.46 MB (94.9%)\u001b[0m\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Load\n",
|
||||
"# Using this GreatAI utility, the data will be made available on S3 (and in our local cache)\n",
|
||||
"\n",
|
||||
"with LargeFile(DATASET_KEY, \"w\", keep_last_n=1) as f:\n",
|
||||
" json.dump(list(chain(clean_chunks)), f)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"interpreter": {
|
||||
"hash": "9583b8c15346c7ad861da481b60c8e8605a71a48eda767f1640baa8f25efebcb"
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3.9.2 ('.venv': venv)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.9.2"
|
||||
},
|
||||
"orig_nbformat": 4
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue