andras/great-ai
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases 4 Packages Wiki Activity Actions

Deployed 0585a4e with MkDocs version: 1.3.0

Browse source
This commit is contained in:
2022-07-12 19:53:07 +00:00
parent 7305d2ead3
commit 8cda980b81
33 changed files with 782 additions and 464 deletions
Download patch file Download diff file Expand all files Collapse all files

19
examples/simple/data/data.ipynb
View file

@ -51,7 +51,8 @@
"\n",
"manifest = (\n",
" urllib.request.urlopen(\n",
" \"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/manifest.txt\"\n",
" \"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/\"\n",
" \"open-corpus/2022-02-01/manifest.txt\"\n",
" )\n",
" .read()\n",
" .decode()\n",
@ -61,7 +62,9 @@
"shuffle(lines)\n",
"chunks = lines[:MAX_CHUNK_COUNT]\n",
"\n",
"f\"Processing {len(chunks)} out of the {len(manifest.split())} available chunks\""
"f\"\"\"Processing {len(chunks)} out of the {\n",
" len(manifest.split())\n",
"} available chunks\"\"\""
]
},
{
@ -104,7 +107,8 @@
"def preprocess_chunk(chunk_key: str) -> List[Tuple[str, List[str]]]:\n",
" # Extract\n",
" response = urllib.request.urlopen(\n",
" f\"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/{chunk_key}\"\n",
" f\"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/\"\n",
" \"open-corpus/2022-02-01/{chunk_key}\"\n",
" ) # a gzipped JSON Lines file\n",
"\n",
" decompressed = gzip.decompress(response.read())\n",
@ -115,13 +119,14 @@
" return [\n",
" (\n",
" clean(\n",
" f'{c[\"title\"]} {c[\"paperAbstract\"]} {c[\"journalName\"]} {c[\"venue\"]}',\n",
" f'{c[\"title\"]} {c[\"paperAbstract\"]} '\n",
" f'{c[\"journalName\"]} {c[\"venue\"]}',\n",
" convert_to_ascii=True,\n",
" ), # The text is cleaned to remove PDF extraction, web scraping, and other common artifacts\n",
" ), # The text is cleaned to remove common artifacts\n",
" c[\"fieldsOfStudy\"],\n",
" ) # Create pairs of `(text, [...domains])`\n",
" for c in chunk\n",
" if c[\"fieldsOfStudy\"] and is_english(predict_language(c[\"paperAbstract\"]))\n",
" if (c[\"fieldsOfStudy\"] and is_english(predict_language(c[\"paperAbstract\"])))\n",
" ]\n",
"\n",
"\n",
@ -145,7 +150,7 @@
"source": [
"### Load\n",
"\n",
"Upload the dataset (or a part of it) to a central repository using `great_ai.add_ground_truth`. This step automatically tags each datapoint with a split label according to the ratios we set. Additional tags can be also given.\n",
"Upload the dataset (or a part of it) to a central repository using `great_ai.add_ground_truth`. This step automatically tags each data-point with a split label according to the ratios we set. Additional tags can be also given.\n",
"\n",
"#### Production-ready backend\n",
"\n",