Streamline example
This commit is contained in:
parent
8f4a320702
commit
266e43b18a
13 changed files with 1255 additions and 524 deletions
7
examples/simple/README.md
Normal file
7
examples/simple/README.md
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
# Train a domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)
|
||||
|
||||
|
||||
|
||||
- [Part 1](data.ipynb)
|
||||
- [Part 2](train.ipynb)
|
||||
- [Part 3](deploy.ipynb)
|
||||
|
|
@ -5,57 +5,50 @@
|
|||
"metadata": {},
|
||||
"source": [
|
||||
"# Train a domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)\n",
|
||||
"> Part 1: obtain and clean data\n",
|
||||
"\n",
|
||||
"## Part 1: obtain clean data\n",
|
||||
"\n",
|
||||
"This can be achieved by downloading a public dataset (such as in this case), or by having a Data Engineer setup and give us access to the organisations's data.\n",
|
||||
"\n",
|
||||
"1. **Extract**: Download the semantic scholar dataset from a public S3 bucket\n",
|
||||
"2. **Transform**: Project it to only keep the necessary components (text and labels), clean the textual content using `great_ai.utilities.clean`\n",
|
||||
"3. **Load**: Upload the dataset (or a part of it) to a shared infrastructure using `great_ai.LargeFile` with its S3 backend\n",
|
||||
"\n",
|
||||
"> We will speed up the processing using `great_ai.utilities.parallel_map`"
|
||||
"\n",
|
||||
"> The blue boxes show the steps implemented in this notebook."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\u001b[38;5;39m2022-05-27 12:12:27,324 | INFO | Environment variable ENVIRONMENT is not set, defaulting to development mode\u001b[0m\n",
|
||||
"\u001b[38;5;226m2022-05-27 12:12:27,324 | WARNING | Running in development mode ‼️\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 12:12:27,325 | INFO | Options: configured ✅\u001b[0m\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import json\n",
|
||||
"from typing import List, Tuple\n",
|
||||
"import urllib.request\n",
|
||||
"from itertools import chain\n",
|
||||
"import gzip\n",
|
||||
"from great_ai.utilities.clean import clean\n",
|
||||
"from great_ai.utilities.parallel_map import parallel_map\n",
|
||||
"from great_ai.utilities.language import is_english, predict_language\n",
|
||||
"from great_ai import configure, LargeFile\n",
|
||||
"MAX_CHUNK_COUNT = 4"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Extract\n",
|
||||
"\n",
|
||||
"This can be achieved by downloading a public dataset (such as in this case), or by having a Data Engineer setup and give us access to the organisation's data.\n",
|
||||
"\n",
|
||||
"DATASET_KEY = \"semantic-scholar-dataset-small\"\n",
|
||||
"MAX_FILE_COUNT = 12\n",
|
||||
"configure()"
|
||||
"In this case, we download the semantic scholar dataset from a public S3 bucket."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'Processing 4 out of the 6002 available chunks'"
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Download the list of available chunks\n",
|
||||
"import urllib.request\n",
|
||||
"\n",
|
||||
"manifest = (\n",
|
||||
" urllib.request.urlopen(\n",
|
||||
|
|
@ -63,8 +56,22 @@
|
|||
" )\n",
|
||||
" .read()\n",
|
||||
" .decode()\n",
|
||||
")\n",
|
||||
"chunks = manifest.split()[:MAX_FILE_COUNT]"
|
||||
") # a list of available chunks separated by '\\n' characters\n",
|
||||
"\n",
|
||||
"chunks = manifest.split()[:MAX_CHUNK_COUNT]\n",
|
||||
"\n",
|
||||
"f\"Processing {len(chunks)} out of the {len(manifest.split())} available chunks\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Transform\n",
|
||||
"\n",
|
||||
"- Filter out non-English abstracts using `great_ai.utilities.predict_language`\n",
|
||||
"- Project it to only keep the necessary components (text and labels), clean the textual content using `great_ai.utilities.clean`\n",
|
||||
"- We will speed up processing using `great_ai.utilities.parallel_map`."
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -76,18 +83,19 @@
|
|||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\u001b[38;5;39m2022-05-27 12:12:28,639 | INFO | Starting parallel map (concurrency: 12, chunk size: 1)\u001b[0m\n"
|
||||
"\u001b[38;5;226m2022-06-19 14:59:12,562 | WARNING | Limiting concurrency to 4 because there are only 4 chunks\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-06-19 14:59:12,563 | INFO | Starting parallel map (concurrency: 4, chunk size: 1)\u001b[0m\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "7a381a677ed240e08c1605cd0917616a",
|
||||
"model_id": "ff8fc113515944cfa75127f4aba3d491",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/12 [00:00<?, ?it/s]"
|
||||
" 0%| | 0/4 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
|
|
@ -95,20 +103,25 @@
|
|||
}
|
||||
],
|
||||
"source": [
|
||||
"# Extract, Transform\n",
|
||||
"from typing import List, Tuple\n",
|
||||
"import json\n",
|
||||
"import gzip\n",
|
||||
"from great_ai import parallel_map, clean, is_english, predict_language\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def process_chunk(chunk_key: str) -> List[Tuple[str, List[str]]]:\n",
|
||||
"def preprocess_chunk(chunk_key: str) -> List[Tuple[str, List[str]]]:\n",
|
||||
" # Extract\n",
|
||||
" response = urllib.request.urlopen(\n",
|
||||
" \"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/\"\n",
|
||||
" + chunk_key\n",
|
||||
" ) # returns a gzipped JSON Lines file\n",
|
||||
" f\"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/{chunk_key}\"\n",
|
||||
" ) # a gzipped JSON Lines file\n",
|
||||
"\n",
|
||||
" decompressed = gzip.decompress(response.read())\n",
|
||||
" decoded = decompressed.decode()\n",
|
||||
" chunk = [json.loads(line) for line in decoded.split(\"\\n\") if line]\n",
|
||||
"\n",
|
||||
" all = [\n",
|
||||
" # Create pairs of `(text, [domains])`\n",
|
||||
" # Transform\n",
|
||||
" return [\n",
|
||||
" # Create pairs of `(text, [...domains])`\n",
|
||||
" # The text is cleaned to remove PDF extraction, web scraping, and other common artifacts\n",
|
||||
" (\n",
|
||||
" clean(\n",
|
||||
|
|
@ -118,61 +131,79 @@
|
|||
" c[\"fieldsOfStudy\"],\n",
|
||||
" )\n",
|
||||
" for c in chunk\n",
|
||||
" ]\n",
|
||||
"\n",
|
||||
" return [\n",
|
||||
" (content, domains)\n",
|
||||
" for content, domains in all\n",
|
||||
" if (domains and content and is_english(predict_language(content)))\n",
|
||||
" if c[\"fieldsOfStudy\"] and is_english(predict_language(c[\"paperAbstract\"]))\n",
|
||||
" ]\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"clean_chunks = parallel_map(process_chunk, chunks)"
|
||||
"preprocessed_chunks = parallel_map(preprocess_chunk, chunks)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from itertools import chain\n",
|
||||
"\n",
|
||||
"preprocessed_data = list(chain(*preprocessed_chunks))\n",
|
||||
"X, y = zip(\n",
|
||||
" *preprocessed_data\n",
|
||||
") # X is the input, y is the expected (ground truth) output"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Load\n",
|
||||
"\n",
|
||||
"Upload the dataset (or a part of it) to a central repository using `great_ai.add_ground_truth`. This step automatically tags each datapoint with a split label according to the ratios we set. Additional tags can be also given.\n",
|
||||
"\n",
|
||||
"#### Use a different repository\n",
|
||||
"\n",
|
||||
"For the sake of simplicity, the tutorial uses the local hard drive (`great_ai.ParallelTinyDbDriver`) as the central repository.\n",
|
||||
"This can be simply changed, for example, by the following snippet:\n",
|
||||
"\n",
|
||||
"```python\n",
|
||||
"from great_ai import configure, MongoDbDriver\n",
|
||||
"\n",
|
||||
"configure(tracing_database=MongoDbDriver('mongodb://localhost:27017_or_something_like_that'))\n",
|
||||
"```"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\u001b[38;5;39m2022-05-27 12:22:35,833 | INFO | Fetching online versions of semantic-scholar-dataset-small\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 12:22:36,133 | INFO | Found versions: [2, 3]\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 12:22:39,542 | INFO | Copying file for semantic-scholar-dataset-small-4\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 12:22:39,843 | INFO | Compressing semantic-scholar-dataset-small-4\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 12:22:54,990 | INFO | Uploading semantic-scholar-dataset-small-4 to S3 from /tmp/large-file-l9z1mo61\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 12:23:19,988 | INFO | Uploading semantic-scholar-dataset-small-4.tar.gz 8.91/88.72 MB (10.0%)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 12:23:41,914 | INFO | Uploading semantic-scholar-dataset-small-4.tar.gz 17.83/88.72 MB (20.1%)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 12:24:06,996 | INFO | Uploading semantic-scholar-dataset-small-4.tar.gz 26.74/88.72 MB (30.1%)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 12:24:30,563 | INFO | Uploading semantic-scholar-dataset-small-4.tar.gz 35.65/88.72 MB (40.2%)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 12:24:59,623 | INFO | Uploading semantic-scholar-dataset-small-4.tar.gz 44.56/88.72 MB (50.2%)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 12:25:28,237 | INFO | Uploading semantic-scholar-dataset-small-4.tar.gz 53.48/88.72 MB (60.3%)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 12:25:59,432 | INFO | Uploading semantic-scholar-dataset-small-4.tar.gz 62.13/88.72 MB (70.0%)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 12:26:25,016 | INFO | Uploading semantic-scholar-dataset-small-4.tar.gz 71.04/88.72 MB (80.1%)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 12:26:55,491 | INFO | Uploading semantic-scholar-dataset-small-4.tar.gz 79.95/88.72 MB (90.1%)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 12:27:26,801 | INFO | Uploading semantic-scholar-dataset-small-4.tar.gz 88.72/88.72 MB (100.0%)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 12:27:28,540 | INFO | Removing old version (keep_last_n=1): semantic-scholar-dataset-small/2\u001b[0m\n"
|
||||
"\u001b[38;5;226m2022-06-19 15:03:30,300 | WARNING | Environment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
|
||||
"\u001b[38;5;226m2022-06-19 15:03:30,301 | WARNING | The selected persistence driver (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-06-19 15:03:30,301 | INFO | Options: configured ✅\u001b[0m\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Load\n",
|
||||
"# Using the LargeFile GreatAI utility, the data will be made available on S3 (and in our local cache)\n",
|
||||
"from great_ai import add_ground_truth\n",
|
||||
"\n",
|
||||
"with LargeFile(DATASET_KEY, \"w\", keep_last_n=1) as f:\n",
|
||||
" json.dump(list(chain(*clean_chunks)), f)"
|
||||
"add_ground_truth(X, y, train_split_ratio=0.8, test_split_ratio=0.2)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Next: [Part 2](train.ipynb)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"interpreter": {
|
||||
"hash": "9583b8c15346c7ad861da481b60c8e8605a71a48eda767f1640baa8f25efebcb"
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3.9.2 ('.venv': venv)",
|
||||
"display_name": "Python 3.10.4 ('.env': venv)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
|
|
@ -186,9 +217,14 @@
|
|||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.9.2"
|
||||
"version": "3.10.4"
|
||||
},
|
||||
"orig_nbformat": 4
|
||||
"orig_nbformat": 4,
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "02dd6d3afbfa9fbbe1037d64ad9014965528a1ccad21929d6e72f466389a68ad"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
1
examples/simple/diagrams/scope-data.svg
Normal file
1
examples/simple/diagrams/scope-data.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 14 KiB |
1
examples/simple/diagrams/scope-deploy.svg
Normal file
1
examples/simple/diagrams/scope-deploy.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 14 KiB |
1
examples/simple/diagrams/scope-train.svg
Normal file
1
examples/simple/diagrams/scope-train.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 14 KiB |
1
examples/simple/diagrams/scope.svg
Normal file
1
examples/simple/diagrams/scope.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 14 KiB |
3
examples/simple/mongo.ini
Normal file
3
examples/simple/mongo.ini
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
[DEFAULT]
|
||||
connection_string=mongodb://localhost:27017/
|
||||
database=large_file_tests
|
||||
File diff suppressed because one or more lines are too long
Loading…
Add table
Add a link
Reference in a new issue