Integrate MongoDB
This commit is contained in:
parent
750ba7b0d4
commit
cf0ac4b161
30 changed files with 1024 additions and 769 deletions
|
|
@ -11,15 +11,6 @@
|
|||
"> The blue boxes show the steps implemented in this notebook."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"MAX_CHUNK_COUNT = 4"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
|
|
@ -31,6 +22,15 @@
|
|||
"In this case, we download the semantic scholar dataset from a public S3 bucket."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"MAX_CHUNK_COUNT = 1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
|
|
@ -39,7 +39,7 @@
|
|||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'Processing 4 out of the 6002 available chunks'"
|
||||
"'Processing 1 out of the 6002 available chunks'"
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
|
|
@ -49,6 +49,7 @@
|
|||
],
|
||||
"source": [
|
||||
"import urllib.request\n",
|
||||
"from random import shuffle\n",
|
||||
"\n",
|
||||
"manifest = (\n",
|
||||
" urllib.request.urlopen(\n",
|
||||
|
|
@ -58,7 +59,9 @@
|
|||
" .decode()\n",
|
||||
") # a list of available chunks separated by '\\n' characters\n",
|
||||
"\n",
|
||||
"chunks = manifest.split()[:MAX_CHUNK_COUNT]\n",
|
||||
"lines = manifest.split()\n",
|
||||
"shuffle(lines)\n",
|
||||
"chunks = lines[:MAX_CHUNK_COUNT]\n",
|
||||
"\n",
|
||||
"f\"Processing {len(chunks)} out of the {len(manifest.split())} available chunks\""
|
||||
]
|
||||
|
|
@ -83,23 +86,11 @@
|
|||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\u001b[38;5;226m2022-06-19 14:59:12,562 | WARNING | Limiting concurrency to 4 because there are only 4 chunks\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-06-19 14:59:12,563 | INFO | Starting parallel map (concurrency: 4, chunk size: 1)\u001b[0m\n"
|
||||
"\u001b[38;5;226m2022-06-25 11:20:01,955 | WARNING | Limiting concurrency to 1 because there are only 1 chunks\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-06-25 11:20:01,956 | INFO | Starting parallel map (concurrency: 1, chunk size: 1)\u001b[0m\n",
|
||||
"\u001b[38;5;226m2022-06-25 11:20:01,956 | WARNING | Running in series, there is no reason for parallelism\u001b[0m\n",
|
||||
"100%|██████████| 1/1 [04:02<00:00, 242.61s/it]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "ff8fc113515944cfa75127f4aba3d491",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/4 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
|
|
@ -121,15 +112,13 @@
|
|||
"\n",
|
||||
" # Transform\n",
|
||||
" return [\n",
|
||||
" # Create pairs of `(text, [...domains])`\n",
|
||||
" # The text is cleaned to remove PDF extraction, web scraping, and other common artifacts\n",
|
||||
" (\n",
|
||||
" clean(\n",
|
||||
" f'{c[\"title\"]} {c[\"paperAbstract\"]} {c[\"journalName\"]} {c[\"venue\"]}',\n",
|
||||
" convert_to_ascii=True,\n",
|
||||
" ),\n",
|
||||
" ), # The text is cleaned to remove PDF extraction, web scraping, and other common artifacts\n",
|
||||
" c[\"fieldsOfStudy\"],\n",
|
||||
" )\n",
|
||||
" ) # Create pairs of `(text, [...domains])`\n",
|
||||
" for c in chunk\n",
|
||||
" if c[\"fieldsOfStudy\"] and is_english(predict_language(c[\"paperAbstract\"]))\n",
|
||||
" ]\n",
|
||||
|
|
@ -160,16 +149,15 @@
|
|||
"\n",
|
||||
"Upload the dataset (or a part of it) to a central repository using `great_ai.add_ground_truth`. This step automatically tags each datapoint with a split label according to the ratios we set. Additional tags can be also given.\n",
|
||||
"\n",
|
||||
"#### Use a different repository\n",
|
||||
"#### Production-ready backend\n",
|
||||
"\n",
|
||||
"For the sake of simplicity, the tutorial uses the local hard drive (`great_ai.ParallelTinyDbDriver`) as the central repository.\n",
|
||||
"This can be simply changed, for example, by the following snippet:\n",
|
||||
"The MongoDB driver is automatically configured if `mongo.ini` exists with the following scheme:\n",
|
||||
"\n",
|
||||
"```python\n",
|
||||
"from great_ai import configure, MongoDbDriver\n",
|
||||
"\n",
|
||||
"configure(tracing_database=MongoDbDriver('mongodb://localhost:27017_or_something_like_that'))\n",
|
||||
"```"
|
||||
"```ini\n",
|
||||
"mongo_connection_string=mongodb://localhost:27017/\n",
|
||||
"mongo_database=my_great_ai_db\n",
|
||||
"```\n",
|
||||
"> You can install MongoDB from [here](https://www.mongodb.com/docs/manual/installation) or [use it as a service](https://www.mongodb.com/cloud/atlas/register)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -181,9 +169,17 @@
|
|||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\u001b[38;5;226m2022-06-19 15:03:30,300 | WARNING | Environment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
|
||||
"\u001b[38;5;226m2022-06-19 15:03:30,301 | WARNING | The selected persistence driver (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-06-19 15:03:30,301 | INFO | Options: configured ✅\u001b[0m\n"
|
||||
"\u001b[38;5;226m2022-06-25 11:24:04,668 | WARNING | Environment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-06-25 11:24:04,669 | INFO | Found credentials file (/data/projects/great-ai/examples/simple/mongo.ini), initialising MongodbDriver\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-06-25 11:24:04,670 | INFO | Found credentials file (/data/projects/great-ai/examples/simple/mongo.ini), initialising LargeFileMongo\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-06-25 11:24:04,671 | INFO | Settings: configured ✅\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-06-25 11:24:04,672 | INFO | 🔩 tracing_database: MongodbDriver\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-06-25 11:24:04,672 | INFO | 🔩 large_file_implementation: LargeFileMongo\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-06-25 11:24:04,673 | INFO | 🔩 is_production: False\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-06-25 11:24:04,673 | INFO | 🔩 should_log_exception_stack: True\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-06-25 11:24:04,674 | INFO | 🔩 prediction_cache_size: 512\u001b[0m\n",
|
||||
"\u001b[38;5;226m2022-06-25 11:24:04,674 | WARNING | You still need to check whether you follow all best practices before trusting your deployment.\u001b[0m\n",
|
||||
"\u001b[38;5;226m2022-06-25 11:24:04,674 | WARNING | > Find out more at https://se-ml.github.io/practices/\u001b[0m\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
BIN
examples/simple/diagrams/ss-confusion.png
Normal file
BIN
examples/simple/diagrams/ss-confusion.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 160 KiB |
BIN
examples/simple/diagrams/ss-distribution.png
Normal file
BIN
examples/simple/diagrams/ss-distribution.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 64 KiB |
|
|
@ -1,3 +1,2 @@
|
|||
[DEFAULT]
|
||||
connection_string=mongodb://localhost:27017/
|
||||
database=large_file_tests
|
||||
mongo_connection_string=mongodb://localhost:27017/ # change this
|
||||
mongo_database=great_ai_db2 # this will be automatically created
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
Loading…
Add table
Add a link
Reference in a new issue