Streamline example
This commit is contained in:
parent
c9d0cdb2df
commit
78ef39c9b6
13 changed files with 1255 additions and 524 deletions
23
.vscode/settings.json
vendored
23
.vscode/settings.json
vendored
|
|
@ -1,18 +1,30 @@
|
|||
{
|
||||
"cSpell.words": [
|
||||
"basereload",
|
||||
"boto",
|
||||
"botocore",
|
||||
"datatable",
|
||||
"displaylogo",
|
||||
"downsample",
|
||||
"fastapi",
|
||||
"finalise",
|
||||
"finalised",
|
||||
"gridfs",
|
||||
"iloc",
|
||||
"initialisation",
|
||||
"initialised",
|
||||
"inplace",
|
||||
"ipynb",
|
||||
"lemmatize",
|
||||
"levelname",
|
||||
"levelno",
|
||||
"matplotlib",
|
||||
"Multinomial",
|
||||
"multiprocess",
|
||||
"nbconvert",
|
||||
"nbins",
|
||||
"optimisation",
|
||||
"organisation's",
|
||||
"Parcoords",
|
||||
"plotly",
|
||||
"proba",
|
||||
|
|
@ -21,23 +33,28 @@
|
|||
"pymongo",
|
||||
"pyplot",
|
||||
"redoc",
|
||||
"serialise",
|
||||
"sklearn",
|
||||
"starlette",
|
||||
"sublinear",
|
||||
"Tfidf",
|
||||
"threadsafe",
|
||||
"ticktext",
|
||||
"tickvals",
|
||||
"tinydb",
|
||||
"tqdm",
|
||||
"uvicorn",
|
||||
"Vectorizer",
|
||||
"xmargin"
|
||||
"xmargin",
|
||||
"xticks"
|
||||
],
|
||||
"files.exclude": {
|
||||
".env": true,
|
||||
".venv": true,
|
||||
"**/.cache": true,
|
||||
"**/__pycache__": true,
|
||||
"**/.ipynb_checkpoints": true,
|
||||
"**/.mypy_cache": true,
|
||||
"**/.pytest_cache": true
|
||||
}
|
||||
},
|
||||
"notebook.output.textLineLimit": 400
|
||||
}
|
||||
|
|
|
|||
8
.vscode/tasks.json
vendored
8
.vscode/tasks.json
vendored
|
|
@ -4,9 +4,9 @@
|
|||
{
|
||||
"label": "Format and lint Python",
|
||||
"type": "shell",
|
||||
"command": "source .venv/bin/activate && scripts/format-python.sh great_ai && scripts/format-python.sh examples/simple",
|
||||
"command": "source .env/bin/activate && scripts/format-python.sh great_ai && scripts/format-python.sh examples/simple",
|
||||
"windows": {
|
||||
"command": ".venv\\bin\\activate.bat; scripts\\format-python.sh great_ai; scripts\\format-python.sh examples\\simple"
|
||||
"command": ".env\\bin\\activate.bat; scripts\\format-python.sh great_ai; scripts\\format-python.sh examples\\simple"
|
||||
},
|
||||
"group": "test",
|
||||
"presentation": {
|
||||
|
|
@ -20,9 +20,9 @@
|
|||
{
|
||||
"label": "Test Python",
|
||||
"type": "shell",
|
||||
"command": "source .venv/bin/activate && python3 -m pytest great_ai",
|
||||
"command": "source .env/bin/activate && python3 -m pytest great_ai",
|
||||
"windows": {
|
||||
"command": ".venv\\bin\\activate.bat; python3 -m pytest great_ai"
|
||||
"command": ".env\\bin\\activate.bat; python3 -m pytest great_ai"
|
||||
},
|
||||
"group": "test",
|
||||
"presentation": {
|
||||
|
|
|
|||
7
examples/simple/README.md
Normal file
7
examples/simple/README.md
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
# Train a domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)
|
||||
|
||||
|
||||
|
||||
- [Part 1](data.ipynb)
|
||||
- [Part 2](train.ipynb)
|
||||
- [Part 3](deploy.ipynb)
|
||||
|
|
@ -5,57 +5,50 @@
|
|||
"metadata": {},
|
||||
"source": [
|
||||
"# Train a domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)\n",
|
||||
"> Part 1: obtain and clean data\n",
|
||||
"\n",
|
||||
"## Part 1: obtain clean data\n",
|
||||
"\n",
|
||||
"This can be achieved by downloading a public dataset (such as in this case), or by having a Data Engineer setup and give us access to the organisations's data.\n",
|
||||
"\n",
|
||||
"1. **Extract**: Download the semantic scholar dataset from a public S3 bucket\n",
|
||||
"2. **Transform**: Project it to only keep the necessary components (text and labels), clean the textual content using `great_ai.utilities.clean`\n",
|
||||
"3. **Load**: Upload the dataset (or a part of it) to a shared infrastructure using `great_ai.LargeFile` with its S3 backend\n",
|
||||
"\n",
|
||||
"> We will speed up the processing using `great_ai.utilities.parallel_map`"
|
||||
"\n",
|
||||
"> The blue boxes show the steps implemented in this notebook."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\u001b[38;5;39m2022-05-27 12:12:27,324 | INFO | Environment variable ENVIRONMENT is not set, defaulting to development mode\u001b[0m\n",
|
||||
"\u001b[38;5;226m2022-05-27 12:12:27,324 | WARNING | Running in development mode ‼️\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 12:12:27,325 | INFO | Options: configured ✅\u001b[0m\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import json\n",
|
||||
"from typing import List, Tuple\n",
|
||||
"import urllib.request\n",
|
||||
"from itertools import chain\n",
|
||||
"import gzip\n",
|
||||
"from great_ai.utilities.clean import clean\n",
|
||||
"from great_ai.utilities.parallel_map import parallel_map\n",
|
||||
"from great_ai.utilities.language import is_english, predict_language\n",
|
||||
"from great_ai import configure, LargeFile\n",
|
||||
"MAX_CHUNK_COUNT = 4"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Extract\n",
|
||||
"\n",
|
||||
"This can be achieved by downloading a public dataset (such as in this case), or by having a Data Engineer setup and give us access to the organisation's data.\n",
|
||||
"\n",
|
||||
"DATASET_KEY = \"semantic-scholar-dataset-small\"\n",
|
||||
"MAX_FILE_COUNT = 12\n",
|
||||
"configure()"
|
||||
"In this case, we download the semantic scholar dataset from a public S3 bucket."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'Processing 4 out of the 6002 available chunks'"
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Download the list of available chunks\n",
|
||||
"import urllib.request\n",
|
||||
"\n",
|
||||
"manifest = (\n",
|
||||
" urllib.request.urlopen(\n",
|
||||
|
|
@ -63,8 +56,22 @@
|
|||
" )\n",
|
||||
" .read()\n",
|
||||
" .decode()\n",
|
||||
")\n",
|
||||
"chunks = manifest.split()[:MAX_FILE_COUNT]"
|
||||
") # a list of available chunks separated by '\\n' characters\n",
|
||||
"\n",
|
||||
"chunks = manifest.split()[:MAX_CHUNK_COUNT]\n",
|
||||
"\n",
|
||||
"f\"Processing {len(chunks)} out of the {len(manifest.split())} available chunks\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Transform\n",
|
||||
"\n",
|
||||
"- Filter out non-English abstracts using `great_ai.utilities.predict_language`\n",
|
||||
"- Project it to only keep the necessary components (text and labels), clean the textual content using `great_ai.utilities.clean`\n",
|
||||
"- We will speed up processing using `great_ai.utilities.parallel_map`."
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -76,18 +83,19 @@
|
|||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\u001b[38;5;39m2022-05-27 12:12:28,639 | INFO | Starting parallel map (concurrency: 12, chunk size: 1)\u001b[0m\n"
|
||||
"\u001b[38;5;226m2022-06-19 14:59:12,562 | WARNING | Limiting concurrency to 4 because there are only 4 chunks\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-06-19 14:59:12,563 | INFO | Starting parallel map (concurrency: 4, chunk size: 1)\u001b[0m\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "7a381a677ed240e08c1605cd0917616a",
|
||||
"model_id": "ff8fc113515944cfa75127f4aba3d491",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/12 [00:00<?, ?it/s]"
|
||||
" 0%| | 0/4 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
|
|
@ -95,20 +103,25 @@
|
|||
}
|
||||
],
|
||||
"source": [
|
||||
"# Extract, Transform\n",
|
||||
"from typing import List, Tuple\n",
|
||||
"import json\n",
|
||||
"import gzip\n",
|
||||
"from great_ai import parallel_map, clean, is_english, predict_language\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def process_chunk(chunk_key: str) -> List[Tuple[str, List[str]]]:\n",
|
||||
"def preprocess_chunk(chunk_key: str) -> List[Tuple[str, List[str]]]:\n",
|
||||
" # Extract\n",
|
||||
" response = urllib.request.urlopen(\n",
|
||||
" \"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/\"\n",
|
||||
" + chunk_key\n",
|
||||
" ) # returns a gzipped JSON Lines file\n",
|
||||
" f\"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/{chunk_key}\"\n",
|
||||
" ) # a gzipped JSON Lines file\n",
|
||||
"\n",
|
||||
" decompressed = gzip.decompress(response.read())\n",
|
||||
" decoded = decompressed.decode()\n",
|
||||
" chunk = [json.loads(line) for line in decoded.split(\"\\n\") if line]\n",
|
||||
"\n",
|
||||
" all = [\n",
|
||||
" # Create pairs of `(text, [domains])`\n",
|
||||
" # Transform\n",
|
||||
" return [\n",
|
||||
" # Create pairs of `(text, [...domains])`\n",
|
||||
" # The text is cleaned to remove PDF extraction, web scraping, and other common artifacts\n",
|
||||
" (\n",
|
||||
" clean(\n",
|
||||
|
|
@ -118,61 +131,79 @@
|
|||
" c[\"fieldsOfStudy\"],\n",
|
||||
" )\n",
|
||||
" for c in chunk\n",
|
||||
" ]\n",
|
||||
"\n",
|
||||
" return [\n",
|
||||
" (content, domains)\n",
|
||||
" for content, domains in all\n",
|
||||
" if (domains and content and is_english(predict_language(content)))\n",
|
||||
" if c[\"fieldsOfStudy\"] and is_english(predict_language(c[\"paperAbstract\"]))\n",
|
||||
" ]\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"clean_chunks = parallel_map(process_chunk, chunks)"
|
||||
"preprocessed_chunks = parallel_map(preprocess_chunk, chunks)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from itertools import chain\n",
|
||||
"\n",
|
||||
"preprocessed_data = list(chain(*preprocessed_chunks))\n",
|
||||
"X, y = zip(\n",
|
||||
" *preprocessed_data\n",
|
||||
") # X is the input, y is the expected (ground truth) output"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Load\n",
|
||||
"\n",
|
||||
"Upload the dataset (or a part of it) to a central repository using `great_ai.add_ground_truth`. This step automatically tags each datapoint with a split label according to the ratios we set. Additional tags can be also given.\n",
|
||||
"\n",
|
||||
"#### Use a different repository\n",
|
||||
"\n",
|
||||
"For the sake of simplicity, the tutorial uses the local hard drive (`great_ai.ParallelTinyDbDriver`) as the central repository.\n",
|
||||
"This can be simply changed, for example, by the following snippet:\n",
|
||||
"\n",
|
||||
"```python\n",
|
||||
"from great_ai import configure, MongoDbDriver\n",
|
||||
"\n",
|
||||
"configure(tracing_database=MongoDbDriver('mongodb://localhost:27017_or_something_like_that'))\n",
|
||||
"```"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\u001b[38;5;39m2022-05-27 12:22:35,833 | INFO | Fetching online versions of semantic-scholar-dataset-small\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 12:22:36,133 | INFO | Found versions: [2, 3]\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 12:22:39,542 | INFO | Copying file for semantic-scholar-dataset-small-4\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 12:22:39,843 | INFO | Compressing semantic-scholar-dataset-small-4\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 12:22:54,990 | INFO | Uploading semantic-scholar-dataset-small-4 to S3 from /tmp/large-file-l9z1mo61\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 12:23:19,988 | INFO | Uploading semantic-scholar-dataset-small-4.tar.gz 8.91/88.72 MB (10.0%)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 12:23:41,914 | INFO | Uploading semantic-scholar-dataset-small-4.tar.gz 17.83/88.72 MB (20.1%)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 12:24:06,996 | INFO | Uploading semantic-scholar-dataset-small-4.tar.gz 26.74/88.72 MB (30.1%)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 12:24:30,563 | INFO | Uploading semantic-scholar-dataset-small-4.tar.gz 35.65/88.72 MB (40.2%)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 12:24:59,623 | INFO | Uploading semantic-scholar-dataset-small-4.tar.gz 44.56/88.72 MB (50.2%)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 12:25:28,237 | INFO | Uploading semantic-scholar-dataset-small-4.tar.gz 53.48/88.72 MB (60.3%)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 12:25:59,432 | INFO | Uploading semantic-scholar-dataset-small-4.tar.gz 62.13/88.72 MB (70.0%)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 12:26:25,016 | INFO | Uploading semantic-scholar-dataset-small-4.tar.gz 71.04/88.72 MB (80.1%)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 12:26:55,491 | INFO | Uploading semantic-scholar-dataset-small-4.tar.gz 79.95/88.72 MB (90.1%)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 12:27:26,801 | INFO | Uploading semantic-scholar-dataset-small-4.tar.gz 88.72/88.72 MB (100.0%)\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-05-27 12:27:28,540 | INFO | Removing old version (keep_last_n=1): semantic-scholar-dataset-small/2\u001b[0m\n"
|
||||
"\u001b[38;5;226m2022-06-19 15:03:30,300 | WARNING | Environment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
|
||||
"\u001b[38;5;226m2022-06-19 15:03:30,301 | WARNING | The selected persistence driver (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n",
|
||||
"\u001b[38;5;39m2022-06-19 15:03:30,301 | INFO | Options: configured ✅\u001b[0m\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Load\n",
|
||||
"# Using the LargeFile GreatAI utility, the data will be made available on S3 (and in our local cache)\n",
|
||||
"from great_ai import add_ground_truth\n",
|
||||
"\n",
|
||||
"with LargeFile(DATASET_KEY, \"w\", keep_last_n=1) as f:\n",
|
||||
" json.dump(list(chain(*clean_chunks)), f)"
|
||||
"add_ground_truth(X, y, train_split_ratio=0.8, test_split_ratio=0.2)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Next: [Part 2](train.ipynb)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"interpreter": {
|
||||
"hash": "9583b8c15346c7ad861da481b60c8e8605a71a48eda767f1640baa8f25efebcb"
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3.9.2 ('.venv': venv)",
|
||||
"display_name": "Python 3.10.4 ('.env': venv)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
|
|
@ -186,9 +217,14 @@
|
|||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.9.2"
|
||||
"version": "3.10.4"
|
||||
},
|
||||
"orig_nbformat": 4
|
||||
"orig_nbformat": 4,
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "02dd6d3afbfa9fbbe1037d64ad9014965528a1ccad21929d6e72f466389a68ad"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
1
examples/simple/diagrams/scope-data.svg
Normal file
1
examples/simple/diagrams/scope-data.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 14 KiB |
1
examples/simple/diagrams/scope-deploy.svg
Normal file
1
examples/simple/diagrams/scope-deploy.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 14 KiB |
1
examples/simple/diagrams/scope-train.svg
Normal file
1
examples/simple/diagrams/scope-train.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 14 KiB |
1
examples/simple/diagrams/scope.svg
Normal file
1
examples/simple/diagrams/scope.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 14 KiB |
File diff suppressed because one or more lines are too long
212
scope-data.drawio.svg
Normal file
212
scope-data.drawio.svg
Normal file
|
|
@ -0,0 +1,212 @@
|
|||
<svg host="65bd71144e" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" width="1181px" height="281px" viewBox="-0.5 -0.5 1181 281" content="<mxfile><diagram id="ccudx8biEBM-x_P_p45R" name="Page-1">1VrbcuI4EP0aqpIHUr5gA49sYGa3KqlJFbs1k0fZFkYV2/JIIsH79duy5bsJBhySzUMstaS2pD59uiUzMu/D/XeG4u0j9XAwMjRvPzKXI8MwdMuAh5QkmcSaK4HPiJeJ9FKwJv9iJdSUdEc8zGsdBaWBIHFd6NIowq6oyRBj9K3ebUOD+ltj5OOWYO2ioC39STyxzaQzY1rK/8TE3+Zv1u151hKivLNaCd8ij75VROZqZN4zSkVWCvf3OJCbl+9LNu7bgdZiYgxHos8ANWMuknxt2IOlqmpEI3j8sRVhADUdihsaCWUPfQL19vvUFDjdMVepVMYViPlY9ZpkIvmyyjA1x++YhliwBDowHCBBXuv7jpT5/KJfuUIoqEV2L1jN5RUFO6V0ZNiBUEurbYX9e0fzhjFPF72ADvok3peNUPLl84nhmFEXc449iWwkUK4YppTpznq2dhwwEMuimwQk8jAzodPblgi8jlG6hW/gRXUzOHQHPb0HpxAg98VnUvpjJ0ANVnKubGW9Z6tXzATev2sH1WrO7WyIclwzx/Fb6QY6vCyVbasuoGmX284+EaywTpb8goqWV55l5c7Kq8t9tXGZVGtPmBGYIWZKeBbyzTbyjc9CvnkQ+U4JzSFcYZliX0Pu7x3hRBAaHfSEQtyagsNyyY3St4p8wDVYJfJvD/oQ2AMFAQ6oz1AIg+OKFWttFfMec7YN2eM8COnD+JE+0+p+ZHb4Ud6n6kezAdxodg3On7SRb30W8ifXQv5qHweUoQzyYCkbhRJL2f82sosOkcPlI1SJkrMjgQcwH8Rt1i7BkYs/12U2JAjuKexN+mpTd5CODbWrFfkm/ZNhSzD6gistmmavFt+GcT7LagSxLuezP8j5dPNKQazptR7iW+zlSvdE/CrVQO250lJqlJWkUmkGxV5UYLWpYD40FaRDF4yhpNIhpiQSvKL5SQpKIMyMOgtPZlrDlJnG0rDF1HrZ2vqgXPNvhiASyjQzZQze5JEu7tEgOxVUJODyMMFog5lkhfSM5OGvlql+KbqYTqbHc97JR+W8+uTr8oV2Nb7ID/4DE0aLEaZ2gxGMhgmzeapR1XN1Q5Gl1xVZzdiRra6l6FIKM+xhKUy3j+dOSwxZT5I6BPiDhCqKPMS84wkPZT6KCE8TJiRfCzPbUo/3y3BOJlOji0ybc7pZJ3zhhQQyONgd7UfMs8ISv0L59lSqvEp21TdBP+GAok+OJ0kFOQ2dJBWscO4RpZO88nKFDw+T17lXB+eSnj7vYD37Oqw3s+okMm0Grt6s1zjW2no/1juDmDov8hqYkWlG6tn0ATk4x0p/7ygukJGTq9S6DaBGzZts3Ok0HSeLQbxmfpyq/+Ey5UPydLr4C/55KXWHODVim2HXCK3hrU/pQ7smK988PqweM+JdvLwQmhWxcO9u29nu/5GRC/wep+QcTB15ZyeY7CHSTu09MF1wcIn4hrIwvdyAY4igx5MEOLh4O1emCGOGkSd3hGP2SlzcD5AApSxwfxE8GJfi4Sr2v/SS8PLrhiICl0H3udp2UQQuosWREPwxFxWtwDmeWt3Z1qkReKw3FPU8d5wTgbs+KJyMmfzQ7uEN2gUlcTd2/ALrXeGCWT8cei+76VkwQTbIFWkqEjokyohTbGUYz6+L1bFLxvKEqljuYC7GMYOhwJP81BB5/S+RJ9Bf/iuC6V0d6vOOaxnro65liu/3p1/LnEhnfb5FnsOvPb1Ka3uVPjgpHth1qJa/hMgYqvw9ibn6Dw==</diagram></mxfile>">
|
||||
<defs/>
|
||||
<g>
|
||||
<path d="M 331 140 L 379.63 140" fill="none" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="stroke"/>
|
||||
<path d="M 384.88 140 L 377.88 143.5 L 379.63 140 L 377.88 136.5 Z" fill="rgb(0, 0, 0)" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="all"/>
|
||||
<path d="M 216 105 C 216 96.72 241.74 90 273.5 90 C 288.75 90 303.38 91.58 314.16 94.39 C 324.94 97.21 331 101.02 331 105 L 331 175 C 331 183.28 305.26 190 273.5 190 C 241.74 190 216 183.28 216 175 Z" fill="rgb(255, 255, 255)" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="all"/>
|
||||
<path d="M 331 105 C 331 113.28 305.26 120 273.5 120 C 241.74 120 216 113.28 216 105" fill="none" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="all"/>
|
||||
<g transform="translate(-0.5 -0.5)">
|
||||
<switch>
|
||||
<foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
|
||||
<div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 113px; height: 1px; padding-top: 153px; margin-left: 217px;">
|
||||
<div data-drawio-colors="color: rgb(0, 0, 0); " style="box-sizing: border-box; font-size: 0px; text-align: center;">
|
||||
<div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;">
|
||||
<font style="font-size: 14px">
|
||||
Preprocessed data
|
||||
</font>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</foreignObject>
|
||||
<text x="274" y="156" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle">
|
||||
Preprocessed data
|
||||
</text>
|
||||
</switch>
|
||||
</g>
|
||||
<path d="M 170 140 L 209.63 140" fill="none" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="stroke"/>
|
||||
<path d="M 214.88 140 L 207.88 143.5 L 209.63 140 L 207.88 136.5 Z" fill="rgb(0, 0, 0)" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="all"/>
|
||||
<path d="M 0 180 L 20 100 L 180 100 L 160 180 Z" fill="rgb(255, 255, 255)" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="all"/>
|
||||
<g transform="translate(-0.5 -0.5)">
|
||||
<switch>
|
||||
<foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
|
||||
<div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 178px; height: 1px; padding-top: 140px; margin-left: 1px;">
|
||||
<div data-drawio-colors="color: rgb(0, 0, 0); " style="box-sizing: border-box; font-size: 0px; text-align: center;">
|
||||
<div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;">
|
||||
<b>
|
||||
<font style="font-size: 14px">
|
||||
Data acquisition
|
||||
</font>
|
||||
</b>
|
||||
<br/>
|
||||
(Data Engineering)
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</foreignObject>
|
||||
<text x="90" y="144" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle">
|
||||
Data acquisition...
|
||||
</text>
|
||||
</switch>
|
||||
</g>
|
||||
<path d="M 526 140 L 560.63 140" fill="none" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="stroke"/>
|
||||
<path d="M 565.88 140 L 558.88 143.5 L 560.63 140 L 558.88 136.5 Z" fill="rgb(0, 0, 0)" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="all"/>
|
||||
<path d="M 376 180 L 396 100 L 536 100 L 516 180 Z" fill="#1ba1e2" stroke="#006eaf" stroke-miterlimit="10" pointer-events="all"/>
|
||||
<g transform="translate(-0.5 -0.5)">
|
||||
<switch>
|
||||
<foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
|
||||
<div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 158px; height: 1px; padding-top: 140px; margin-left: 377px;">
|
||||
<div data-drawio-colors="color: #ffffff; " style="box-sizing: border-box; font-size: 0px; text-align: center;">
|
||||
<div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(255, 255, 255); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;">
|
||||
<b>
|
||||
<font style="font-size: 14px">
|
||||
Exploration &
|
||||
<br/>
|
||||
model building
|
||||
</font>
|
||||
</b>
|
||||
<br/>
|
||||
(Data Science)
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</foreignObject>
|
||||
<text x="456" y="144" fill="#ffffff" font-family="Helvetica" font-size="12px" text-anchor="middle">
|
||||
Exploration &...
|
||||
</text>
|
||||
</switch>
|
||||
</g>
|
||||
<path d="M 639.5 190 L 639.92 240 Q 640 250 650 250 L 743.63 250" fill="none" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" stroke-dasharray="3 3" pointer-events="stroke"/>
|
||||
<path d="M 748.88 250 L 741.88 253.5 L 743.63 250 L 741.88 246.5 Z" fill="rgb(0, 0, 0)" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="all"/>
|
||||
<path d="M 567 105 C 567 96.72 599.46 90 639.5 90 C 658.73 90 677.17 91.58 690.77 94.39 C 704.36 97.21 712 101.02 712 105 L 712 175 C 712 183.28 679.54 190 639.5 190 C 599.46 190 567 183.28 567 175 Z" fill="#1ba1e2" stroke="#006eaf" stroke-miterlimit="10" pointer-events="all"/>
|
||||
<path d="M 712 105 C 712 113.28 679.54 120 639.5 120 C 599.46 120 567 113.28 567 105" fill="none" stroke="#006eaf" stroke-miterlimit="10" pointer-events="all"/>
|
||||
<g transform="translate(-0.5 -0.5)">
|
||||
<switch>
|
||||
<foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
|
||||
<div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 143px; height: 1px; padding-top: 153px; margin-left: 568px;">
|
||||
<div data-drawio-colors="color: #ffffff; " style="box-sizing: border-box; font-size: 0px; text-align: center;">
|
||||
<div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(255, 255, 255); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;">
|
||||
<font style="font-size: 14px">
|
||||
Trained models & prototype inference code
|
||||
</font>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</foreignObject>
|
||||
<text x="640" y="156" fill="#ffffff" font-family="Helvetica" font-size="12px" text-anchor="middle">
|
||||
Trained models & protot...
|
||||
</text>
|
||||
</switch>
|
||||
</g>
|
||||
<path d="M 639.5 90 L 639.92 40 Q 640 30 650 30 L 743.63 30" fill="none" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" stroke-dasharray="3 3" pointer-events="stroke"/>
|
||||
<path d="M 748.88 30 L 741.88 33.5 L 743.63 30 L 741.88 26.5 Z" fill="rgb(0, 0, 0)" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="all"/>
|
||||
<path d="M 966 180 L 986 100 L 1180 100 L 1160 180 Z" fill="rgb(255, 255, 255)" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="all"/>
|
||||
<g transform="translate(-0.5 -0.5)">
|
||||
<switch>
|
||||
<foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
|
||||
<div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 212px; height: 1px; padding-top: 140px; margin-left: 967px;">
|
||||
<div data-drawio-colors="color: rgb(0, 0, 0); " style="box-sizing: border-box; font-size: 0px; text-align: center;">
|
||||
<div style="display: inline-block; font-size: 14px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;">
|
||||
<b>
|
||||
Deploy with standard
|
||||
<br/>
|
||||
organisational methods
|
||||
</b>
|
||||
<br/>
|
||||
<font style="font-size: 12px">
|
||||
(SysAdmin, Ops, DevOps)
|
||||
</font>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</foreignObject>
|
||||
<text x="1073" y="144" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="14px" text-anchor="middle">
|
||||
Deploy with standard...
|
||||
</text>
|
||||
</switch>
|
||||
</g>
|
||||
<path d="M 922.5 140 L 969.63 140" fill="none" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" stroke-dasharray="3 3" pointer-events="stroke"/>
|
||||
<path d="M 974.88 140 L 967.88 143.5 L 969.63 140 L 967.88 136.5 Z" fill="rgb(0, 0, 0)" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="all"/>
|
||||
<path d="M 740 280 L 760 220 L 950 220 L 930 280 Z" fill="rgb(255, 255, 255)" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="none"/>
|
||||
<g transform="translate(-0.5 -0.5)">
|
||||
<switch>
|
||||
<foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
|
||||
<div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 208px; height: 1px; padding-top: 250px; margin-left: 741px;">
|
||||
<div data-drawio-colors="color: rgb(0, 0, 0); " style="box-sizing: border-box; font-size: 0px; text-align: center;">
|
||||
<div style="display: inline-block; font-size: 14px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: none; white-space: normal; overflow-wrap: normal;">
|
||||
<b>
|
||||
Use an AI deployment
|
||||
<br/>
|
||||
SaaS/PaaS
|
||||
</b>
|
||||
<br/>
|
||||
<font style="font-size: 12px">
|
||||
(MLEM, Akkio, etc.)
|
||||
</font>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</foreignObject>
|
||||
<text x="845" y="254" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="14px" text-anchor="middle">
|
||||
Use an AI deployment...
|
||||
</text>
|
||||
</switch>
|
||||
</g>
|
||||
<path d="M 740 60 L 760 0 L 950 0 L 930 60 Z" fill="rgb(255, 255, 255)" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="none"/>
|
||||
<g transform="translate(-0.5 -0.5)">
|
||||
<switch>
|
||||
<foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
|
||||
<div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 208px; height: 1px; padding-top: 30px; margin-left: 741px;">
|
||||
<div data-drawio-colors="color: rgb(0, 0, 0); " style="box-sizing: border-box; font-size: 0px; text-align: center;">
|
||||
<div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: none; white-space: normal; overflow-wrap: normal;">
|
||||
<b style="font-size: 14px">
|
||||
Transforming into
|
||||
<br/>
|
||||
production-ready service
|
||||
</b>
|
||||
<br/>
|
||||
(MLOps)
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</foreignObject>
|
||||
<text x="845" y="34" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle">
|
||||
Transforming into...
|
||||
</text>
|
||||
</switch>
|
||||
</g>
|
||||
<path d="M 845 190 L 845 213.63" fill="none" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" stroke-dasharray="3 3" pointer-events="none"/>
|
||||
<path d="M 845 218.88 L 841.5 211.88 L 845 213.63 L 848.5 211.88 Z" fill="rgb(0, 0, 0)" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="none"/>
|
||||
<path d="M 845 190 L 845 213.63" fill="none" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="none"/>
|
||||
<path d="M 845 218.88 L 841.5 211.88 L 845 213.63 L 848.5 211.88 Z" fill="rgb(0, 0, 0)" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="none"/>
|
||||
<path d="M 767.5 105 C 767.5 96.72 802.2 90 845 90 C 865.55 90 885.27 91.58 899.8 94.39 C 914.33 97.21 922.5 101.02 922.5 105 L 922.5 175 C 922.5 183.28 887.8 190 845 190 C 802.2 190 767.5 183.28 767.5 175 Z" fill="rgb(255, 255, 255)" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="none"/>
|
||||
<path d="M 922.5 105 C 922.5 113.28 887.8 120 845 120 C 802.2 120 767.5 113.28 767.5 105" fill="none" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="none"/>
|
||||
<g transform="translate(-0.5 -0.5)">
|
||||
<switch>
|
||||
<foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
|
||||
<div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 153px; height: 1px; padding-top: 153px; margin-left: 769px;">
|
||||
<div data-drawio-colors="color: rgb(0, 0, 0); " style="box-sizing: border-box; font-size: 0px; text-align: center;">
|
||||
<div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: none; white-space: normal; overflow-wrap: normal;">
|
||||
<font style="font-size: 14px">
|
||||
Artifact combining the model with deplyoment best-practices
|
||||
</font>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</foreignObject>
|
||||
<text x="845" y="156" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle">
|
||||
Artifact combining the mod...
|
||||
</text>
|
||||
</switch>
|
||||
</g>
|
||||
<path d="M 845 60 L 845 83.63" fill="none" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="none"/>
|
||||
<path d="M 845 88.88 L 841.5 81.88 L 845 83.63 L 848.5 81.88 Z" fill="rgb(0, 0, 0)" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="none"/>
|
||||
</g>
|
||||
<switch>
|
||||
<g requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"/>
|
||||
<a transform="translate(0,-5)" xlink:href="https://www.diagrams.net/doc/faq/svg-export-text-problems" target="_blank">
|
||||
<text text-anchor="middle" font-size="10px" x="50%" y="100%">
|
||||
Viewer does not support full SVG 1.1
|
||||
</text>
|
||||
</a>
|
||||
</switch>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 19 KiB |
|
|
@ -1 +0,0 @@
|
|||
<mxfile host="app.diagrams.net" modified="2022-06-17T15:50:41.770Z" agent="5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36" version="20.0.1" etag="3pJNM2M7l2PHQ0unmgej" type="device"><diagram id="ccudx8biEBM-x_P_p45R" name="Page-1">1Vpbb+I4FP41SO0Do9xI4ZEFZnalVlOJXe300SQmWE3ijGMo2V+/x4mdOyVAoG0fID62T2yf7zsX04E5C/Y/GIo2T9TF/sDQ3P3AnA8Mw5hYBnwJSZJJdN00M4nHiCtlhWBJ/sNSqEnplrg4rgzklPqcRFWhQ8MQO7wiQ4zRt+qwNfWrb42QhxuCpYP8pvRf4vJNJh0bD4X8T0y8jXqzbk+yngCpwXIn8Qa59K0kMhcDc8Yo5dlTsJ9hX5yeOpds3vcDvfnCGA55lwlyxTFP1N6wC1uVzZCG8PXHhgc+tHR4XNOQS3voFrSb75NLiOmWOVKlNDdHzMNylJWJxMtK0+Qaf2AaYM4SGMCwjzjZVc8dSfN5+bhih/AgN9m+YbmWHfK3UukzwxGjDo5j7ApQIo4ahwJmisSjk/gkdDEzYetvG8LxMkLpLt8A6dWTWtEtjHQfV7kAOa8eE9KfWw5qsJTH8jhH7x3nDjOO9+8eley1FLQkuUzVfiuQqmtStimjVAkvOV77RDzBPlnyCxqaaryIxreRas735c55Um49Y0ZghZhJ4VngNJvgND4KnGYDnAPD9rkADzx4PN1TJhB7rRy0/XtLVccww9QUBuhWtC86lZZ5CnINOb+3JCac0FDphSVmqquvA3FjCSumJHdS3yL0ANdgldC7P8ghsAfyfexTj6EAJkclK1b6SuY9RrY12WMVJ/R+eGRoNR6ZLTwat/Bo3AONxrdwy1YT+aOPQr51K+Qv9pFPGcogD5ayUSCwlH02kZ0PCFex+ApkMrPaEt8FmPdCm6VDcOjgL0+Zkd2BMvaVKKObNwo9da65KN5gVyndE/6rUAOtl1JPoVE0klKjHso6EXjUJPCkbwKnU6eMoaQ0IKIk5HFJ87MQFEAY13IQa6zVTJlpLAybL62TrUcNb/E3QxB5RP6WMjSu87aN6xqkfZzyBCgGqsM1ZoKFadng4q+cAj7UediWAhrXSgF16/MSUbsZEVWp2jMTG1Sr2zpP/5WKbJ1yVrkSrCka6VVFo7pTznbXUHSpbzDsfn2Dbh9PJeYYkoAkJQTwQUAVhS5i7vH4T5mHQhKn+QMSr4WVbagbdwv4J6cwRlsKU1/T3Rzvfkbx/cFc5AOzis6JaXfvpuuWfTTNyL1Q32lGTv9zU/NWL6WeS47vsJc6t2Q+17vpkxb3Zt/GvY1HVW/xUI9Qnd3buKrI1ru5tzM8UMsdUwMzIguIupMhv8pEK6VBaz9vOWtS97KtHGlJAXohyeS4C/4nFpkWEkXY9C/4cFOXHODUZk3PuURoCW99Tr+0W3rbu6fHxdPAmImFvr4Smj1i7ny7byaZX9EB53A97oEVmFryyVYw2X2kk9p7YDr3GgDqhTBeUxakNTxk/5weD/5QL7hbR4T+IcPIFScSY7YjomjoVO8/PWZx+pPgwUj7fX9GfcrSpZj6CunYkONK8nX6J0oVzugrLvVomr2Yfr8UWTdB0qW3apdX+nnoLqL1S7nvotCdh5kjsfs6dwSNiDt8GLWnaaeG7qFeU9SxMjmneGhGrinjZI0cnsbgYEXCzGPwDVaXDUUdIYJYQmUQW+GYDyMGU8FBxAdJf+37hE9CcNlrVk05abmYsK51MZH/5nr6xcSJdO3y49Q5/qMj57Um5/XeSX/g1KFZ/HqdMbD4JwBz8T8=</diagram></mxfile>
|
||||
Loading…
Add table
Add a link
Reference in a new issue