Setup docs page
This commit is contained in:
parent
dadbe24c82
commit
c39feeee53
33 changed files with 2724 additions and 379 deletions
29
.github/workflows/docs.yaml
vendored
Normal file
29
.github/workflows/docs.yaml
vendored
Normal file
|
|
@ -0,0 +1,29 @@
|
||||||
|
name: Publish on PyPI
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
tags:
|
||||||
|
- '*'
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
publish:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v3
|
||||||
|
|
||||||
|
- name: Set up Python 3.9
|
||||||
|
uses: actions/setup-python@v2
|
||||||
|
with:
|
||||||
|
python-version: 3.9
|
||||||
|
|
||||||
|
- name: Install dependencies
|
||||||
|
run: pip install --upgrade './[dev]'
|
||||||
|
|
||||||
|
- name: Build documentation
|
||||||
|
run: mkdocs gh-deploy
|
||||||
|
|
||||||
|
- name: Publish distribution to PyPI
|
||||||
|
uses: pypa/gh-action-pypi-publish@master
|
||||||
|
with:
|
||||||
|
user: __token__
|
||||||
|
password: ${{ secrets.PYPI_API_TOKEN }}
|
||||||
2
.vscode/settings.json
vendored
2
.vscode/settings.json
vendored
|
|
@ -73,5 +73,5 @@
|
||||||
"python.linting.pylintEnabled": false,
|
"python.linting.pylintEnabled": false,
|
||||||
"python.linting.mypyEnabled": true,
|
"python.linting.mypyEnabled": true,
|
||||||
"python.testing.unittestEnabled": false,
|
"python.testing.unittestEnabled": false,
|
||||||
"python.testing.pytestEnabled": true
|
"python.testing.pytestEnabled": true,
|
||||||
}
|
}
|
||||||
|
|
|
||||||
10
README.md
10
README.md
|
|
@ -40,7 +40,15 @@ Find the dashboard at [http://localhost:6060](http://localhost:6060/dashboard/).
|
||||||
|
|
||||||
### Contribute
|
### Contribute
|
||||||
|
|
||||||
|
|
||||||
|
#### Install
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
pip install 'great-ai[dev]'
|
pip install 'great-ai[dev]'
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Documentation
|
||||||
|
|
||||||
|
```sh
|
||||||
|
mkdocs serve
|
||||||
```
|
```
|
||||||
0
docs/explanation.md
Normal file
0
docs/explanation.md
Normal file
6
docs/great_ai_example-main/.dockerignore
Normal file
6
docs/great_ai_example-main/.dockerignore
Normal file
|
|
@ -0,0 +1,6 @@
|
||||||
|
.venv
|
||||||
|
.env
|
||||||
|
**/.cache
|
||||||
|
.git
|
||||||
|
**/__pycache__
|
||||||
|
.dockerignore
|
||||||
10
docs/great_ai_example-main/.gitignore
vendored
Normal file
10
docs/great_ai_example-main/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
.env
|
||||||
|
.venv
|
||||||
|
.DS_Store
|
||||||
|
__pycache__
|
||||||
|
.cache
|
||||||
|
.mypy_cache
|
||||||
|
.pytest_cache
|
||||||
|
**/.ipynb_checkpoints
|
||||||
|
**/tracing_database.json
|
||||||
|
*.egg-info
|
||||||
10
docs/great_ai_example-main/.vscode/settings.json
vendored
Normal file
10
docs/great_ai_example-main/.vscode/settings.json
vendored
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
{
|
||||||
|
"files.exclude": {
|
||||||
|
"**/__pycache__": true,
|
||||||
|
"**/.ipynb_checkpoints": true,
|
||||||
|
"**/.mypy_cache": true,
|
||||||
|
"**/.pytest_cache": true
|
||||||
|
},
|
||||||
|
"notebook.output.textLineLimit": 400,
|
||||||
|
"python.defaultInterpreterPath": ".env/bin/python"
|
||||||
|
}
|
||||||
21
docs/great_ai_example-main/.vscode/tasks.json
vendored
Normal file
21
docs/great_ai_example-main/.vscode/tasks.json
vendored
Normal file
|
|
@ -0,0 +1,21 @@
|
||||||
|
{
|
||||||
|
"version": "2.0.0",
|
||||||
|
"tasks": [
|
||||||
|
{
|
||||||
|
"label": "Format and lint",
|
||||||
|
"type": "shell",
|
||||||
|
"command": "source .env/bin/activate; ./format.sh .",
|
||||||
|
"windows": {
|
||||||
|
"command": ".env\\bin\\activate.bat; .\\format.sh ."
|
||||||
|
},
|
||||||
|
"group": "test",
|
||||||
|
"presentation": {
|
||||||
|
"reveal": "always",
|
||||||
|
"panel": "new"
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
11
docs/great_ai_example-main/Dockerfile
Normal file
11
docs/great_ai_example-main/Dockerfile
Normal file
|
|
@ -0,0 +1,11 @@
|
||||||
|
FROM test
|
||||||
|
|
||||||
|
COPY requirements.txt .
|
||||||
|
RUN pip install --no-cache-dir ./great_ai
|
||||||
|
|
||||||
|
COPY mongo.ini .
|
||||||
|
COPY deploy.ipynb .
|
||||||
|
|
||||||
|
# RUN python3 -m large_file --backend s3 -secrets ~/.aws/credentials --cache my_first_file.json:3 my_second_file my_folder:0
|
||||||
|
|
||||||
|
CMD ["deploy.ipynb"]
|
||||||
23
docs/great_ai_example-main/README.md
Normal file
23
docs/great_ai_example-main/README.md
Normal file
|
|
@ -0,0 +1,23 @@
|
||||||
|
# Train a domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Install
|
||||||
|
|
||||||
|
### System dependencies
|
||||||
|
|
||||||
|
Make sure you have `python3`, `pip`, and `venv` installed.
|
||||||
|
> On Ubuntu, execute: `sudo apt install -y python3 python3-pip python3-venv`
|
||||||
|
|
||||||
|
### Install dependencies
|
||||||
|
```sh
|
||||||
|
python3 -m venv --copies .env
|
||||||
|
source .env/bin/activate
|
||||||
|
|
||||||
|
pip install -r requirements.txt
|
||||||
|
```
|
||||||
|
## Execute
|
||||||
|
|
||||||
|
- [Part 1](src/data.ipynb)
|
||||||
|
- [Part 2](src/train.ipynb)
|
||||||
|
- [Part 3](src/deploy.ipynb)
|
||||||
1
docs/great_ai_example-main/diagrams/scope-data.svg
Normal file
1
docs/great_ai_example-main/diagrams/scope-data.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 14 KiB |
1
docs/great_ai_example-main/diagrams/scope-deploy.svg
Normal file
1
docs/great_ai_example-main/diagrams/scope-deploy.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 14 KiB |
1
docs/great_ai_example-main/diagrams/scope-train.svg
Normal file
1
docs/great_ai_example-main/diagrams/scope-train.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 14 KiB |
1
docs/great_ai_example-main/diagrams/scope.svg
Normal file
1
docs/great_ai_example-main/diagrams/scope.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 14 KiB |
31
docs/great_ai_example-main/format.sh
Executable file
31
docs/great_ai_example-main/format.sh
Executable file
|
|
@ -0,0 +1,31 @@
|
||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
echo "Installing dependencies if necessary"
|
||||||
|
python3 -m pip install --upgrade autoflake isort black[jupyter] mypy flake8
|
||||||
|
|
||||||
|
echo "Formatting and checking $1"
|
||||||
|
|
||||||
|
cd $1
|
||||||
|
|
||||||
|
echo Running autoflake
|
||||||
|
python3 -m autoflake --expand-star-imports --remove-all-unused-imports --ignore-init-module-imports --remove-unused-variables --in-place -r .
|
||||||
|
|
||||||
|
echo Running isort
|
||||||
|
python3 -m isort --profile black --skip .env .
|
||||||
|
|
||||||
|
echo Running black
|
||||||
|
python3 -m black . --exclude .env
|
||||||
|
|
||||||
|
if ls *.py 1> /dev/null 2>&1; then
|
||||||
|
echo Running mypy
|
||||||
|
python3 -m mypy --namespace-packages --ignore-missing-imports --install-types --non-interactive --disallow-untyped-defs --disallow-incomplete-defs --pretty --follow-imports=silent --exclude=external/ --exclude=/build/ .
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo Running Flake8
|
||||||
|
python3 -m flake8 . --count --show-source --statistics --exclude=__init__.py,.env,external --ignore=E501,E722,E402,W503,E203
|
||||||
|
|
||||||
|
cd -
|
||||||
|
|
||||||
|
echo "Finished formatting"
|
||||||
4
docs/great_ai_example-main/requirements.txt
Normal file
4
docs/great_ai_example-main/requirements.txt
Normal file
|
|
@ -0,0 +1,4 @@
|
||||||
|
great_ai
|
||||||
|
notebook
|
||||||
|
nbformat
|
||||||
|
nbconvert
|
||||||
265
docs/great_ai_example-main/src/data.ipynb
Normal file
265
docs/great_ai_example-main/src/data.ipynb
Normal file
|
|
@ -0,0 +1,265 @@
|
||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"# Train a domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)\n",
|
||||||
|
"> Part 1: obtain and clean data\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"> The blue boxes show the steps implemented in this notebook.\n",
|
||||||
|
"\n",
|
||||||
|
"### Extract\n",
|
||||||
|
"\n",
|
||||||
|
"This can be achieved by downloading a public dataset (such as in this case), or by having a Data Engineer setup and give us access to the organisation's data.\n",
|
||||||
|
"\n",
|
||||||
|
"In this case, we download the semantic scholar dataset from a public S3 bucket."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 1,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"MAX_CHUNK_COUNT = 1"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 2,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"'Processing 1 out of the 6002 available chunks'"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_count": 2,
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "execute_result"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"import urllib.request\n",
|
||||||
|
"from random import shuffle\n",
|
||||||
|
"\n",
|
||||||
|
"manifest = (\n",
|
||||||
|
" urllib.request.urlopen(\n",
|
||||||
|
" \"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/manifest.txt\"\n",
|
||||||
|
" )\n",
|
||||||
|
" .read()\n",
|
||||||
|
" .decode()\n",
|
||||||
|
") # a list of available chunks separated by '\\n' characters\n",
|
||||||
|
"\n",
|
||||||
|
"lines = manifest.split()\n",
|
||||||
|
"shuffle(lines)\n",
|
||||||
|
"chunks = lines[:MAX_CHUNK_COUNT]\n",
|
||||||
|
"\n",
|
||||||
|
"f\"Processing {len(chunks)} out of the {len(manifest.split())} available chunks\""
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### Transform\n",
|
||||||
|
"\n",
|
||||||
|
"- Filter out non-English abstracts using `great_ai.utilities.predict_language`\n",
|
||||||
|
"- Project it to only keep the necessary components (text and labels), clean the textual content using `great_ai.utilities.clean`\n",
|
||||||
|
"- We will speed up processing using `great_ai.utilities.parallel_map`."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 3,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Spacy model en_core_web_sm not found locally, downloading...\n",
|
||||||
|
"Collecting en-core-web-sm==3.3.0\n",
|
||||||
|
" Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl (12.8 MB)\n",
|
||||||
|
" ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 12.8/12.8 MB 3.6 MB/s eta 0:00:00\n",
|
||||||
|
"Requirement already satisfied: spacy<3.4.0,>=3.3.0.dev0 in ./.env/lib/python3.10/site-packages (from en-core-web-sm==3.3.0) (3.3.1)\n",
|
||||||
|
"Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (4.64.0)\n",
|
||||||
|
"Requirement already satisfied: wasabi<1.1.0,>=0.9.1 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (0.9.1)\n",
|
||||||
|
"Requirement already satisfied: srsly<3.0.0,>=2.4.3 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.4.3)\n",
|
||||||
|
"Requirement already satisfied: typer<0.5.0,>=0.3.0 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (0.4.1)\n",
|
||||||
|
"Requirement already satisfied: setuptools in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (59.6.0)\n",
|
||||||
|
"Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.0.7)\n",
|
||||||
|
"Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (1.0.2)\n",
|
||||||
|
"Requirement already satisfied: blis<0.8.0,>=0.4.0 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (0.7.8)\n",
|
||||||
|
"Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (1.0.7)\n",
|
||||||
|
"Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.9 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.0.9)\n",
|
||||||
|
"Requirement already satisfied: requests<3.0.0,>=2.13.0 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.28.0)\n",
|
||||||
|
"Requirement already satisfied: numpy>=1.15.0 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (1.23.0)\n",
|
||||||
|
"Requirement already satisfied: cymem<2.1.0,>=2.0.2 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.0.6)\n",
|
||||||
|
"Requirement already satisfied: pathy>=0.3.5 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (0.6.1)\n",
|
||||||
|
"Requirement already satisfied: jinja2 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.1.2)\n",
|
||||||
|
"Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.3.0)\n",
|
||||||
|
"Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (1.8.2)\n",
|
||||||
|
"Requirement already satisfied: preshed<3.1.0,>=3.0.2 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.0.6)\n",
|
||||||
|
"Requirement already satisfied: thinc<8.1.0,>=8.0.14 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (8.0.17)\n",
|
||||||
|
"Requirement already satisfied: packaging>=20.0 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (21.3)\n",
|
||||||
|
"Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in ./.env/lib/python3.10/site-packages (from packaging>=20.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.0.9)\n",
|
||||||
|
"Requirement already satisfied: smart-open<6.0.0,>=5.0.0 in ./.env/lib/python3.10/site-packages (from pathy>=0.3.5->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (5.2.1)\n",
|
||||||
|
"Requirement already satisfied: typing-extensions>=3.7.4.3 in ./.env/lib/python3.10/site-packages (from pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (4.2.0)\n",
|
||||||
|
"Requirement already satisfied: charset-normalizer~=2.0.0 in ./.env/lib/python3.10/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.0.12)\n",
|
||||||
|
"Requirement already satisfied: certifi>=2017.4.17 in ./.env/lib/python3.10/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2022.6.15)\n",
|
||||||
|
"Requirement already satisfied: idna<4,>=2.5 in ./.env/lib/python3.10/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.3)\n",
|
||||||
|
"Requirement already satisfied: urllib3<1.27,>=1.21.1 in ./.env/lib/python3.10/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (1.26.9)\n",
|
||||||
|
"Requirement already satisfied: click<9.0.0,>=7.1.1 in ./.env/lib/python3.10/site-packages (from typer<0.5.0,>=0.3.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (8.1.3)\n",
|
||||||
|
"Requirement already satisfied: MarkupSafe>=2.0 in ./.env/lib/python3.10/site-packages (from jinja2->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.1.1)\n",
|
||||||
|
"Installing collected packages: en-core-web-sm\n",
|
||||||
|
"Successfully installed en-core-web-sm-3.3.0\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "stderr",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"\u001b[38;5;226m2022-06-25 14:21:57,983 | WARNING | Limiting concurrency to 1 because there are only 1 chunks\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-06-25 14:21:57,984 | INFO | Starting parallel map (concurrency: 1, chunk size: 1)\u001b[0m\n",
|
||||||
|
"\u001b[38;5;226m2022-06-25 14:21:57,984 | WARNING | Running in series, there is no reason for parallelism\u001b[0m\n",
|
||||||
|
"100%|██████████| 1/1 [03:26<00:00, 206.86s/it]\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"from typing import List, Tuple\n",
|
||||||
|
"import json\n",
|
||||||
|
"import gzip\n",
|
||||||
|
"from great_ai import parallel_map, clean, is_english, predict_language\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"def preprocess_chunk(chunk_key: str) -> List[Tuple[str, List[str]]]:\n",
|
||||||
|
" # Extract\n",
|
||||||
|
" response = urllib.request.urlopen(\n",
|
||||||
|
" f\"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/{chunk_key}\"\n",
|
||||||
|
" ) # a gzipped JSON Lines file\n",
|
||||||
|
"\n",
|
||||||
|
" decompressed = gzip.decompress(response.read())\n",
|
||||||
|
" decoded = decompressed.decode()\n",
|
||||||
|
" chunk = [json.loads(line) for line in decoded.split(\"\\n\") if line]\n",
|
||||||
|
"\n",
|
||||||
|
" # Transform\n",
|
||||||
|
" return [\n",
|
||||||
|
" (\n",
|
||||||
|
" clean(\n",
|
||||||
|
" f'{c[\"title\"]} {c[\"paperAbstract\"]} {c[\"journalName\"]} {c[\"venue\"]}',\n",
|
||||||
|
" convert_to_ascii=True,\n",
|
||||||
|
" ), # The text is cleaned to remove PDF extraction, web scraping, and other common artifacts\n",
|
||||||
|
" c[\"fieldsOfStudy\"],\n",
|
||||||
|
" ) # Create pairs of `(text, [...domains])`\n",
|
||||||
|
" for c in chunk\n",
|
||||||
|
" if c[\"fieldsOfStudy\"] and is_english(predict_language(c[\"paperAbstract\"]))\n",
|
||||||
|
" ]\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"preprocessed_chunks = parallel_map(preprocess_chunk, chunks)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 4,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"from itertools import chain\n",
|
||||||
|
"\n",
|
||||||
|
"preprocessed_data = list(chain(*preprocessed_chunks))\n",
|
||||||
|
"X, y = zip(\n",
|
||||||
|
" *preprocessed_data\n",
|
||||||
|
") # X is the input, y is the expected (ground truth) output"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### Load\n",
|
||||||
|
"\n",
|
||||||
|
"Upload the dataset (or a part of it) to a central repository using `great_ai.add_ground_truth`. This step automatically tags each datapoint with a split label according to the ratios we set. Additional tags can be also given.\n",
|
||||||
|
"\n",
|
||||||
|
"#### Production-ready backend\n",
|
||||||
|
"\n",
|
||||||
|
"The MongoDB driver is automatically configured if `mongo.ini` exists with the following scheme:\n",
|
||||||
|
"\n",
|
||||||
|
"```ini\n",
|
||||||
|
"mongo_connection_string=mongodb://localhost:27017/\n",
|
||||||
|
"mongo_database=my_great_ai_db\n",
|
||||||
|
"```\n",
|
||||||
|
"> You can install MongoDB from [here](https://www.mongodb.com/docs/manual/installation) or [use it as a service](https://www.mongodb.com/cloud/atlas/register)\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 5,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stderr",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"\u001b[38;5;226m2022-06-25 14:25:24,989 | WARNING | Environment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-06-25 14:25:24,990 | INFO | Found credentials file (/data/projects/great_ai_example/mongo.ini), initialising MongodbDriver\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-06-25 14:25:24,991 | INFO | Found credentials file (/data/projects/great_ai_example/mongo.ini), initialising LargeFileMongo\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-06-25 14:25:24,992 | INFO | Settings: configured ✅\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-06-25 14:25:24,993 | INFO | 🔩 tracing_database: MongodbDriver\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-06-25 14:25:24,994 | INFO | 🔩 large_file_implementation: LargeFileMongo\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-06-25 14:25:24,994 | INFO | 🔩 is_production: False\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-06-25 14:25:24,995 | INFO | 🔩 should_log_exception_stack: True\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-06-25 14:25:24,996 | INFO | 🔩 prediction_cache_size: 512\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-06-25 14:25:24,997 | INFO | 🔩 dashboard_table_size: 50\u001b[0m\n",
|
||||||
|
"\u001b[38;5;226m2022-06-25 14:25:24,998 | WARNING | You still need to check whether you follow all best practices before trusting your deployment.\u001b[0m\n",
|
||||||
|
"\u001b[38;5;226m2022-06-25 14:25:24,998 | WARNING | > Find out more at https://se-ml.github.io/practices/\u001b[0m\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"from great_ai import add_ground_truth\n",
|
||||||
|
"\n",
|
||||||
|
"add_ground_truth(X, y, train_split_ratio=0.8, test_split_ratio=0.2)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### Next: [Part 2](train.ipynb)"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3.10.4 ('.env': venv)",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.10.4"
|
||||||
|
},
|
||||||
|
"orig_nbformat": 4,
|
||||||
|
"vscode": {
|
||||||
|
"interpreter": {
|
||||||
|
"hash": "c1f394f9662881005685eeb18d8f9f77079b1b8b9a5ece1f825bfa01fcb7f52f"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 2
|
||||||
|
}
|
||||||
239
docs/great_ai_example-main/src/deploy.ipynb
Normal file
239
docs/great_ai_example-main/src/deploy.ipynb
Normal file
File diff suppressed because one or more lines are too long
2
docs/great_ai_example-main/src/mongo.ini
Normal file
2
docs/great_ai_example-main/src/mongo.ini
Normal file
|
|
@ -0,0 +1,2 @@
|
||||||
|
mongo_connection_string=mongodb://localhost:27017/ # change this
|
||||||
|
mongo_database=great_ai_example # this will be automatically created
|
||||||
1811
docs/great_ai_example-main/src/train.ipynb
Normal file
1811
docs/great_ai_example-main/src/train.ipynb
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -2,24 +2,85 @@
|
||||||
"cells": [
|
"cells": [
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "code",
|
||||||
"execution_count": null,
|
"execution_count": 1,
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stderr",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"\u001b[38;5;226m2022-07-08 10:26:12,806 | WARNING | Environment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
|
||||||
|
"\u001b[38;5;226m2022-07-08 10:26:12,807 | WARNING | Cannot find credentials files, defaulting to using ParallelTinyDbDriver\u001b[0m\n",
|
||||||
|
"\u001b[38;5;226m2022-07-08 10:26:12,807 | WARNING | The selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n",
|
||||||
|
"\u001b[38;5;226m2022-07-08 10:26:12,808 | WARNING | Cannot find credentials files, defaulting to using LargeFileLocal\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-07-08 10:26:12,808 | INFO | Settings: configured ✅\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-07-08 10:26:12,809 | INFO | 🔩 tracing_database: ParallelTinyDbDriver\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-07-08 10:26:12,809 | INFO | 🔩 large_file_implementation: LargeFileLocal\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-07-08 10:26:12,811 | INFO | 🔩 is_production: False\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-07-08 10:26:12,811 | INFO | 🔩 should_log_exception_stack: True\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-07-08 10:26:12,812 | INFO | 🔩 prediction_cache_size: 512\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-07-08 10:26:12,813 | INFO | 🔩 dashboard_table_size: 20\u001b[0m\n",
|
||||||
|
"\u001b[38;5;226m2022-07-08 10:26:12,813 | WARNING | You still need to check whether you follow all best practices before trusting your deployment.\u001b[0m\n",
|
||||||
|
"\u001b[38;5;226m2022-07-08 10:26:12,814 | WARNING | > Find out more at https://se-ml.github.io/practices/\u001b[0m\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
"source": [
|
"source": [
|
||||||
"from great_ai import GreatAI\n",
|
"from great_ai import GreatAI\n",
|
||||||
|
"from asyncio import sleep\n",
|
||||||
"\n",
|
"\n",
|
||||||
"\n",
|
"\n",
|
||||||
"@GreatAI.create\n",
|
"@GreatAI.create\n",
|
||||||
"def hello_world(name: str) -> str:\n",
|
"async def hello_world(name: str) -> str:\n",
|
||||||
|
" await sleep(1)\n",
|
||||||
" return f\"Hello {name}!\""
|
" return f\"Hello {name}!\""
|
||||||
]
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 2,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"Trace(trace_id='27c5e581-ad6e-4d50-bb77-862dfb7963fb', created='2022-07-08T08:26:12.911975', original_execution_time_ms=1.933, logged_values={'arg:name:value': 'hi', 'arg:name:length': 2}, models=[], exception=None, output='Hello hi!', feedback=None, tags=['hello_world', 'online', 'development'])"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_count": 2,
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "execute_result"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"await hello_world('hi')"
|
||||||
|
]
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"language_info": {
|
"kernelspec": {
|
||||||
"name": "python"
|
"display_name": "Python 3.10.4 ('.env': venv)",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
},
|
},
|
||||||
"orig_nbformat": 4
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.10.4"
|
||||||
|
},
|
||||||
|
"orig_nbformat": 4,
|
||||||
|
"vscode": {
|
||||||
|
"interpreter": {
|
||||||
|
"hash": "02dd6d3afbfa9fbbe1037d64ad9014965528a1ccad21929d6e72f466389a68ad"
|
||||||
|
}
|
||||||
|
}
|
||||||
},
|
},
|
||||||
"nbformat": 4,
|
"nbformat": 4,
|
||||||
"nbformat_minor": 2
|
"nbformat_minor": 2
|
||||||
|
|
|
||||||
|
|
@ -1,6 +0,0 @@
|
||||||
from great_ai import GreatAI
|
|
||||||
|
|
||||||
|
|
||||||
@GreatAI.create
|
|
||||||
def hello_world(name: str) -> str:
|
|
||||||
return f"Hello {name}!"
|
|
||||||
0
docs/how-to-guides.md
Normal file
0
docs/how-to-guides.md
Normal file
17
docs/index.md
Normal file
17
docs/index.md
Normal file
|
|
@ -0,0 +1,17 @@
|
||||||
|
# Welcome to MkDocs
|
||||||
|
|
||||||
|
For full documentation visit [mkdocs.org](https://www.mkdocs.org).
|
||||||
|
|
||||||
|
## Commands
|
||||||
|
|
||||||
|
* `mkdocs new [dir-name]` - Create a new project.
|
||||||
|
* `mkdocs serve` - Start the live-reloading docs server.
|
||||||
|
* `mkdocs build` - Build the documentation site.
|
||||||
|
* `mkdocs -h` - Print help message and exit.
|
||||||
|
|
||||||
|
## Project layout
|
||||||
|
|
||||||
|
mkdocs.yml # The configuration file.
|
||||||
|
docs/
|
||||||
|
index.md # The documentation homepage.
|
||||||
|
... # Other markdown pages, images and other files.
|
||||||
115
docs/map.ipynb
115
docs/map.ipynb
|
|
@ -1,115 +0,0 @@
|
||||||
{
|
|
||||||
"cells": [
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 1,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stderr",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"/data/projects/great_ai/.env/lib/python3.10/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
|
|
||||||
" from .autonotebook import tqdm as notebook_tqdm\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"from great_ai.utilities import parallel_map"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 2,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stderr",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"\u001b[38;5;39m2022-06-29 09:11:18,070 | INFO | Parallel map: configured ✅\u001b[0m\n",
|
|
||||||
"\u001b[38;5;39m2022-06-29 09:11:18,071 | INFO | ⚙️ concurrency: 200\u001b[0m\n",
|
|
||||||
"\u001b[38;5;39m2022-06-29 09:11:18,071 | INFO | ⚙️ chunk length: 1\u001b[0m\n",
|
|
||||||
"\u001b[38;5;39m2022-06-29 09:11:18,071 | INFO | ⚙️ chunk count: 1000\u001b[0m\n",
|
|
||||||
"\u001b[38;5;39m2022-06-29 09:11:18,072 | INFO | ⚙️ function size: 0 kB\u001b[0m\n",
|
|
||||||
"Parallel map: 100%|██████████| 1000/1000 [00:00<00:00, 1345.90it/s]\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"from time import sleep\n",
|
|
||||||
"\n",
|
|
||||||
"parallel_map(lambda x: sleep(0.1), range(int(1000)), concurrency=200)\n",
|
|
||||||
"None"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 3,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stderr",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"\u001b[38;5;39m2022-06-29 09:11:20,399 | INFO | Parallel map: configured ✅\u001b[0m\n",
|
|
||||||
"\u001b[38;5;39m2022-06-29 09:11:20,401 | INFO | ⚙️ concurrency: 12\u001b[0m\n",
|
|
||||||
"\u001b[38;5;39m2022-06-29 09:11:20,401 | INFO | ⚙️ chunk length: 1\u001b[0m\n",
|
|
||||||
"\u001b[38;5;39m2022-06-29 09:11:20,402 | INFO | ⚙️ chunk count: unknown\u001b[0m\n",
|
|
||||||
"\u001b[38;5;39m2022-06-29 09:11:20,403 | INFO | ⚙️ function size: 0 kB\u001b[0m\n",
|
|
||||||
"Parallel map: 10it [00:02, 4.93it/s]\n"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"data": {
|
|
||||||
"text/plain": [
|
|
||||||
"[0, 1, 8, 27, 64, 125, 216, 343, 512, 729]"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"execution_count": 3,
|
|
||||||
"metadata": {},
|
|
||||||
"output_type": "execute_result"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"from time import sleep\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"def my_generator():\n",
|
|
||||||
" for i in range(10):\n",
|
|
||||||
" yield i\n",
|
|
||||||
" sleep(0.2)\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"parallel_map(lambda x: x**3, my_generator(), chunk_length=1)"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"kernelspec": {
|
|
||||||
"display_name": "Python 3.10.4 ('.env': venv)",
|
|
||||||
"language": "python",
|
|
||||||
"name": "python3"
|
|
||||||
},
|
|
||||||
"language_info": {
|
|
||||||
"codemirror_mode": {
|
|
||||||
"name": "ipython",
|
|
||||||
"version": 3
|
|
||||||
},
|
|
||||||
"file_extension": ".py",
|
|
||||||
"mimetype": "text/x-python",
|
|
||||||
"name": "python",
|
|
||||||
"nbconvert_exporter": "python",
|
|
||||||
"pygments_lexer": "ipython3",
|
|
||||||
"version": "3.10.4"
|
|
||||||
},
|
|
||||||
"orig_nbformat": 4,
|
|
||||||
"vscode": {
|
|
||||||
"interpreter": {
|
|
||||||
"hash": "d88b0dd276e3f918f7798b7e97af2e3c2f843817b9e5b55a9df0a682ffd80f44"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"nbformat": 4,
|
|
||||||
"nbformat_minor": 2
|
|
||||||
}
|
|
||||||
|
|
@ -15,7 +15,7 @@ pip install open-large
|
||||||
### Simple example
|
### Simple example
|
||||||
|
|
||||||
```python
|
```python
|
||||||
from large_file import LargeFileS3
|
from great_ai.large_file import LargeFileS3
|
||||||
|
|
||||||
LargeFileS3.configure_credentials({
|
LargeFileS3.configure_credentials({
|
||||||
"aws_region_name": "your_region_like_eu-west-2",
|
"aws_region_name": "your_region_like_eu-west-2",
|
||||||
|
|
|
||||||
11
docs/overrides/main.html
Normal file
11
docs/overrides/main.html
Normal file
|
|
@ -0,0 +1,11 @@
|
||||||
|
{% extends "base.html" %}
|
||||||
|
|
||||||
|
{% block content %}
|
||||||
|
{% if page.nb_url %}
|
||||||
|
<a href="{{ page.nb_url }}" title="Download Notebook" class="md-content__button md-icon">
|
||||||
|
{% include ".icons/material/download.svg" %}
|
||||||
|
</a>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
{{ super() }}
|
||||||
|
{% endblock content %}
|
||||||
1
docs/reference.md
Normal file
1
docs/reference.md
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
::: great_ai.utilities.unique
|
||||||
|
|
@ -1,90 +0,0 @@
|
||||||
{
|
|
||||||
"cells": [
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 1,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"data": {
|
|
||||||
"text/plain": [
|
|
||||||
"Trace(trace_id='07719c28-e248-43f8-a652-608c49ad5a17', created='2022-06-26T07:55:30.587753', original_execution_time_ms=178.268, logged_values={'arg:text:value': 'I love chemical compounds and nuclear fission.', 'arg:text:length': 46, 'arg:target_confidence:value': 50}, models=[Model(key='small-domain-prediction', version=0)], exception=None, output={'labels': [{'label': 'Physics', 'confidence': 28.0, 'explanation': ['nuclear', 'chemical', 'fission', 'compounds', 'love']}, {'label': 'Chemistry', 'confidence': 22.0, 'explanation': ['chemical', 'compounds', 'nuclear', 'fission', 'love']}]}, feedback=None, tags=['predict_domain', 'online', 'development'])"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"execution_count": 1,
|
|
||||||
"metadata": {},
|
|
||||||
"output_type": "execute_result"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"from great_ai import call_remote_great_ai_async\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"await call_remote_great_ai_async(\n",
|
|
||||||
" \"http://localhost:6060\", {\"text\": \"I love chemical compounds and nuclear fission.\"}\n",
|
|
||||||
")"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 1,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"<_UnixSelectorEventLoop running=True closed=False debug=False>\n"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"ename": "Exception",
|
|
||||||
"evalue": "Already running in an event loop, you have to call call_remote_great_ai_async.",
|
|
||||||
"output_type": "error",
|
|
||||||
"traceback": [
|
|
||||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
|
||||||
"\u001b[0;31mException\u001b[0m Traceback (most recent call last)",
|
|
||||||
"\u001b[1;32m/data/projects/great_ai/docs/remote-test/remote.ipynb Cell 1'\u001b[0m in \u001b[0;36m<cell line: 4>\u001b[0;34m()\u001b[0m\n\u001b[1;32m <a href='vscode-notebook-cell:/data/projects/great_ai/docs/remote-test/remote.ipynb#ch0000000?line=0'>1</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mgreat_ai\u001b[39;00m \u001b[39mimport\u001b[39;00m call_remote_great_ai\n\u001b[0;32m----> <a href='vscode-notebook-cell:/data/projects/great_ai/docs/remote-test/remote.ipynb#ch0000000?line=3'>4</a>\u001b[0m call_remote_great_ai(\u001b[39m'\u001b[39;49m\u001b[39mhttp://localhost:6060\u001b[39;49m\u001b[39m'\u001b[39;49m, {\n\u001b[1;32m <a href='vscode-notebook-cell:/data/projects/great_ai/docs/remote-test/remote.ipynb#ch0000000?line=4'>5</a>\u001b[0m \u001b[39m'\u001b[39;49m\u001b[39mtext\u001b[39;49m\u001b[39m'\u001b[39;49m: \u001b[39m'\u001b[39;49m\u001b[39mI love chemical compounds and nuclear fission.\u001b[39;49m\u001b[39m'\u001b[39;49m\n\u001b[1;32m <a href='vscode-notebook-cell:/data/projects/great_ai/docs/remote-test/remote.ipynb#ch0000000?line=5'>6</a>\u001b[0m })\n",
|
|
||||||
"File \u001b[0;32m/data/projects/great_ai/src/great_ai/great_ai/remote/call_remote_great_ai.py:17\u001b[0m, in \u001b[0;36mcall_remote_great_ai\u001b[0;34m(base_uri, data, retry_count)\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m 16\u001b[0m \u001b[39mprint\u001b[39m(asyncio\u001b[39m.\u001b[39mget_running_loop())\n\u001b[0;32m---> 17\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mException\u001b[39;00m(\u001b[39mf\u001b[39m\u001b[39m'\u001b[39m\u001b[39mAlready running in an event loop, you have to call \u001b[39m\u001b[39m{\u001b[39;00mcall_remote_great_ai_async\u001b[39m.\u001b[39m\u001b[39m__name__\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m.\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[1;32m 18\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mRuntimeError\u001b[39;00m:\n\u001b[1;32m 19\u001b[0m \u001b[39mpass\u001b[39;00m\n",
|
|
||||||
"\u001b[0;31mException\u001b[0m: Already running in an event loop, you have to call call_remote_great_ai_async."
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"from great_ai import call_remote_great_ai\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"call_remote_great_ai(\n",
|
|
||||||
" \"http://localhost:6060\", {\"text\": \"I love chemical compounds and nuclear fission.\"}\n",
|
|
||||||
")"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"kernelspec": {
|
|
||||||
"display_name": "Python 3.10.4 ('.env': venv)",
|
|
||||||
"language": "python",
|
|
||||||
"name": "python3"
|
|
||||||
},
|
|
||||||
"language_info": {
|
|
||||||
"codemirror_mode": {
|
|
||||||
"name": "ipython",
|
|
||||||
"version": 3
|
|
||||||
},
|
|
||||||
"file_extension": ".py",
|
|
||||||
"mimetype": "text/x-python",
|
|
||||||
"name": "python",
|
|
||||||
"nbconvert_exporter": "python",
|
|
||||||
"pygments_lexer": "ipython3",
|
|
||||||
"version": "3.10.4"
|
|
||||||
},
|
|
||||||
"orig_nbformat": 4,
|
|
||||||
"vscode": {
|
|
||||||
"interpreter": {
|
|
||||||
"hash": "d88b0dd276e3f918f7798b7e97af2e3c2f843817b9e5b55a9df0a682ffd80f44"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"nbformat": 4,
|
|
||||||
"nbformat_minor": 2
|
|
||||||
}
|
|
||||||
|
|
@ -19,49 +19,16 @@
|
||||||
"name": "stderr",
|
"name": "stderr",
|
||||||
"output_type": "stream",
|
"output_type": "stream",
|
||||||
"text": [
|
"text": [
|
||||||
"\u001b[38;5;39m2022-06-20 14:33:17,570 | INFO | Starting parallel map (concurrency: 12, chunk size: 700)\u001b[0m\n"
|
"84000it [00:09, 8647.60it/s] \n",
|
||||||
|
"22399it [00:03, 6368.63it/s]\n"
|
||||||
]
|
]
|
||||||
},
|
|
||||||
{
|
|
||||||
"data": {
|
|
||||||
"application/vnd.jupyter.widget-view+json": {
|
|
||||||
"model_id": "e6f76bf83615422cb4352dcb3af48a26",
|
|
||||||
"version_major": 2,
|
|
||||||
"version_minor": 0
|
|
||||||
},
|
|
||||||
"text/plain": [
|
|
||||||
" 0%| | 0/84000 [00:00<?, ?it/s]"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"metadata": {},
|
|
||||||
"output_type": "display_data"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "stderr",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"\u001b[38;5;39m2022-06-20 14:33:24,846 | INFO | Starting parallel map (concurrency: 12, chunk size: 187)\u001b[0m\n"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"data": {
|
|
||||||
"application/vnd.jupyter.widget-view+json": {
|
|
||||||
"model_id": "ab012f9276d74bdda15b00cfec6c8f56",
|
|
||||||
"version_major": 2,
|
|
||||||
"version_minor": 0
|
|
||||||
},
|
|
||||||
"text/plain": [
|
|
||||||
" 0%| | 0/22399 [00:00<?, ?it/s]"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"metadata": {},
|
|
||||||
"output_type": "display_data"
|
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"source": [
|
"source": [
|
||||||
"import json\n",
|
"import json\n",
|
||||||
"from typing import Tuple\n",
|
"from typing import Tuple\n",
|
||||||
"from great_ai.utilities import clean, parallel_map\n",
|
"from great_ai.utilities import clean, parallel_map\n",
|
||||||
|
"from tqdm.cli import tqdm\n",
|
||||||
"\n",
|
"\n",
|
||||||
"\n",
|
"\n",
|
||||||
"def preprocess(line: str) -> Tuple[str, str]:\n",
|
"def preprocess(line: str) -> Tuple[str, str]:\n",
|
||||||
|
|
@ -71,14 +38,14 @@
|
||||||
"\n",
|
"\n",
|
||||||
"\n",
|
"\n",
|
||||||
"with open(\"mag/train.txt\", encoding=\"utf-8\") as f:\n",
|
"with open(\"mag/train.txt\", encoding=\"utf-8\") as f:\n",
|
||||||
" training_data = parallel_map(preprocess, f.readlines())\n",
|
" training_data = list(tqdm(parallel_map(preprocess, f.readlines())))\n",
|
||||||
"\n",
|
"\n",
|
||||||
"X_train = [d[0] for d in training_data]\n",
|
"X_train = [d[0] for d in training_data]\n",
|
||||||
"y_train = [d[1] for d in training_data]\n",
|
"y_train = [d[1] for d in training_data]\n",
|
||||||
"\n",
|
"\n",
|
||||||
"\n",
|
"\n",
|
||||||
"with open(\"mag/test.txt\", encoding=\"utf-8\") as f:\n",
|
"with open(\"mag/test.txt\", encoding=\"utf-8\") as f:\n",
|
||||||
" test_data = parallel_map(preprocess, f.readlines())\n",
|
" test_data = list(tqdm(parallel_map(preprocess, f.readlines())))\n",
|
||||||
"\n",
|
"\n",
|
||||||
"X_test = [d[0] for d in test_data]\n",
|
"X_test = [d[0] for d in test_data]\n",
|
||||||
"y_test = [d[1] for d in test_data]"
|
"y_test = [d[1] for d in test_data]"
|
||||||
|
|
@ -1070,10 +1037,10 @@
|
||||||
" <tbody>\n",
|
" <tbody>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>12</th>\n",
|
" <th>12</th>\n",
|
||||||
" <td>1.172014</td>\n",
|
" <td>1.227354</td>\n",
|
||||||
" <td>0.033318</td>\n",
|
" <td>0.011710</td>\n",
|
||||||
" <td>0.625773</td>\n",
|
" <td>0.652844</td>\n",
|
||||||
" <td>0.013720</td>\n",
|
" <td>0.040511</td>\n",
|
||||||
" <td>1</td>\n",
|
" <td>1</td>\n",
|
||||||
" <td>True</td>\n",
|
" <td>True</td>\n",
|
||||||
" <td>0.05</td>\n",
|
" <td>0.05</td>\n",
|
||||||
|
|
@ -1088,10 +1055,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>15</th>\n",
|
" <th>15</th>\n",
|
||||||
" <td>1.201231</td>\n",
|
" <td>1.279783</td>\n",
|
||||||
" <td>0.016728</td>\n",
|
" <td>0.037969</td>\n",
|
||||||
" <td>0.613568</td>\n",
|
" <td>0.634761</td>\n",
|
||||||
" <td>0.013720</td>\n",
|
" <td>0.068606</td>\n",
|
||||||
" <td>1</td>\n",
|
" <td>1</td>\n",
|
||||||
" <td>True</td>\n",
|
" <td>True</td>\n",
|
||||||
" <td>0.1</td>\n",
|
" <td>0.1</td>\n",
|
||||||
|
|
@ -1106,10 +1073,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>18</th>\n",
|
" <th>18</th>\n",
|
||||||
" <td>1.211270</td>\n",
|
" <td>1.332854</td>\n",
|
||||||
" <td>0.036636</td>\n",
|
" <td>0.171405</td>\n",
|
||||||
" <td>0.670754</td>\n",
|
" <td>0.743611</td>\n",
|
||||||
" <td>0.076857</td>\n",
|
" <td>0.109792</td>\n",
|
||||||
" <td>1</td>\n",
|
" <td>1</td>\n",
|
||||||
" <td>False</td>\n",
|
" <td>False</td>\n",
|
||||||
" <td>0.05</td>\n",
|
" <td>0.05</td>\n",
|
||||||
|
|
@ -1124,10 +1091,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>21</th>\n",
|
" <th>21</th>\n",
|
||||||
" <td>1.165763</td>\n",
|
" <td>1.253222</td>\n",
|
||||||
" <td>0.061206</td>\n",
|
" <td>0.072134</td>\n",
|
||||||
" <td>0.549646</td>\n",
|
" <td>0.612344</td>\n",
|
||||||
" <td>0.046776</td>\n",
|
" <td>0.016771</td>\n",
|
||||||
" <td>1</td>\n",
|
" <td>1</td>\n",
|
||||||
" <td>False</td>\n",
|
" <td>False</td>\n",
|
||||||
" <td>0.1</td>\n",
|
" <td>0.1</td>\n",
|
||||||
|
|
@ -1142,10 +1109,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>3</th>\n",
|
" <th>3</th>\n",
|
||||||
" <td>1.277603</td>\n",
|
" <td>1.472035</td>\n",
|
||||||
" <td>0.127785</td>\n",
|
" <td>0.080849</td>\n",
|
||||||
" <td>0.650135</td>\n",
|
" <td>0.654935</td>\n",
|
||||||
" <td>0.023038</td>\n",
|
" <td>0.044302</td>\n",
|
||||||
" <td>0.5</td>\n",
|
" <td>0.5</td>\n",
|
||||||
" <td>True</td>\n",
|
" <td>True</td>\n",
|
||||||
" <td>0.1</td>\n",
|
" <td>0.1</td>\n",
|
||||||
|
|
@ -1160,10 +1127,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>0</th>\n",
|
" <th>0</th>\n",
|
||||||
" <td>1.118055</td>\n",
|
" <td>1.380641</td>\n",
|
||||||
" <td>0.058978</td>\n",
|
" <td>0.054306</td>\n",
|
||||||
" <td>0.622284</td>\n",
|
" <td>0.739966</td>\n",
|
||||||
" <td>0.014230</td>\n",
|
" <td>0.053385</td>\n",
|
||||||
" <td>0.5</td>\n",
|
" <td>0.5</td>\n",
|
||||||
" <td>True</td>\n",
|
" <td>True</td>\n",
|
||||||
" <td>0.05</td>\n",
|
" <td>0.05</td>\n",
|
||||||
|
|
@ -1178,10 +1145,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>9</th>\n",
|
" <th>9</th>\n",
|
||||||
" <td>1.166192</td>\n",
|
" <td>1.284987</td>\n",
|
||||||
" <td>0.044232</td>\n",
|
" <td>0.113903</td>\n",
|
||||||
" <td>0.619351</td>\n",
|
" <td>0.696876</td>\n",
|
||||||
" <td>0.024824</td>\n",
|
" <td>0.003757</td>\n",
|
||||||
" <td>0.5</td>\n",
|
" <td>0.5</td>\n",
|
||||||
" <td>False</td>\n",
|
" <td>False</td>\n",
|
||||||
" <td>0.1</td>\n",
|
" <td>0.1</td>\n",
|
||||||
|
|
@ -1196,10 +1163,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>6</th>\n",
|
" <th>6</th>\n",
|
||||||
" <td>1.233377</td>\n",
|
" <td>1.291148</td>\n",
|
||||||
" <td>0.063715</td>\n",
|
" <td>0.101837</td>\n",
|
||||||
" <td>0.666418</td>\n",
|
" <td>0.686561</td>\n",
|
||||||
" <td>0.061664</td>\n",
|
" <td>0.083989</td>\n",
|
||||||
" <td>0.5</td>\n",
|
" <td>0.5</td>\n",
|
||||||
" <td>False</td>\n",
|
" <td>False</td>\n",
|
||||||
" <td>0.05</td>\n",
|
" <td>0.05</td>\n",
|
||||||
|
|
@ -1214,10 +1181,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>13</th>\n",
|
" <th>13</th>\n",
|
||||||
" <td>1.245147</td>\n",
|
" <td>1.268873</td>\n",
|
||||||
" <td>0.089528</td>\n",
|
" <td>0.042412</td>\n",
|
||||||
" <td>0.645836</td>\n",
|
" <td>0.645649</td>\n",
|
||||||
" <td>0.006279</td>\n",
|
" <td>0.022738</td>\n",
|
||||||
" <td>1</td>\n",
|
" <td>1</td>\n",
|
||||||
" <td>True</td>\n",
|
" <td>True</td>\n",
|
||||||
" <td>0.05</td>\n",
|
" <td>0.05</td>\n",
|
||||||
|
|
@ -1232,10 +1199,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>16</th>\n",
|
" <th>16</th>\n",
|
||||||
" <td>1.145362</td>\n",
|
" <td>1.147120</td>\n",
|
||||||
" <td>0.071149</td>\n",
|
" <td>0.009335</td>\n",
|
||||||
" <td>0.637540</td>\n",
|
" <td>0.599006</td>\n",
|
||||||
" <td>0.028854</td>\n",
|
" <td>0.045637</td>\n",
|
||||||
" <td>1</td>\n",
|
" <td>1</td>\n",
|
||||||
" <td>True</td>\n",
|
" <td>True</td>\n",
|
||||||
" <td>0.1</td>\n",
|
" <td>0.1</td>\n",
|
||||||
|
|
@ -1250,10 +1217,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>4</th>\n",
|
" <th>4</th>\n",
|
||||||
" <td>1.179663</td>\n",
|
" <td>1.186988</td>\n",
|
||||||
" <td>0.046808</td>\n",
|
" <td>0.057330</td>\n",
|
||||||
" <td>0.598699</td>\n",
|
" <td>0.642233</td>\n",
|
||||||
" <td>0.035973</td>\n",
|
" <td>0.078248</td>\n",
|
||||||
" <td>0.5</td>\n",
|
" <td>0.5</td>\n",
|
||||||
" <td>True</td>\n",
|
" <td>True</td>\n",
|
||||||
" <td>0.1</td>\n",
|
" <td>0.1</td>\n",
|
||||||
|
|
@ -1268,10 +1235,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>1</th>\n",
|
" <th>1</th>\n",
|
||||||
" <td>1.097773</td>\n",
|
" <td>1.413231</td>\n",
|
||||||
" <td>0.005871</td>\n",
|
" <td>0.153288</td>\n",
|
||||||
" <td>0.596514</td>\n",
|
" <td>0.765960</td>\n",
|
||||||
" <td>0.025899</td>\n",
|
" <td>0.099535</td>\n",
|
||||||
" <td>0.5</td>\n",
|
" <td>0.5</td>\n",
|
||||||
" <td>True</td>\n",
|
" <td>True</td>\n",
|
||||||
" <td>0.05</td>\n",
|
" <td>0.05</td>\n",
|
||||||
|
|
@ -1286,10 +1253,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>19</th>\n",
|
" <th>19</th>\n",
|
||||||
" <td>1.161582</td>\n",
|
" <td>1.193013</td>\n",
|
||||||
" <td>0.040011</td>\n",
|
" <td>0.079000</td>\n",
|
||||||
" <td>0.586831</td>\n",
|
" <td>0.691768</td>\n",
|
||||||
" <td>0.006144</td>\n",
|
" <td>0.027448</td>\n",
|
||||||
" <td>1</td>\n",
|
" <td>1</td>\n",
|
||||||
" <td>False</td>\n",
|
" <td>False</td>\n",
|
||||||
" <td>0.05</td>\n",
|
" <td>0.05</td>\n",
|
||||||
|
|
@ -1304,10 +1271,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>22</th>\n",
|
" <th>22</th>\n",
|
||||||
" <td>1.106277</td>\n",
|
" <td>1.043618</td>\n",
|
||||||
" <td>0.060809</td>\n",
|
" <td>0.098733</td>\n",
|
||||||
" <td>0.371986</td>\n",
|
" <td>0.450375</td>\n",
|
||||||
" <td>0.014114</td>\n",
|
" <td>0.061660</td>\n",
|
||||||
" <td>1</td>\n",
|
" <td>1</td>\n",
|
||||||
" <td>False</td>\n",
|
" <td>False</td>\n",
|
||||||
" <td>0.1</td>\n",
|
" <td>0.1</td>\n",
|
||||||
|
|
@ -1322,10 +1289,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>7</th>\n",
|
" <th>7</th>\n",
|
||||||
" <td>1.217713</td>\n",
|
" <td>1.301459</td>\n",
|
||||||
" <td>0.044787</td>\n",
|
" <td>0.062143</td>\n",
|
||||||
" <td>0.587896</td>\n",
|
" <td>0.660748</td>\n",
|
||||||
" <td>0.024613</td>\n",
|
" <td>0.056054</td>\n",
|
||||||
" <td>0.5</td>\n",
|
" <td>0.5</td>\n",
|
||||||
" <td>False</td>\n",
|
" <td>False</td>\n",
|
||||||
" <td>0.05</td>\n",
|
" <td>0.05</td>\n",
|
||||||
|
|
@ -1340,10 +1307,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>10</th>\n",
|
" <th>10</th>\n",
|
||||||
" <td>1.176654</td>\n",
|
" <td>1.433934</td>\n",
|
||||||
" <td>0.045971</td>\n",
|
" <td>0.155240</td>\n",
|
||||||
" <td>0.628005</td>\n",
|
" <td>0.636608</td>\n",
|
||||||
" <td>0.082375</td>\n",
|
" <td>0.024064</td>\n",
|
||||||
" <td>0.5</td>\n",
|
" <td>0.5</td>\n",
|
||||||
" <td>False</td>\n",
|
" <td>False</td>\n",
|
||||||
" <td>0.1</td>\n",
|
" <td>0.1</td>\n",
|
||||||
|
|
@ -1358,10 +1325,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>14</th>\n",
|
" <th>14</th>\n",
|
||||||
" <td>1.207308</td>\n",
|
" <td>1.325535</td>\n",
|
||||||
" <td>0.080923</td>\n",
|
" <td>0.073539</td>\n",
|
||||||
" <td>0.610284</td>\n",
|
" <td>0.672542</td>\n",
|
||||||
" <td>0.039157</td>\n",
|
" <td>0.085835</td>\n",
|
||||||
" <td>1</td>\n",
|
" <td>1</td>\n",
|
||||||
" <td>True</td>\n",
|
" <td>True</td>\n",
|
||||||
" <td>0.05</td>\n",
|
" <td>0.05</td>\n",
|
||||||
|
|
@ -1376,10 +1343,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>17</th>\n",
|
" <th>17</th>\n",
|
||||||
" <td>1.201498</td>\n",
|
" <td>1.237677</td>\n",
|
||||||
" <td>0.035161</td>\n",
|
" <td>0.070063</td>\n",
|
||||||
" <td>0.574234</td>\n",
|
" <td>0.651091</td>\n",
|
||||||
" <td>0.019247</td>\n",
|
" <td>0.102538</td>\n",
|
||||||
" <td>1</td>\n",
|
" <td>1</td>\n",
|
||||||
" <td>True</td>\n",
|
" <td>True</td>\n",
|
||||||
" <td>0.1</td>\n",
|
" <td>0.1</td>\n",
|
||||||
|
|
@ -1394,10 +1361,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>2</th>\n",
|
" <th>2</th>\n",
|
||||||
" <td>1.157947</td>\n",
|
" <td>1.394873</td>\n",
|
||||||
" <td>0.096841</td>\n",
|
" <td>0.105286</td>\n",
|
||||||
" <td>0.586462</td>\n",
|
" <td>0.637073</td>\n",
|
||||||
" <td>0.010952</td>\n",
|
" <td>0.041708</td>\n",
|
||||||
" <td>0.5</td>\n",
|
" <td>0.5</td>\n",
|
||||||
" <td>True</td>\n",
|
" <td>True</td>\n",
|
||||||
" <td>0.05</td>\n",
|
" <td>0.05</td>\n",
|
||||||
|
|
@ -1412,10 +1379,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>5</th>\n",
|
" <th>5</th>\n",
|
||||||
" <td>1.150511</td>\n",
|
" <td>1.270732</td>\n",
|
||||||
" <td>0.027807</td>\n",
|
" <td>0.013414</td>\n",
|
||||||
" <td>0.562870</td>\n",
|
" <td>0.620984</td>\n",
|
||||||
" <td>0.013724</td>\n",
|
" <td>0.025393</td>\n",
|
||||||
" <td>0.5</td>\n",
|
" <td>0.5</td>\n",
|
||||||
" <td>True</td>\n",
|
" <td>True</td>\n",
|
||||||
" <td>0.1</td>\n",
|
" <td>0.1</td>\n",
|
||||||
|
|
@ -1430,10 +1397,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>20</th>\n",
|
" <th>20</th>\n",
|
||||||
" <td>1.237030</td>\n",
|
" <td>1.177219</td>\n",
|
||||||
" <td>0.006805</td>\n",
|
" <td>0.039633</td>\n",
|
||||||
" <td>0.529591</td>\n",
|
" <td>0.586308</td>\n",
|
||||||
" <td>0.064449</td>\n",
|
" <td>0.044247</td>\n",
|
||||||
" <td>1</td>\n",
|
" <td>1</td>\n",
|
||||||
" <td>False</td>\n",
|
" <td>False</td>\n",
|
||||||
" <td>0.05</td>\n",
|
" <td>0.05</td>\n",
|
||||||
|
|
@ -1448,10 +1415,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>8</th>\n",
|
" <th>8</th>\n",
|
||||||
" <td>1.103937</td>\n",
|
" <td>1.165583</td>\n",
|
||||||
" <td>0.046513</td>\n",
|
" <td>0.046999</td>\n",
|
||||||
" <td>0.569367</td>\n",
|
" <td>0.603675</td>\n",
|
||||||
" <td>0.011001</td>\n",
|
" <td>0.031774</td>\n",
|
||||||
" <td>0.5</td>\n",
|
" <td>0.5</td>\n",
|
||||||
" <td>False</td>\n",
|
" <td>False</td>\n",
|
||||||
" <td>0.05</td>\n",
|
" <td>0.05</td>\n",
|
||||||
|
|
@ -1466,10 +1433,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>23</th>\n",
|
" <th>23</th>\n",
|
||||||
" <td>1.080252</td>\n",
|
" <td>0.907388</td>\n",
|
||||||
" <td>0.040597</td>\n",
|
" <td>0.121543</td>\n",
|
||||||
" <td>0.326627</td>\n",
|
" <td>0.354655</td>\n",
|
||||||
" <td>0.002905</td>\n",
|
" <td>0.023775</td>\n",
|
||||||
" <td>1</td>\n",
|
" <td>1</td>\n",
|
||||||
" <td>False</td>\n",
|
" <td>False</td>\n",
|
||||||
" <td>0.1</td>\n",
|
" <td>0.1</td>\n",
|
||||||
|
|
@ -1484,10 +1451,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>11</th>\n",
|
" <th>11</th>\n",
|
||||||
" <td>1.275601</td>\n",
|
" <td>1.210845</td>\n",
|
||||||
" <td>0.087971</td>\n",
|
" <td>0.041339</td>\n",
|
||||||
" <td>0.580025</td>\n",
|
" <td>0.653606</td>\n",
|
||||||
" <td>0.034914</td>\n",
|
" <td>0.028117</td>\n",
|
||||||
" <td>0.5</td>\n",
|
" <td>0.5</td>\n",
|
||||||
" <td>False</td>\n",
|
" <td>False</td>\n",
|
||||||
" <td>0.1</td>\n",
|
" <td>0.1</td>\n",
|
||||||
|
|
@ -1506,30 +1473,30 @@
|
||||||
],
|
],
|
||||||
"text/plain": [
|
"text/plain": [
|
||||||
" mean_fit_time std_fit_time mean_score_time std_score_time \\\n",
|
" mean_fit_time std_fit_time mean_score_time std_score_time \\\n",
|
||||||
"12 1.172014 0.033318 0.625773 0.013720 \n",
|
"12 1.227354 0.011710 0.652844 0.040511 \n",
|
||||||
"15 1.201231 0.016728 0.613568 0.013720 \n",
|
"15 1.279783 0.037969 0.634761 0.068606 \n",
|
||||||
"18 1.211270 0.036636 0.670754 0.076857 \n",
|
"18 1.332854 0.171405 0.743611 0.109792 \n",
|
||||||
"21 1.165763 0.061206 0.549646 0.046776 \n",
|
"21 1.253222 0.072134 0.612344 0.016771 \n",
|
||||||
"3 1.277603 0.127785 0.650135 0.023038 \n",
|
"3 1.472035 0.080849 0.654935 0.044302 \n",
|
||||||
"0 1.118055 0.058978 0.622284 0.014230 \n",
|
"0 1.380641 0.054306 0.739966 0.053385 \n",
|
||||||
"9 1.166192 0.044232 0.619351 0.024824 \n",
|
"9 1.284987 0.113903 0.696876 0.003757 \n",
|
||||||
"6 1.233377 0.063715 0.666418 0.061664 \n",
|
"6 1.291148 0.101837 0.686561 0.083989 \n",
|
||||||
"13 1.245147 0.089528 0.645836 0.006279 \n",
|
"13 1.268873 0.042412 0.645649 0.022738 \n",
|
||||||
"16 1.145362 0.071149 0.637540 0.028854 \n",
|
"16 1.147120 0.009335 0.599006 0.045637 \n",
|
||||||
"4 1.179663 0.046808 0.598699 0.035973 \n",
|
"4 1.186988 0.057330 0.642233 0.078248 \n",
|
||||||
"1 1.097773 0.005871 0.596514 0.025899 \n",
|
"1 1.413231 0.153288 0.765960 0.099535 \n",
|
||||||
"19 1.161582 0.040011 0.586831 0.006144 \n",
|
"19 1.193013 0.079000 0.691768 0.027448 \n",
|
||||||
"22 1.106277 0.060809 0.371986 0.014114 \n",
|
"22 1.043618 0.098733 0.450375 0.061660 \n",
|
||||||
"7 1.217713 0.044787 0.587896 0.024613 \n",
|
"7 1.301459 0.062143 0.660748 0.056054 \n",
|
||||||
"10 1.176654 0.045971 0.628005 0.082375 \n",
|
"10 1.433934 0.155240 0.636608 0.024064 \n",
|
||||||
"14 1.207308 0.080923 0.610284 0.039157 \n",
|
"14 1.325535 0.073539 0.672542 0.085835 \n",
|
||||||
"17 1.201498 0.035161 0.574234 0.019247 \n",
|
"17 1.237677 0.070063 0.651091 0.102538 \n",
|
||||||
"2 1.157947 0.096841 0.586462 0.010952 \n",
|
"2 1.394873 0.105286 0.637073 0.041708 \n",
|
||||||
"5 1.150511 0.027807 0.562870 0.013724 \n",
|
"5 1.270732 0.013414 0.620984 0.025393 \n",
|
||||||
"20 1.237030 0.006805 0.529591 0.064449 \n",
|
"20 1.177219 0.039633 0.586308 0.044247 \n",
|
||||||
"8 1.103937 0.046513 0.569367 0.011001 \n",
|
"8 1.165583 0.046999 0.603675 0.031774 \n",
|
||||||
"23 1.080252 0.040597 0.326627 0.002905 \n",
|
"23 0.907388 0.121543 0.354655 0.023775 \n",
|
||||||
"11 1.275601 0.087971 0.580025 0.034914 \n",
|
"11 1.210845 0.041339 0.653606 0.028117 \n",
|
||||||
"\n",
|
"\n",
|
||||||
" param_classifier__alpha param_classifier__fit_prior \\\n",
|
" param_classifier__alpha param_classifier__fit_prior \\\n",
|
||||||
"12 1 True \n",
|
"12 1 True \n",
|
||||||
|
|
@ -1726,7 +1693,7 @@
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "code",
|
||||||
"execution_count": 7,
|
"execution_count": 6,
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [
|
"outputs": [
|
||||||
{
|
{
|
||||||
|
|
|
||||||
0
docs/tutorials.md
Normal file
0
docs/tutorials.md
Normal file
23
mkdocs.yml
23
mkdocs.yml
|
|
@ -1,10 +1,32 @@
|
||||||
site_name: GreatAI documentation
|
site_name: GreatAI documentation
|
||||||
|
|
||||||
|
repo_url: https://github.com/ScoutinScience/great-ai
|
||||||
|
edit_uri: edit/main/docs/
|
||||||
|
|
||||||
theme:
|
theme:
|
||||||
name: "material"
|
name: "material"
|
||||||
|
custom_dir: docs/overrides
|
||||||
|
palette:
|
||||||
|
- scheme: default
|
||||||
|
toggle:
|
||||||
|
icon: material/brightness-7
|
||||||
|
name: Switch to dark mode
|
||||||
|
|
||||||
|
- scheme: slate
|
||||||
|
toggle:
|
||||||
|
icon: material/brightness-4
|
||||||
|
name: Switch to light mode
|
||||||
|
|
||||||
plugins:
|
plugins:
|
||||||
|
- git-revision-date-localized:
|
||||||
- mkdocstrings
|
- mkdocstrings
|
||||||
|
- search
|
||||||
|
- mkdocs-jupyter:
|
||||||
|
include_source: true
|
||||||
|
# allow_errors: false
|
||||||
|
# execute: false
|
||||||
|
# execute_ignore: "my-secret-files/*.ipynb"
|
||||||
|
# theme: dark
|
||||||
|
|
||||||
nav:
|
nav:
|
||||||
- Home: index.md
|
- Home: index.md
|
||||||
|
|
@ -12,3 +34,4 @@ nav:
|
||||||
- How-To Guides: how-to-guides.md
|
- How-To Guides: how-to-guides.md
|
||||||
- reference.md
|
- reference.md
|
||||||
- explanation.md
|
- explanation.md
|
||||||
|
- Notebook page: simple-mag/train.ipynb
|
||||||
|
|
|
||||||
|
|
@ -50,6 +50,7 @@ dev = [
|
||||||
"mkdocstrings[python]",
|
"mkdocstrings[python]",
|
||||||
"mkdocs-material",
|
"mkdocs-material",
|
||||||
"mkdocs-jupyter",
|
"mkdocs-jupyter",
|
||||||
|
"mkdocs-git-revision-date-localized-plugin",
|
||||||
"autoflake",
|
"autoflake",
|
||||||
"isort",
|
"isort",
|
||||||
"black[jupyter]",
|
"black[jupyter]",
|
||||||
|
|
@ -59,6 +60,7 @@ dev = [
|
||||||
"pytest-cov",
|
"pytest-cov",
|
||||||
"pytest-subtests",
|
"pytest-subtests",
|
||||||
"pytest-asyncio",
|
"pytest-asyncio",
|
||||||
|
'tqdm',
|
||||||
]
|
]
|
||||||
|
|
||||||
[project.urls]
|
[project.urls]
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue