Move files

.github/workflows/publish.yaml

@@ -0,0 +1,31 @@
+name: Publish package
+
+on:
+  workflow_run:
+    workflows: Run tests
+    branches: main
+    types: completed
+
+jobs:
+  build:
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: Install pypa/build
+        run: python -m pip install build --user
+
+      - name: Build a binary wheel and a source tarball
+        run: python -m build --sdist --wheel --outdir dist/
+
+      - name: Publish distribution to PyPI
+        uses: pypa/gh-action-pypi-publish@master
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}

.gitignore

@@ -7,5 +7,4 @@ __pycache__
 .pytest_cache
 **/.ipynb_checkpoints
 **/tracing_database.json
-**/s3.ini
 *.egg-info

.vscode/settings.json

@@ -1,5 +1,6 @@
 {
   "cSpell.words": [
+    "Analyse",
     "basereload",
     "boto",
     "botocore",

.vscode/tasks.json

@@ -4,9 +4,9 @@
     {
       "label": "Format and lint Python",
       "type": "shell",
-      "command": "source .env/bin/activate && scripts/format-python.sh great_ai && scripts/format-python.sh examples/simple",
+      "command": "source .env/bin/activate && scripts/format-python.sh .",
       "windows": {
-        "command": ".env\\bin\\activate.bat; scripts\\format-python.sh great_ai; scripts\\format-python.sh examples\\simple"
+        "command": ".env\\bin\\activate.bat; scripts\\format-python.sh ."
       },
       "group": "test",
       "presentation": {
@@ -20,9 +20,9 @@
     {
       "label": "Test Python",
       "type": "shell",
-      "command": "source .env/bin/activate && python3 -m pytest great_ai",
+      "command": "source .env/bin/activate && python3 -m pytest .",
       "windows": {
-        "command": ".env\\bin\\activate.bat; python3 -m pytest great_ai"
+        "command": ".env\\bin\\activate.bat; python3 -m pytest ."
       },
       "group": "test",
       "presentation": {

README.md

@@ -1,3 +1,34 @@
-# GreatAI
+# **S**coutinScience **U**tilitie**S** for text processing [![Lint and test ScoutinScience utilities](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml/badge.svg)](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml)
 
-[![Quality Gate Status](https://sonar.scoutinscience.com/api/project_badges/measure?project=great-ai&metric=alert_status)](https://sonar.scoutinscience.com/dashboard?id=great-ai)
+> amogus
+
+## Exports
+
+- [clean](src/sus/clean.py)
+- [unique](src/sus/unique.py)
+- [parallel_map](src/sus/parallel_map.py)
+- [match_names](src/sus/match_names/match_names.py)
+- [evaluate_ranking](src/sus/evaluate_ranking/evaluate_ranking.py)
+- [get_sentences](src/sus/get_sentences.py)
+
+### Requires loading spacy model
+
+> This is automatic but will require some time.
+
+> Add this to the Dockerfile for caching the spaCy model:
+>
+> ```docker
+> RUN pip install --no-cache-dir en-core-web-sm@https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl
+> ```
+
+- [publication TEI](src/sus/publication_tei/publication_tei.py)
+- [lemmatize_text](src/sus/lemmatize_text.py)
+- [lemmatize_token](src/sus/lemmatize_token.py)
+- [spacy model (nlp)](src/sus/nlp.py)
+- [filter_sentences](src/sus/matcher/filter_sentences.py)
+
+## Development
+
+- Optional booleans must have a default value of `False`.
+- No imports in top-level `__init__.py`, in order to not load anything unnecessary automatically
+- Should only be updated through a PR

docs/simple-mag/train.ipynb

@@ -67,20 +67,17 @@
     "def preprocess(line: str) -> Tuple[str, str]:\n",
     "    data_point = json.loads(line)\n",
     "\n",
-    "    return (\n",
+    "    return (clean(data_point[\"text\"], convert_to_ascii=True), data_point[\"label\"])\n",
-    "        clean(data_point['text'], convert_to_ascii=True), \n",
-    "        data_point['label']\n",
-    "    )\n",
     "\n",
     "\n",
-    "with open('mag/train.txt', encoding='utf-8') as f:\n",
+    "with open(\"mag/train.txt\", encoding=\"utf-8\") as f:\n",
     "    training_data = parallel_map(preprocess, f.readlines())\n",
     "\n",
     "X_train = [d[0] for d in training_data]\n",
     "y_train = [d[1] for d in training_data]\n",
     "\n",
     "\n",
-    "with open('mag/test.txt', encoding='utf-8') as f:\n",
+    "with open(\"mag/test.txt\", encoding=\"utf-8\") as f:\n",
     "    test_data = parallel_map(preprocess, f.readlines())\n",
     "\n",
     "X_test = [d[0] for d in test_data]\n",

examples/simple/README.md

@@ -1,7 +0,0 @@
-# Train a domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)
-
-![steps](diagrams/scope.svg)
-
-- [Part 1](data.ipynb)
-- [Part 2](train.ipynb)
-- [Part 3](deploy.ipynb)

examples/simple/data.ipynb

@@ -1,227 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Train a domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)\n",
-    "> Part 1: obtain and clean data\n",
-    "\n",
-    "![position of this step in the lifecycle](diagrams/scope-data.svg)\n",
-    "> The blue boxes show the steps implemented in this notebook."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Extract\n",
-    "\n",
-    "This can be achieved by downloading a public dataset (such as in this case), or by having a Data Engineer setup and give us access to the organisation's data.\n",
-    "\n",
-    "In this case, we download the semantic scholar dataset from a public S3 bucket."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "MAX_CHUNK_COUNT = 1"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "'Processing 1 out of the 6002 available chunks'"
-      ]
-     },
-     "execution_count": 2,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "import urllib.request\n",
-    "from random import shuffle\n",
-    "\n",
-    "manifest = (\n",
-    "    urllib.request.urlopen(\n",
-    "        \"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/manifest.txt\"\n",
-    "    )\n",
-    "    .read()\n",
-    "    .decode()\n",
-    ")  # a list of available chunks separated by '\\n' characters\n",
-    "\n",
-    "lines = manifest.split()\n",
-    "shuffle(lines)\n",
-    "chunks = lines[:MAX_CHUNK_COUNT]\n",
-    "\n",
-    "f\"Processing {len(chunks)} out of the {len(manifest.split())} available chunks\""
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Transform\n",
-    "\n",
-    "- Filter out non-English abstracts using `great_ai.utilities.predict_language`\n",
-    "- Project it to only keep the necessary components (text and labels), clean the textual content using `great_ai.utilities.clean`\n",
-    "- We will speed up processing using `great_ai.utilities.parallel_map`."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\u001b[38;5;226m2022-06-25 11:20:01,955 |  WARNING | Limiting concurrency to 1 because there are only 1 chunks\u001b[0m\n",
-      "\u001b[38;5;39m2022-06-25 11:20:01,956 |     INFO | Starting parallel map (concurrency: 1, chunk size: 1)\u001b[0m\n",
-      "\u001b[38;5;226m2022-06-25 11:20:01,956 |  WARNING | Running in series, there is no reason for parallelism\u001b[0m\n",
-      "100%|██████████| 1/1 [04:02<00:00, 242.61s/it]\n"
-     ]
-    }
-   ],
-   "source": [
-    "from typing import List, Tuple\n",
-    "import json\n",
-    "import gzip\n",
-    "from great_ai import parallel_map, clean, is_english, predict_language\n",
-    "\n",
-    "\n",
-    "def preprocess_chunk(chunk_key: str) -> List[Tuple[str, List[str]]]:\n",
-    "    # Extract\n",
-    "    response = urllib.request.urlopen(\n",
-    "        f\"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/{chunk_key}\"\n",
-    "    )  # a gzipped JSON Lines file\n",
-    "\n",
-    "    decompressed = gzip.decompress(response.read())\n",
-    "    decoded = decompressed.decode()\n",
-    "    chunk = [json.loads(line) for line in decoded.split(\"\\n\") if line]\n",
-    "\n",
-    "    # Transform\n",
-    "    return [\n",
-    "        (\n",
-    "            clean(\n",
-    "                f'{c[\"title\"]} {c[\"paperAbstract\"]} {c[\"journalName\"]} {c[\"venue\"]}',\n",
-    "                convert_to_ascii=True,\n",
-    "            ),  # The text is cleaned to remove PDF extraction, web scraping, and other common artifacts\n",
-    "            c[\"fieldsOfStudy\"],\n",
-    "        )  # Create pairs of `(text, [...domains])`\n",
-    "        for c in chunk\n",
-    "        if c[\"fieldsOfStudy\"] and is_english(predict_language(c[\"paperAbstract\"]))\n",
-    "    ]\n",
-    "\n",
-    "\n",
-    "preprocessed_chunks = parallel_map(preprocess_chunk, chunks)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from itertools import chain\n",
-    "\n",
-    "preprocessed_data = list(chain(*preprocessed_chunks))\n",
-    "X, y = zip(\n",
-    "    *preprocessed_data\n",
-    ")  # X is the input, y is the expected (ground truth) output"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Load\n",
-    "\n",
-    "Upload the dataset (or a part of it) to a central repository using `great_ai.add_ground_truth`. This step automatically tags each datapoint with a split label according to the ratios we set. Additional tags can be also given.\n",
-    "\n",
-    "#### Production-ready backend\n",
-    "\n",
-    "The MongoDB driver is automatically configured if  `mongo.ini` exists with the following scheme:\n",
-    "\n",
-    "```ini\n",
-    "mongo_connection_string=mongodb://localhost:27017/\n",
-    "mongo_database=my_great_ai_db\n",
-    "```\n",
-    "> You can install MongoDB from [here](https://www.mongodb.com/docs/manual/installation) or [use it as a service](https://www.mongodb.com/cloud/atlas/register)\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\u001b[38;5;226m2022-06-25 11:24:04,668 |  WARNING | Environment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
-      "\u001b[38;5;39m2022-06-25 11:24:04,669 |     INFO | Found credentials file (/data/projects/great-ai/examples/simple/mongo.ini), initialising MongodbDriver\u001b[0m\n",
-      "\u001b[38;5;39m2022-06-25 11:24:04,670 |     INFO | Found credentials file (/data/projects/great-ai/examples/simple/mongo.ini), initialising LargeFileMongo\u001b[0m\n",
-      "\u001b[38;5;39m2022-06-25 11:24:04,671 |     INFO | Settings: configured ✅\u001b[0m\n",
-      "\u001b[38;5;39m2022-06-25 11:24:04,672 |     INFO | 🔩 tracing_database: MongodbDriver\u001b[0m\n",
-      "\u001b[38;5;39m2022-06-25 11:24:04,672 |     INFO | 🔩 large_file_implementation: LargeFileMongo\u001b[0m\n",
-      "\u001b[38;5;39m2022-06-25 11:24:04,673 |     INFO | 🔩 is_production: False\u001b[0m\n",
-      "\u001b[38;5;39m2022-06-25 11:24:04,673 |     INFO | 🔩 should_log_exception_stack: True\u001b[0m\n",
-      "\u001b[38;5;39m2022-06-25 11:24:04,674 |     INFO | 🔩 prediction_cache_size: 512\u001b[0m\n",
-      "\u001b[38;5;226m2022-06-25 11:24:04,674 |  WARNING | You still need to check whether you follow all best practices before trusting your deployment.\u001b[0m\n",
-      "\u001b[38;5;226m2022-06-25 11:24:04,674 |  WARNING | > Find out more at https://se-ml.github.io/practices/\u001b[0m\n"
-     ]
-    }
-   ],
-   "source": [
-    "from great_ai import add_ground_truth\n",
-    "\n",
-    "add_ground_truth(X, y, train_split_ratio=0.8, test_split_ratio=0.2)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Next: [Part 2](train.ipynb)"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3.10.4 ('.env': venv)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.4"
-  },
-  "orig_nbformat": 4,
-  "vscode": {
-   "interpreter": {
-    "hash": "02dd6d3afbfa9fbbe1037d64ad9014965528a1ccad21929d6e72f466389a68ad"
-   }
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}

examples/simple/mongo.ini

@@ -1,2 +0,0 @@
-mongo_connection_string=mongodb://localhost:27017/  # change this
-mongo_database=great_ai_db2                          # this will be automatically created

great_ai/.gitignore

@@ -1 +0,0 @@
-build

great_ai/README.md

@@ -1,34 +0,0 @@
-# **S**coutinScience **U**tilitie**S** for text processing [![Lint and test ScoutinScience utilities](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml/badge.svg)](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml)
-
-> amogus
-
-## Exports
-
-- [clean](src/sus/clean.py)
-- [unique](src/sus/unique.py)
-- [parallel_map](src/sus/parallel_map.py)
-- [match_names](src/sus/match_names/match_names.py)
-- [evaluate_ranking](src/sus/evaluate_ranking/evaluate_ranking.py)
-- [get_sentences](src/sus/get_sentences.py)
-
-### Requires loading spacy model
-
-> This is automatic but will require some time.
-
-> Add this to the Dockerfile for caching the spaCy model:
->
-> ```docker
-> RUN pip install --no-cache-dir en-core-web-sm@https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl
-> ```
-
-- [publication TEI](src/sus/publication_tei/publication_tei.py)
-- [lemmatize_text](src/sus/lemmatize_text.py)
-- [lemmatize_token](src/sus/lemmatize_token.py)
-- [spacy model (nlp)](src/sus/nlp.py)
-- [filter_sentences](src/sus/matcher/filter_sentences.py)
-
-## Development
-
-- Optional booleans must have a default value of `False`.
-- No imports in top-level `__init__.py`, in order to not load anything unnecessary automatically
-- Should only be updated through a PR

great_ai/src/great_ai/utilities/config_file/pattern.py

@@ -1,18 +0,0 @@
-import re
-
-pattern = re.compile(
-    r"""
-    \s*             # leading whitespace is allowed
-    (\w+?)          # then comes the key
-    \s*=\s*         # the key and value are separated by an equal sign
-    (?:             # then comes the value
-         "([^"]*)"     # the value can be surrounded by quotes: "value"
-       | '([^']*)'     # the value can be surrounded by quotes: 'value'
-       | `([^`]*)`     # the value can be surrounded by quotes: `value`
-       | ([^#\n]*?)    # or it is bare, in that case, the trailing whitespace is ignored
-    )
-    \s*(?:\#.*)?        # comments can be added with the `#` symbol
-    (?:\n|$)            # a key-value pairs are separated by new lines
-""",
-    flags=re.UNICODE | re.VERBOSE,
-)

setup.cfg

@@ -1,14 +1,14 @@
 [metadata]
-name = great-ai
+name = great_ai
 version = 0.0.1
 author = András Schmelczer
 author_email = andras@scoutinscience.com
 description = 
 long_description = file: README.md
 long_description_content_type = text/markdown
-url = https://github.com/ScoutinScience/great-ai
+url = https://github.com/ScoutinScience/great_ai
 project_urls =
-    Bug Tracker = https://github.com/ScoutinScience/great-ai/issues
+    Bug Tracker = https://github.com/ScoutinScience/great_ai/issues
 classifiers =
     Programming Language :: Python :: 3
     Operating System :: OS Independent

src/great_ai/great_ai/constants.py

@@ -15,7 +15,7 @@ DEFAULT_LARGE_FILE_CONFIG_PATHS = {
     LargeFileMongo: MONGO_CONFIG_PATHS,
 }
 
-GITHUB_LINK = "https://github.com/ScoutinScience/great-ai"
+GITHUB_LINK = "https://github.com/ScoutinScience/great_ai"
 
 TRAIN_SPLIT_TAG_NAME = "train"
 TEST_SPLIT_TAG_NAME = "test"

src/great_ai/great_ai/context.py

@@ -6,9 +6,8 @@ from typing import Any, Dict, Optional, Type, cast
 
 from pydantic import BaseModel
 
-from great_ai.large_file import LargeFile, LargeFileLocal
+from ..large_file import LargeFile, LargeFileLocal
-from great_ai.utilities import get_logger
+from ..utilities import get_logger
-
 from .constants import (
     DEFAULT_LARGE_FILE_CONFIG_PATHS,
     DEFAULT_TRACING_DATABASE_CONFIG_PATHS,

src/great_ai/great_ai/deploy/great_ai.py

@@ -16,10 +16,7 @@ from typing import (
 from fastapi import APIRouter, FastAPI, status
 from pydantic import BaseModel, create_model
 
-from great_ai.great_ai.deploy.routes.bootstrap_dashboard import bootstrap_dashboard
+from ...utilities import parallel_map
-from great_ai.great_ai.views.cache_statistics import CacheStatistics
-from great_ai.utilities import parallel_map
-
 from ..constants import DASHBOARD_PATH
 from ..context import get_context
 from ..helper import (
@@ -30,12 +27,13 @@ from ..helper import (
 )
 from ..parameters import automatically_decorate_parameters
 from ..tracing.tracing_context import TracingContext
-from ..views import ApiMetadata, HealthCheckResponse, Trace
+from ..views import ApiMetadata, CacheStatistics, HealthCheckResponse, Trace
 from .routes import (
     bootstrap_docs_endpoints,
     bootstrap_feedback_endpoints,
     bootstrap_trace_endpoints,
 )
+from .routes.bootstrap_dashboard import bootstrap_dashboard
 
 T = TypeVar("T")
 

src/great_ai/great_ai/deploy/routes/dashboard/create_dash_app.py

@@ -8,8 +8,7 @@ from dash import Dash, dcc, html
 from dash.dependencies import Input, Output
 from flask import Flask
 
-from great_ai.utilities import unique
+from .....utilities import unique
-
 from ....constants import DASHBOARD_PATH, ONLINE_TAG_NAME
 from ....context import get_context
 from ....helper import snake_case_to_text, text_to_hex_color

src/great_ai/great_ai/deploy/routes/dashboard/get_traces_table.py

@@ -1,6 +1,6 @@
 from dash import dash_table
 
-from great_ai.great_ai.context import get_context
+from ....context import get_context
 
 
 def get_traces_table() -> dash_table.DataTable: