Format

Signed-off-by: András Schmelczer <andras@schmelczer.dev>

check-python.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+set -e
+
+echo "Installing dependencies if necessary"
+python3 -m pip install --upgrade autoflake isort black black[jupyter] mypy flake8
+
+echo "Checking $1"
+
+python3 -m autoflake --expand-star-imports --remove-all-unused-imports --ignore-init-module-imports --remove-unused-variables --in-place -r $1 --check
+python3 -m isort --profile black --skip .env $1 --check
+python3 -m black $1 --exclude .env --check
+
+yes | python3 -m mypy $1 --install-types > /dev/null || true
+python3 -m mypy --namespace-packages --ignore-missing-imports --install-types --non-interactive --disallow-untyped-defs --disallow-incomplete-defs --follow-imports=silent --exclude=external/ --exclude=/build/ --pretty $1
+
+python3 -m flake8 $1 --count --show-source --statistics --exclude=__init__.py,.env,external --ignore=E501,E402,F821,W503,E722,E203

example/config.py

@@ -1 +1 @@
-model_key = 
+model_key = "domain-prediction-v2"

example/helper/preprocess.py

@@ -1,12 +1,10 @@
+import re
+
 from sus.clean import clean
 from sus.lemmatize_text import lemmatize_text
-import re
 
 
 def preprocess(text: str) -> str:
     cleaned = clean(text, convert_to_ascii=True)
-    lemmas = [
+    lemmas = [re.sub(r"\d+", "NUM", lemma) for lemma in lemmatize_text(cleaned)]
-        re.sub(r'\d+', 'NUM', lemma)
-        for lemma in lemmatize_text(cleaned)
-    ]
     return " ".join(lemmas)

example/models/domain_prediction.py

@@ -1,6 +1,7 @@
-from pydantic import BaseModel
 from typing import List
 
+from pydantic import BaseModel
+
 
 class DomainPrediction(BaseModel):
     domain: str

example/predict.py

@@ -1,42 +1,46 @@
-
-from models import DomainPrediction
-from typing import Iterable, List, Dict
-from sklearn.pipeline import Pipeline 
-from helper import preprocess
 import re
+from typing import Dict, Iterable, List
+
+from helper import preprocess
+from models import DomainPrediction
+from sklearn.pipeline import Pipeline
+
 # from sus.use_model import use_model
-from config import model_key
 
 
 # @use_model(model_key, version="latest")
-def predict(text: str, model: Pipeline, cut_off_probability: float=0.2) -> List[DomainPrediction]:
+def predict(
+    text: str, model: Pipeline, cut_off_probability: float = 0.2
+) -> List[DomainPrediction]:
     assert 0 <= cut_off_probability <= 1
 
-    feature_names = model.named_steps['vectorizer'].get_feature_names_out()
+    feature_names = model.named_steps["vectorizer"].get_feature_names_out()
 
     token_mapping = {
         preprocess(original): original
-        for original in re.sub(r'[^a-zA-Z0-9]', ' ', text).split(' ')
+        for original in re.sub(r"[^a-zA-Z0-9]", " ", text).split(" ")
     }
 
-    features = model.named_steps['vectorizer'].transform([text])
+    features = model.named_steps["vectorizer"].transform([text])
-    prediction = model.named_steps['classifier'].predict_proba(features)[0]
+    prediction = model.named_steps["classifier"].predict_proba(features)[0]
     best_classes = sorted(enumerate(prediction), key=lambda v: v[1], reverse=True)
 
     results: List[DomainPrediction] = []
     for class_index, probability in best_classes:
-        weights = model.named_steps['classifier'].feature_log_prob_[class_index]
+        weights = model.named_steps["classifier"].feature_log_prob_[class_index]
 
-        results.append(DomainPrediction(
+        results.append(
-            domain=model.named_steps['classifier'].classes_[class_index],
+            DomainPrediction(
-            probability=round(probability * 100),
+                domain=model.named_steps["classifier"].classes_[class_index],
-            explanation=_get_explanation(
+                probability=round(probability * 100),
-                feature_names=feature_names,
+                explanation=_get_explanation(
-                features=features.A[0],
+                    feature_names=feature_names,
-                weights=weights,
+                    features=features.A[0],
-                token_mapping=token_mapping
+                    weights=weights,
+                    token_mapping=token_mapping,
+                ),
             )
-        ))
+        )
 
         if sum(r.probability for r in results) >= cut_off_probability:
             break
@@ -44,7 +48,12 @@ def predict(text: str, model: Pipeline, cut_off_probability: float=0.2) -> List[
     return results
 
 
-def _get_explanation(feature_names: Iterable[str], features: Iterable[float], weights: Iterable[float], token_mapping: Dict[str, str]) -> List[str]:
+def _get_explanation(
+    feature_names: Iterable[str],
+    features: Iterable[float],
+    weights: Iterable[float],
+    token_mapping: Dict[str, str],
+) -> List[str]:
     influential = [
         (value * weight, name)
         for name, value, weight in zip(feature_names, features, weights)

example/train.ipynb

@@ -85,10 +85,7 @@
    "source": [
     "def preprocess(text: str) -> str:\n",
     "    cleaned = clean(text, convert_to_ascii=True)\n",
-    "    lemmas = [\n",
+    "    lemmas = [re.sub(r\"\\d+\", \"NUM\", lemma) for lemma in lemmatize_text(cleaned)]\n",
-    "        re.sub(r'\\d+', 'NUM', lemma)\n",
-    "        for lemma in lemmatize_text(cleaned)\n",
-    "    ]\n",
     "    return \" \".join(lemmas)"
    ]
   },
@@ -157,14 +154,14 @@
    "source": [
     "corpora = list(SS_CORPUS_PATH.glob(f\"{PREFIX}*.json\"))\n",
     "shuffle(corpora)\n",
-    "print(f'Found {len(corpora)} files')\n",
+    "print(f\"Found {len(corpora)} files\")\n",
     "\n",
     "data = []\n",
     "for p in corpora[:MAX_FILE_COUNT]:\n",
     "    with open(p) as f:\n",
     "        data.extend(json.load(f).items())\n",
     "\n",
-    "print(f'Found {len(data)} documents')"
+    "print(f\"Found {len(data)} documents\")"
    ]
   },
   {
@@ -174,23 +171,12 @@
    "outputs": [],
    "source": [
     "X_train, X_test, y_train, y_test = train_test_split(\n",
-    "    [d[0] for d in data],\n",
+    "    [d[0] for d in data], [d[1] for d in data], test_size=0.1, random_state=SEED\n",
-    "    [d[1] for d in data],\n",
-    "    test_size=0.1, \n",
-    "    random_state=SEED\n",
     ")\n",
     "\n",
-    "X_train = [\n",
+    "X_train = [x for x, y in zip(X_train, y_train) for domain in y]\n",
-    "    x\n",
-    "    for x, y in zip(X_train, y_train)\n",
-    "    for domain in y\n",
-    "]\n",
     "\n",
-    "y_train = [\n",
+    "y_train = [domain for x, y in zip(X_train, y_train) for domain in y]"
-    "    domain\n",
-    "    for x, y in zip(X_train, y_train)\n",
-    "    for domain in y\n",
-    "]"
    ]
   },
   {
@@ -207,23 +193,23 @@
    "outputs": [],
    "source": [
     "classifier = GridSearchCV(\n",
-    "    Pipeline(steps=[\n",
+    "    Pipeline(steps=[(\"vectorizer\", TfidfVectorizer()), (\"classifier\", ComplementNB())]),\n",
-    "        ('vectorizer', TfidfVectorizer()),\n",
-    "        ('classifier', ComplementNB())\n",
-    "    ]),\n",
     "    {\n",
     "        \"vectorizer__max_df\": [0.05, 0.1, 0.3],\n",
     "        \"vectorizer__min_df\": [5, 10, 30, 100],\n",
     "        \"vectorizer__sublinear_tf\": [True, False],\n",
     "        \"classifier__alpha\": [0.001, 0.1, 0.5, 1],\n",
-    "        \"classifier__fit_prior\": [True, False]\n",
+    "        \"classifier__fit_prior\": [True, False],\n",
     "    },\n",
     "    scoring=\"f1_macro\",\n",
-    "        cv=3,\n",
+    "    cv=3,\n",
-    "        n_jobs=12,\n",
+    "    n_jobs=12,\n",
-    "        verbose=7,\n",
+    "    verbose=7,\n",
+    ")\n",
+    "classifier.fit(\n",
+    "    X_train[:HYPERPARAMETER_OPTIMISATION_SIZE],\n",
+    "    y_train[:HYPERPARAMETER_OPTIMISATION_SIZE],\n",
     ")\n",
-    "classifier.fit(X_train[:HYPERPARAMETER_OPTIMISATION_SIZE], y_train[:HYPERPARAMETER_OPTIMISATION_SIZE])\n",
     "\n",
     "results = pd.DataFrame(classifier.cv_results_)\n",
     "results.sort_values(\"rank_test_score\")"
@@ -247,10 +233,12 @@
     }
    ],
    "source": [
-    "classifier = Pipeline(steps=[\n",
+    "classifier = Pipeline(\n",
-    "    ('vectorizer', TfidfVectorizer(min_df=10, max_df=0.05)),\n",
+    "    steps=[\n",
-    "    ('classifier', ComplementNB(alpha=0.5, fit_prior=False))\n",
+    "        (\"vectorizer\", TfidfVectorizer(min_df=10, max_df=0.05)),\n",
-    "])\n",
+    "        (\"classifier\", ComplementNB(alpha=0.5, fit_prior=False)),\n",
+    "    ]\n",
+    ")\n",
     "\n",
     "classifier.fit(X_train, y_train)"
    ]
@@ -310,7 +298,11 @@
     "\n",
     "print(metrics.classification_report(y_test_aligned, predicted))\n",
     "metrics.ConfusionMatrixDisplay.from_predictions(\n",
-    "    y_true=y_test_aligned, y_pred=predicted, xticks_rotation=\"vertical\", normalize=\"pred\", values_format='.2f'\n",
+    "    y_true=y_test_aligned,\n",
+    "    y_pred=predicted,\n",
+    "    xticks_rotation=\"vertical\",\n",
+    "    normalize=\"pred\",\n",
+    "    values_format=\".2f\",\n",
     ")\n",
     "None"
    ]
@@ -333,7 +325,7 @@
    "outputs": [],
    "source": [
     "for X, y in zip(X_test[:50], y_test):\n",
-    "    print(', '.join(y))\n",
+    "    print(\", \".join(y))\n",
     "    pprint(predict(X))\n",
     "    print()"
    ]

format-python.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -e
+
+echo "Installing dependencies if necessary"
+python3 -m pip install --upgrade autoflake isort black black[jupyter] mypy
+
+echo "Formatting and checking $1"
+
+echo Running autoflake
+python3 -m autoflake --expand-star-imports --remove-all-unused-imports --ignore-init-module-imports --remove-unused-variables --in-place -r $1
+
+echo Running isort
+python3 -m isort --profile black --skip .env $1
+
+echo Running black
+python3 -m black $1 --exclude .env
+
+echo Running mypy
+python3 -m mypy --namespace-packages --ignore-missing-imports --install-types --non-interactive --disallow-untyped-defs --disallow-incomplete-defs --pretty --follow-imports=silent --exclude=external/ --exclude=/build/ $1

good_ai/src/open_s3/helper/__init__.py

@@ -1,2 +1,2 @@
-from .progress_bar import DownloadProgressBar, UploadProgressBar
 from .human_readable_to_byte import human_readable_to_byte
+from .progress_bar import DownloadProgressBar, UploadProgressBar

good_ai/src/open_s3/helper/human_readable_to_byte.py

@@ -27,5 +27,5 @@ def human_readable_to_byte(size: str) -> int:
 
     scalar = float(results["scalar"])
     idx = possible_units.index(results["unit"].upper())
-    factor = 1024 ** idx
+    factor = 1024**idx
     return round(scalar * factor)

good_ai/src/open_s3/helper/progress_bar.py

@@ -1,40 +1,35 @@
 import os
 import threading
 from logging import Logger
-from typing import IO, Any, Optional
-
-from tqdm.auto import tqdm
 from pathlib import Path
 
-import os
-import sys
-import threading
-
 
 class ProgressBar:
     def __init__(self, file_size: int, logger: Logger, prefix: str):
         self._file_size = file_size
         self._logger = logger
-        self._prefix=prefix
+        self._prefix = prefix
 
         self._seen_so_far = 0
         self._lock = threading.Lock()
 
-    def __call__(self, bytes_amount: int):
+    def __call__(self, bytes_amount: int) -> None:
         with self._lock:
             self._seen_so_far += bytes_amount
             percentage = (self._seen_so_far / float(self._file_size)) * 100
             size_length = len(str(self._file_size))
             progress = str(self._seen_so_far).rjust(size_length)
-            self._logger.info(f"{self._prefix} {progress}/{self._file_size} bytes ({percentage:.1f}%)")
+            self._logger.info(
+                f"{self._prefix} {progress}/{self._file_size} bytes ({percentage:.1f}%)"
+            )
 
 
 class DownloadProgressBar(ProgressBar):
     def __init__(self, name: str, size: int, logger: Logger):
-        super().__init__(file_size=size, logger=logger, prefix=f'Downloading {name}')
+        super().__init__(file_size=size, logger=logger, prefix=f"Downloading {name}")
 
 
 class UploadProgressBar(ProgressBar):
     def __init__(self, path: Path, logger: Logger):
         size = os.path.getsize(path)
-        super().__init__(file_size=size, logger=logger, prefix=f'Uploading {path.name}')
+        super().__init__(file_size=size, logger=logger, prefix=f"Uploading {path.name}")

good_ai/src/open_s3/large_file.py

@@ -6,9 +6,8 @@ import tempfile
 from pathlib import Path
 from types import TracebackType
 from typing import IO, Any, Dict, List, Optional, Type, Union
-import boto3
-from botocore.exceptions import ClientError
 
+import boto3
 from helper import DownloadProgressBar, UploadProgressBar, human_readable_to_byte
 
 logger = logging.getLogger("open_s3")
@@ -62,7 +61,7 @@ class LargeFile:
         newline: Optional[str] = None,
         version: Optional[int] = None,
         keep_last_n: Optional[int] = None,
-        offline_mode: bool = False
+        offline_mode: bool = False,
     ):
         self._name: str = name
         self._version = version
@@ -175,8 +174,17 @@ class LargeFile:
             with tempfile.TemporaryDirectory() as tmp:
                 tmp_file_archive = Path(tmp) / f"{self._local_name}.tar.gz"
 
-                size = self._client.head_object(Bucket=self.bucket_name, Key=key)['ContentLength']
+                size = self._client.head_object(Bucket=self.bucket_name, Key=key)[
-                self._client.download_file(Bucket=self.bucket_name, Key=key, Filename=str(tmp_file_archive), Callback=None if hide_progress else DownloadProgressBar(size=size, name=key, logger=logger))
+                    "ContentLength"
+                ]
+                self._client.download_file(
+                    Bucket=self.bucket_name,
+                    Key=key,
+                    Filename=str(tmp_file_archive),
+                    Callback=None
+                    if hide_progress
+                    else DownloadProgressBar(size=size, name=key, logger=logger),
+                )
                 logger.info(f"Decompressing {self._local_name}")
                 shutil.unpack_archive(str(tmp_file_archive), tmp, "gztar")
                 tmp_file = Path(tmp) / self._local_name
@@ -193,7 +201,7 @@ class LargeFile:
         with tempfile.TemporaryDirectory() as tmp:
             if path.is_file():
                 logger.info(f"Copying file for {self._local_name}")
-                copy = shutil.copy 
+                copy: Any = shutil.copy
             else:
                 logger.info(f"Copying directory for {self._local_name}")
                 copy = shutil.copytree
@@ -216,7 +224,14 @@ class LargeFile:
             logger.info(f"Uploading {self._local_name} to S3 from {path}")
 
             file_to_be_uploaded = Path(tmp) / f"{self._local_name}.tar.gz"
-            self._client.upload_file(Filename=str(file_to_be_uploaded), Bucket=self.bucket_name, Key=self._s3_name, Callback=None if hide_progress else UploadProgressBar(file_to_be_uploaded, logger=logger))
+            self._client.upload_file(
+                Filename=str(file_to_be_uploaded),
+                Bucket=self.bucket_name,
+                Key=self._s3_name,
+                Callback=None
+                if hide_progress
+                else UploadProgressBar(file_to_be_uploaded, logger=logger),
+            )
 
         self.clean_up()
 
@@ -239,7 +254,13 @@ class LargeFile:
                 "Please configure the S3 access options by calling LargeFile.configure_credentials or set offline_mode=True in the constructor."
             )
 
-        self._client = boto3.client('s3', aws_access_key_id=self.access_key_id, aws_secret_access_key=self.secret_access_key, region_name=self.region_name, endpoint_url=self.endpoint_url)
+        self._client = boto3.client(
+            "s3",
+            aws_access_key_id=self.access_key_id,
+            aws_secret_access_key=self.secret_access_key,
+            region_name=self.region_name,
+            endpoint_url=self.endpoint_url,
+        )
 
     def _find_versions(self) -> None:
         if self._offline_mode:
@@ -257,8 +278,7 @@ class LargeFile:
         logger.info(f"Fetching offline versions of {self._name}")
 
         self._versions = [
-            path
+            path for path in self.cache_path.glob(f"{self._local_name}-*")
-            for path in self.cache_path.glob(f'{self._local_name}-*')
         ]
 
     def _fetch_versions_from_s3(self) -> None:
@@ -267,10 +287,7 @@ class LargeFile:
             Bucket=self.bucket_name, Prefix=self._name
         )
         self._versions = (
-            sorted(
+            sorted(o["Key"] for o in found_objects["Contents"])
-                o["Key"]
-                for o in found_objects["Contents"]
-            )
             if "Contents" in found_objects
             else []
         )
@@ -316,10 +333,13 @@ class LargeFile:
             return int(key.name.split("-")[-1])
         return int(key.split("/")[-1])
 
-
     def _delete_old_versions_from_s3(self) -> None:
         if self._keep_last_n is not None:
-            for key in (self._versions[: -self._keep_last_n] if self._keep_last_n > 0 else self._versions):
+            for key in (
+                self._versions[: -self._keep_last_n]
+                if self._keep_last_n > 0
+                else self._versions
+            ):
                 logger.info(
                     f"Removing old version (keep_last_n={self._keep_last_n}): {key}"
                 )

good_ai/tests/open_s3/test_human_readable_to_byte.py

@@ -4,22 +4,22 @@ from src.open_s3.helper import human_readable_to_byte
 
 
 class TestHumanReadableToByte(unittest.TestCase):
-    def test_simple_cases(self):
+    def test_simple_cases(self) -> None:
         self.assertEqual(human_readable_to_byte("1KB"), 1024)
         self.assertEqual(human_readable_to_byte("2KB"), 2048)
 
-    def test_fractions(self):
+    def test_fractions(self) -> None:
         self.assertEqual(human_readable_to_byte("0.5KB"), 512)
         self.assertEqual(human_readable_to_byte("20.5KB"), 1024 * 20 + 512)
 
-    def test_formating(self):
+    def test_formating(self) -> None:
         self.assertEqual(human_readable_to_byte(" 1MB"), 1024 * 1024)
         self.assertEqual(human_readable_to_byte(" 2  MB"), 1024 * 1024 * 2)
         self.assertEqual(human_readable_to_byte("    4   MB "), 1024 * 1024 * 4)
         self.assertEqual(human_readable_to_byte("8MB    "), 1024 * 1024 * 8)
         self.assertEqual(human_readable_to_byte(" 1.5   MB  "), 1024 * 1024 * 1.5)
 
-    def test_casing(self):
+    def test_casing(self) -> None:
         self.assertEqual(human_readable_to_byte("0.5GB"), 0.5 * 1024 * 1024 * 1024)
         self.assertEqual(human_readable_to_byte("0.5gB"), 0.5 * 1024 * 1024 * 1024)
         self.assertEqual(human_readable_to_byte("0.5Gb"), 0.5 * 1024 * 1024 * 1024)

good_ai/tests/open_s3/test_large_file.py

@@ -1,5 +1,6 @@
-from pathlib import Path
 import unittest
+from pathlib import Path
+from typing import Any
 from unittest.mock import Mock, create_autospec, patch
 
 import botocore.session
@@ -14,23 +15,22 @@ credentials = {
     "aws_access_key_id": "YOUR_ACCESS_KEY_ID",
     "aws_secret_access_key": "YOUR_VERY_SECRET_ACCESS_KEY",
     "large_files_bucket_name": "create_a_bucket_and_put_its_name_here",
-    "other_key": 23,
     "endpoint_url": "this is optional, for backblaze, use this: https://s3.us-west-002.backblazeb2.com",
 }
 
 
 class TestLargeFile(unittest.TestCase):
-    def test_uninitialized(self):
+    def test_uninitialized(self) -> None:
         self.assertRaises(ValueError, LargeFile, "test-file")
 
-    def test_bad_file_modes(self):
+    def test_bad_file_modes(self) -> None:
         self.assertRaises(ValueError, LargeFile, "test-file", "w", version=3)
         self.assertRaises(ValueError, LargeFile, "test-file", "wb", version=3)
         self.assertRaises(ValueError, LargeFile, "test-file", "w+r")
         self.assertRaises(ValueError, LargeFile, "test-file", "test")
 
     @patch("botocore.session")
-    def test_initialized_with_dict(self, session):
+    def test_initialized_with_dict(self, session) -> None:
         session_mock = Mock()
         session.get_session = create_autospec(
             botocore.session.get_session, return_value=session_mock
@@ -79,7 +79,7 @@ class TestLargeFile(unittest.TestCase):
         self.assertEqual(lf._s3_name, "test-file/2")
 
     @patch("botocore.session")
-    def test_initialized_with_file(self, session):
+    def test_initialized_with_file(self, session: Any) -> None:
         session_mock = Mock()
         session.get_session = create_autospec(
             botocore.session.get_session, return_value=session_mock