diff --git a/check-python.sh b/check-python.sh new file mode 100755 index 0000000..04365a0 --- /dev/null +++ b/check-python.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +set -e + +echo "Installing dependencies if necessary" +python3 -m pip install --upgrade autoflake isort black black[jupyter] mypy flake8 + +echo "Checking $1" + +python3 -m autoflake --expand-star-imports --remove-all-unused-imports --ignore-init-module-imports --remove-unused-variables --in-place -r $1 --check +python3 -m isort --profile black --skip .env $1 --check +python3 -m black $1 --exclude .env --check + +yes | python3 -m mypy $1 --install-types > /dev/null || true +python3 -m mypy --namespace-packages --ignore-missing-imports --install-types --non-interactive --disallow-untyped-defs --disallow-incomplete-defs --follow-imports=silent --exclude=external/ --exclude=/build/ --pretty $1 + +python3 -m flake8 $1 --count --show-source --statistics --exclude=__init__.py,.env,external --ignore=E501,E402,F821,W503,E722,E203 diff --git a/example/config.py b/example/config.py index 5276f11..143bb15 100644 --- a/example/config.py +++ b/example/config.py @@ -1 +1 @@ -model_key = \ No newline at end of file +model_key = "domain-prediction-v2" diff --git a/example/helper/preprocess.py b/example/helper/preprocess.py index 8d9e70f..2afd1b9 100644 --- a/example/helper/preprocess.py +++ b/example/helper/preprocess.py @@ -1,12 +1,10 @@ +import re + from sus.clean import clean from sus.lemmatize_text import lemmatize_text -import re def preprocess(text: str) -> str: cleaned = clean(text, convert_to_ascii=True) - lemmas = [ - re.sub(r'\d+', 'NUM', lemma) - for lemma in lemmatize_text(cleaned) - ] + lemmas = [re.sub(r"\d+", "NUM", lemma) for lemma in lemmatize_text(cleaned)] return " ".join(lemmas) diff --git a/example/models/domain_prediction.py b/example/models/domain_prediction.py index 4dce180..0ca53c1 100644 --- a/example/models/domain_prediction.py +++ b/example/models/domain_prediction.py @@ -1,8 +1,9 @@ -from pydantic import BaseModel from typing import List +from pydantic import BaseModel + class DomainPrediction(BaseModel): domain: str probability: float - explanation: List[str] \ No newline at end of file + explanation: List[str] diff --git a/example/predict.py b/example/predict.py index e6c3fc8..ebd1796 100644 --- a/example/predict.py +++ b/example/predict.py @@ -1,56 +1,65 @@ - -from models import DomainPrediction -from typing import Iterable, List, Dict -from sklearn.pipeline import Pipeline -from helper import preprocess import re +from typing import Dict, Iterable, List + +from helper import preprocess +from models import DomainPrediction +from sklearn.pipeline import Pipeline + # from sus.use_model import use_model -from config import model_key # @use_model(model_key, version="latest") -def predict(text: str, model: Pipeline, cut_off_probability: float=0.2) -> List[DomainPrediction]: +def predict( + text: str, model: Pipeline, cut_off_probability: float = 0.2 +) -> List[DomainPrediction]: assert 0 <= cut_off_probability <= 1 - feature_names = model.named_steps['vectorizer'].get_feature_names_out() + feature_names = model.named_steps["vectorizer"].get_feature_names_out() token_mapping = { preprocess(original): original - for original in re.sub(r'[^a-zA-Z0-9]', ' ', text).split(' ') + for original in re.sub(r"[^a-zA-Z0-9]", " ", text).split(" ") } - - features = model.named_steps['vectorizer'].transform([text]) - prediction = model.named_steps['classifier'].predict_proba(features)[0] + + features = model.named_steps["vectorizer"].transform([text]) + prediction = model.named_steps["classifier"].predict_proba(features)[0] best_classes = sorted(enumerate(prediction), key=lambda v: v[1], reverse=True) results: List[DomainPrediction] = [] for class_index, probability in best_classes: - weights = model.named_steps['classifier'].feature_log_prob_[class_index] - - results.append(DomainPrediction( - domain=model.named_steps['classifier'].classes_[class_index], - probability=round(probability * 100), - explanation=_get_explanation( - feature_names=feature_names, - features=features.A[0], - weights=weights, - token_mapping=token_mapping + weights = model.named_steps["classifier"].feature_log_prob_[class_index] + + results.append( + DomainPrediction( + domain=model.named_steps["classifier"].classes_[class_index], + probability=round(probability * 100), + explanation=_get_explanation( + feature_names=feature_names, + features=features.A[0], + weights=weights, + token_mapping=token_mapping, + ), ) - )) + ) if sum(r.probability for r in results) >= cut_off_probability: break - + return results -def _get_explanation(feature_names: Iterable[str], features: Iterable[float], weights: Iterable[float], token_mapping: Dict[str, str]) -> List[str]: +def _get_explanation( + feature_names: Iterable[str], + features: Iterable[float], + weights: Iterable[float], + token_mapping: Dict[str, str], +) -> List[str]: influential = [ (value * weight, name) for name, value, weight in zip(feature_names, features, weights) if value ] - + most_influential = sorted(influential, reverse=True)[:5] return [token_mapping[v[1]] for v in most_influential] diff --git a/example/train.ipynb b/example/train.ipynb index 0a8c3e1..7a2dd39 100644 --- a/example/train.ipynb +++ b/example/train.ipynb @@ -85,10 +85,7 @@ "source": [ "def preprocess(text: str) -> str:\n", " cleaned = clean(text, convert_to_ascii=True)\n", - " lemmas = [\n", - " re.sub(r'\\d+', 'NUM', lemma)\n", - " for lemma in lemmatize_text(cleaned)\n", - " ]\n", + " lemmas = [re.sub(r\"\\d+\", \"NUM\", lemma) for lemma in lemmatize_text(cleaned)]\n", " return \" \".join(lemmas)" ] }, @@ -157,14 +154,14 @@ "source": [ "corpora = list(SS_CORPUS_PATH.glob(f\"{PREFIX}*.json\"))\n", "shuffle(corpora)\n", - "print(f'Found {len(corpora)} files')\n", + "print(f\"Found {len(corpora)} files\")\n", "\n", "data = []\n", "for p in corpora[:MAX_FILE_COUNT]:\n", " with open(p) as f:\n", " data.extend(json.load(f).items())\n", "\n", - "print(f'Found {len(data)} documents')" + "print(f\"Found {len(data)} documents\")" ] }, { @@ -174,23 +171,12 @@ "outputs": [], "source": [ "X_train, X_test, y_train, y_test = train_test_split(\n", - " [d[0] for d in data],\n", - " [d[1] for d in data],\n", - " test_size=0.1, \n", - " random_state=SEED\n", + " [d[0] for d in data], [d[1] for d in data], test_size=0.1, random_state=SEED\n", ")\n", "\n", - "X_train = [\n", - " x\n", - " for x, y in zip(X_train, y_train)\n", - " for domain in y\n", - "]\n", + "X_train = [x for x, y in zip(X_train, y_train) for domain in y]\n", "\n", - "y_train = [\n", - " domain\n", - " for x, y in zip(X_train, y_train)\n", - " for domain in y\n", - "]" + "y_train = [domain for x, y in zip(X_train, y_train) for domain in y]" ] }, { @@ -207,23 +193,23 @@ "outputs": [], "source": [ "classifier = GridSearchCV(\n", - " Pipeline(steps=[\n", - " ('vectorizer', TfidfVectorizer()),\n", - " ('classifier', ComplementNB())\n", - " ]),\n", + " Pipeline(steps=[(\"vectorizer\", TfidfVectorizer()), (\"classifier\", ComplementNB())]),\n", " {\n", " \"vectorizer__max_df\": [0.05, 0.1, 0.3],\n", " \"vectorizer__min_df\": [5, 10, 30, 100],\n", " \"vectorizer__sublinear_tf\": [True, False],\n", " \"classifier__alpha\": [0.001, 0.1, 0.5, 1],\n", - " \"classifier__fit_prior\": [True, False]\n", + " \"classifier__fit_prior\": [True, False],\n", " },\n", " scoring=\"f1_macro\",\n", - " cv=3,\n", - " n_jobs=12,\n", - " verbose=7,\n", + " cv=3,\n", + " n_jobs=12,\n", + " verbose=7,\n", + ")\n", + "classifier.fit(\n", + " X_train[:HYPERPARAMETER_OPTIMISATION_SIZE],\n", + " y_train[:HYPERPARAMETER_OPTIMISATION_SIZE],\n", ")\n", - "classifier.fit(X_train[:HYPERPARAMETER_OPTIMISATION_SIZE], y_train[:HYPERPARAMETER_OPTIMISATION_SIZE])\n", "\n", "results = pd.DataFrame(classifier.cv_results_)\n", "results.sort_values(\"rank_test_score\")" @@ -247,10 +233,12 @@ } ], "source": [ - "classifier = Pipeline(steps=[\n", - " ('vectorizer', TfidfVectorizer(min_df=10, max_df=0.05)),\n", - " ('classifier', ComplementNB(alpha=0.5, fit_prior=False))\n", - "])\n", + "classifier = Pipeline(\n", + " steps=[\n", + " (\"vectorizer\", TfidfVectorizer(min_df=10, max_df=0.05)),\n", + " (\"classifier\", ComplementNB(alpha=0.5, fit_prior=False)),\n", + " ]\n", + ")\n", "\n", "classifier.fit(X_train, y_train)" ] @@ -310,7 +298,11 @@ "\n", "print(metrics.classification_report(y_test_aligned, predicted))\n", "metrics.ConfusionMatrixDisplay.from_predictions(\n", - " y_true=y_test_aligned, y_pred=predicted, xticks_rotation=\"vertical\", normalize=\"pred\", values_format='.2f'\n", + " y_true=y_test_aligned,\n", + " y_pred=predicted,\n", + " xticks_rotation=\"vertical\",\n", + " normalize=\"pred\",\n", + " values_format=\".2f\",\n", ")\n", "None" ] @@ -333,7 +325,7 @@ "outputs": [], "source": [ "for X, y in zip(X_test[:50], y_test):\n", - " print(', '.join(y))\n", + " print(\", \".join(y))\n", " pprint(predict(X))\n", " print()" ] diff --git a/format-python.sh b/format-python.sh new file mode 100755 index 0000000..298bd7b --- /dev/null +++ b/format-python.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -e + +echo "Installing dependencies if necessary" +python3 -m pip install --upgrade autoflake isort black black[jupyter] mypy + +echo "Formatting and checking $1" + +echo Running autoflake +python3 -m autoflake --expand-star-imports --remove-all-unused-imports --ignore-init-module-imports --remove-unused-variables --in-place -r $1 + +echo Running isort +python3 -m isort --profile black --skip .env $1 + +echo Running black +python3 -m black $1 --exclude .env + +echo Running mypy +python3 -m mypy --namespace-packages --ignore-missing-imports --install-types --non-interactive --disallow-untyped-defs --disallow-incomplete-defs --pretty --follow-imports=silent --exclude=external/ --exclude=/build/ $1 \ No newline at end of file diff --git a/good_ai/src/open_s3/helper/__init__.py b/good_ai/src/open_s3/helper/__init__.py index ba3ed01..9639296 100644 --- a/good_ai/src/open_s3/helper/__init__.py +++ b/good_ai/src/open_s3/helper/__init__.py @@ -1,2 +1,2 @@ -from .progress_bar import DownloadProgressBar, UploadProgressBar from .human_readable_to_byte import human_readable_to_byte +from .progress_bar import DownloadProgressBar, UploadProgressBar diff --git a/good_ai/src/open_s3/helper/human_readable_to_byte.py b/good_ai/src/open_s3/helper/human_readable_to_byte.py index 9be0fe0..bd0d0ca 100644 --- a/good_ai/src/open_s3/helper/human_readable_to_byte.py +++ b/good_ai/src/open_s3/helper/human_readable_to_byte.py @@ -27,5 +27,5 @@ def human_readable_to_byte(size: str) -> int: scalar = float(results["scalar"]) idx = possible_units.index(results["unit"].upper()) - factor = 1024 ** idx + factor = 1024**idx return round(scalar * factor) diff --git a/good_ai/src/open_s3/helper/progress_bar.py b/good_ai/src/open_s3/helper/progress_bar.py index ecd01e4..545db48 100644 --- a/good_ai/src/open_s3/helper/progress_bar.py +++ b/good_ai/src/open_s3/helper/progress_bar.py @@ -1,40 +1,35 @@ import os import threading from logging import Logger -from typing import IO, Any, Optional - -from tqdm.auto import tqdm from pathlib import Path -import os -import sys -import threading - class ProgressBar: def __init__(self, file_size: int, logger: Logger, prefix: str): self._file_size = file_size self._logger = logger - self._prefix=prefix + self._prefix = prefix self._seen_so_far = 0 self._lock = threading.Lock() - def __call__(self, bytes_amount: int): + def __call__(self, bytes_amount: int) -> None: with self._lock: self._seen_so_far += bytes_amount percentage = (self._seen_so_far / float(self._file_size)) * 100 size_length = len(str(self._file_size)) progress = str(self._seen_so_far).rjust(size_length) - self._logger.info(f"{self._prefix} {progress}/{self._file_size} bytes ({percentage:.1f}%)") + self._logger.info( + f"{self._prefix} {progress}/{self._file_size} bytes ({percentage:.1f}%)" + ) class DownloadProgressBar(ProgressBar): def __init__(self, name: str, size: int, logger: Logger): - super().__init__(file_size=size, logger=logger, prefix=f'Downloading {name}') + super().__init__(file_size=size, logger=logger, prefix=f"Downloading {name}") class UploadProgressBar(ProgressBar): def __init__(self, path: Path, logger: Logger): size = os.path.getsize(path) - super().__init__(file_size=size, logger=logger, prefix=f'Uploading {path.name}') + super().__init__(file_size=size, logger=logger, prefix=f"Uploading {path.name}") diff --git a/good_ai/src/open_s3/large_file.py b/good_ai/src/open_s3/large_file.py index eff3931..3c665da 100644 --- a/good_ai/src/open_s3/large_file.py +++ b/good_ai/src/open_s3/large_file.py @@ -6,9 +6,8 @@ import tempfile from pathlib import Path from types import TracebackType from typing import IO, Any, Dict, List, Optional, Type, Union -import boto3 -from botocore.exceptions import ClientError +import boto3 from helper import DownloadProgressBar, UploadProgressBar, human_readable_to_byte logger = logging.getLogger("open_s3") @@ -62,7 +61,7 @@ class LargeFile: newline: Optional[str] = None, version: Optional[int] = None, keep_last_n: Optional[int] = None, - offline_mode: bool = False + offline_mode: bool = False, ): self._name: str = name self._version = version @@ -79,7 +78,7 @@ class LargeFile: self._find_versions() self._check_mode_and_set_version() - + @classmethod def configure_credentials( cls, @@ -154,7 +153,7 @@ class LargeFile: logger.exception("Could not finish operation.") return True - + @property def version_ids(self) -> List[int]: return [self._get_version_from_key(key) for key in self._versions] @@ -175,8 +174,17 @@ class LargeFile: with tempfile.TemporaryDirectory() as tmp: tmp_file_archive = Path(tmp) / f"{self._local_name}.tar.gz" - size = self._client.head_object(Bucket=self.bucket_name, Key=key)['ContentLength'] - self._client.download_file(Bucket=self.bucket_name, Key=key, Filename=str(tmp_file_archive), Callback=None if hide_progress else DownloadProgressBar(size=size, name=key, logger=logger)) + size = self._client.head_object(Bucket=self.bucket_name, Key=key)[ + "ContentLength" + ] + self._client.download_file( + Bucket=self.bucket_name, + Key=key, + Filename=str(tmp_file_archive), + Callback=None + if hide_progress + else DownloadProgressBar(size=size, name=key, logger=logger), + ) logger.info(f"Decompressing {self._local_name}") shutil.unpack_archive(str(tmp_file_archive), tmp, "gztar") tmp_file = Path(tmp) / self._local_name @@ -193,7 +201,7 @@ class LargeFile: with tempfile.TemporaryDirectory() as tmp: if path.is_file(): logger.info(f"Copying file for {self._local_name}") - copy = shutil.copy + copy: Any = shutil.copy else: logger.info(f"Copying directory for {self._local_name}") copy = shutil.copytree @@ -216,7 +224,14 @@ class LargeFile: logger.info(f"Uploading {self._local_name} to S3 from {path}") file_to_be_uploaded = Path(tmp) / f"{self._local_name}.tar.gz" - self._client.upload_file(Filename=str(file_to_be_uploaded), Bucket=self.bucket_name, Key=self._s3_name, Callback=None if hide_progress else UploadProgressBar(file_to_be_uploaded, logger=logger)) + self._client.upload_file( + Filename=str(file_to_be_uploaded), + Bucket=self.bucket_name, + Key=self._s3_name, + Callback=None + if hide_progress + else UploadProgressBar(file_to_be_uploaded, logger=logger), + ) self.clean_up() @@ -239,15 +254,21 @@ class LargeFile: "Please configure the S3 access options by calling LargeFile.configure_credentials or set offline_mode=True in the constructor." ) - self._client = boto3.client('s3', aws_access_key_id=self.access_key_id, aws_secret_access_key=self.secret_access_key, region_name=self.region_name, endpoint_url=self.endpoint_url) - + self._client = boto3.client( + "s3", + aws_access_key_id=self.access_key_id, + aws_secret_access_key=self.secret_access_key, + region_name=self.region_name, + endpoint_url=self.endpoint_url, + ) + def _find_versions(self) -> None: if self._offline_mode: self._fetch_versions_from_cache() else: self._create_client() self._fetch_versions_from_s3() - + if self._versions: logger.info(f"Found versions: {self.version_ids}") else: @@ -257,8 +278,7 @@ class LargeFile: logger.info(f"Fetching offline versions of {self._name}") self._versions = [ - path - for path in self.cache_path.glob(f'{self._local_name}-*') + path for path in self.cache_path.glob(f"{self._local_name}-*") ] def _fetch_versions_from_s3(self) -> None: @@ -267,10 +287,7 @@ class LargeFile: Bucket=self.bucket_name, Prefix=self._name ) self._versions = ( - sorted( - o["Key"] - for o in found_objects["Contents"] - ) + sorted(o["Key"] for o in found_objects["Contents"]) if "Contents" in found_objects else [] ) @@ -309,22 +326,25 @@ class LargeFile: @property def _s3_name(self) -> str: return f"{self._name}/{self._version}" - + @staticmethod def _get_version_from_key(key: Union[str, Path]) -> int: if isinstance(key, Path): return int(key.name.split("-")[-1]) return int(key.split("/")[-1]) - def _delete_old_versions_from_s3(self) -> None: if self._keep_last_n is not None: - for key in (self._versions[: -self._keep_last_n] if self._keep_last_n > 0 else self._versions): + for key in ( + self._versions[: -self._keep_last_n] + if self._keep_last_n > 0 + else self._versions + ): logger.info( f"Removing old version (keep_last_n={self._keep_last_n}): {key}" ) self._client.delete_object(Bucket=self.bucket_name, Key=key) - + def _delete_old_versions_from_disk(self) -> None: self.cache_path.mkdir(parents=True, exist_ok=True) @@ -343,4 +363,4 @@ class LargeFile: logger.info( f"Deleting file from cache to meet quota (max_cache_size={self.max_cache_size}): {file}" ) - os.unlink(file) \ No newline at end of file + os.unlink(file) diff --git a/good_ai/tests/open_s3/test_human_readable_to_byte.py b/good_ai/tests/open_s3/test_human_readable_to_byte.py index 85689df..f4fe84d 100644 --- a/good_ai/tests/open_s3/test_human_readable_to_byte.py +++ b/good_ai/tests/open_s3/test_human_readable_to_byte.py @@ -4,22 +4,22 @@ from src.open_s3.helper import human_readable_to_byte class TestHumanReadableToByte(unittest.TestCase): - def test_simple_cases(self): + def test_simple_cases(self) -> None: self.assertEqual(human_readable_to_byte("1KB"), 1024) self.assertEqual(human_readable_to_byte("2KB"), 2048) - def test_fractions(self): + def test_fractions(self) -> None: self.assertEqual(human_readable_to_byte("0.5KB"), 512) self.assertEqual(human_readable_to_byte("20.5KB"), 1024 * 20 + 512) - def test_formating(self): + def test_formating(self) -> None: self.assertEqual(human_readable_to_byte(" 1MB"), 1024 * 1024) self.assertEqual(human_readable_to_byte(" 2 MB"), 1024 * 1024 * 2) self.assertEqual(human_readable_to_byte(" 4 MB "), 1024 * 1024 * 4) self.assertEqual(human_readable_to_byte("8MB "), 1024 * 1024 * 8) self.assertEqual(human_readable_to_byte(" 1.5 MB "), 1024 * 1024 * 1.5) - def test_casing(self): + def test_casing(self) -> None: self.assertEqual(human_readable_to_byte("0.5GB"), 0.5 * 1024 * 1024 * 1024) self.assertEqual(human_readable_to_byte("0.5gB"), 0.5 * 1024 * 1024 * 1024) self.assertEqual(human_readable_to_byte("0.5Gb"), 0.5 * 1024 * 1024 * 1024) diff --git a/good_ai/tests/open_s3/test_large_file.py b/good_ai/tests/open_s3/test_large_file.py index 4901931..e37d60b 100644 --- a/good_ai/tests/open_s3/test_large_file.py +++ b/good_ai/tests/open_s3/test_large_file.py @@ -1,5 +1,6 @@ -from pathlib import Path import unittest +from pathlib import Path +from typing import Any from unittest.mock import Mock, create_autospec, patch import botocore.session @@ -14,23 +15,22 @@ credentials = { "aws_access_key_id": "YOUR_ACCESS_KEY_ID", "aws_secret_access_key": "YOUR_VERY_SECRET_ACCESS_KEY", "large_files_bucket_name": "create_a_bucket_and_put_its_name_here", - "other_key": 23, "endpoint_url": "this is optional, for backblaze, use this: https://s3.us-west-002.backblazeb2.com", } class TestLargeFile(unittest.TestCase): - def test_uninitialized(self): + def test_uninitialized(self) -> None: self.assertRaises(ValueError, LargeFile, "test-file") - def test_bad_file_modes(self): + def test_bad_file_modes(self) -> None: self.assertRaises(ValueError, LargeFile, "test-file", "w", version=3) self.assertRaises(ValueError, LargeFile, "test-file", "wb", version=3) self.assertRaises(ValueError, LargeFile, "test-file", "w+r") self.assertRaises(ValueError, LargeFile, "test-file", "test") @patch("botocore.session") - def test_initialized_with_dict(self, session): + def test_initialized_with_dict(self, session) -> None: session_mock = Mock() session.get_session = create_autospec( botocore.session.get_session, return_value=session_mock @@ -79,7 +79,7 @@ class TestLargeFile(unittest.TestCase): self.assertEqual(lf._s3_name, "test-file/2") @patch("botocore.session") - def test_initialized_with_file(self, session): + def test_initialized_with_file(self, session: Any) -> None: session_mock = Mock() session.get_session = create_autospec( botocore.session.get_session, return_value=session_mock