andras/great-ai
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases 4 Packages Wiki Activity Actions

Format

Browse source 
Signed-off-by: András Schmelczer <andras@schmelczer.dev>
This commit is contained in:
Andras Schmelczer 2022-04-02 13:57:16 +02:00
parent d79a75d352
commit ea54cd9c8c
13 changed files with 168 additions and 116 deletions
Download patch file Download diff file Expand all files Collapse all files

17
check-python.sh Executable file
View file

@ -0,0 +1,17 @@
#!/bin/sh
set -e
echo "Installing dependencies if necessary"
python3 -m pip install --upgrade autoflake isort black black[jupyter] mypy flake8
echo "Checking $1"
python3 -m autoflake --expand-star-imports --remove-all-unused-imports --ignore-init-module-imports --remove-unused-variables --in-place -r $1 --check
python3 -m isort --profile black --skip .env $1 --check
python3 -m black $1 --exclude .env --check
yes | python3 -m mypy $1 --install-types > /dev/null || true
python3 -m mypy --namespace-packages --ignore-missing-imports --install-types --non-interactive --disallow-untyped-defs --disallow-incomplete-defs --follow-imports=silent --exclude=external/ --exclude=/build/ --pretty $1
python3 -m flake8 $1 --count --show-source --statistics --exclude=__init__.py,.env,external --ignore=E501,E402,F821,W503,E722,E203

2
example/config.py
View file

@ -1 +1 @@
model_key =
model_key = "domain-prediction-v2"

8
example/helper/preprocess.py
View file

@ -1,12 +1,10 @@
import re
from sus.clean import clean
from sus.lemmatize_text import lemmatize_text
import re
def preprocess(text: str) -> str:
cleaned = clean(text, convert_to_ascii=True)
lemmas = [
re.sub(r'\d+', 'NUM', lemma)
for lemma in lemmatize_text(cleaned)
]
lemmas = [re.sub(r"\d+", "NUM", lemma) for lemma in lemmatize_text(cleaned)]
return " ".join(lemmas)

3
example/models/domain_prediction.py
View file

@ -1,6 +1,7 @@
from pydantic import BaseModel
from typing import List
from pydantic import BaseModel
class DomainPrediction(BaseModel):
domain: str

53
example/predict.py
View file

@ -1,42 +1,46 @@
from models import DomainPrediction
from typing import Iterable, List, Dict
from sklearn.pipeline import Pipeline
from helper import preprocess
import re
from typing import Dict, Iterable, List
from helper import preprocess
from models import DomainPrediction
from sklearn.pipeline import Pipeline
# from sus.use_model import use_model
from config import model_key
# @use_model(model_key, version="latest")
def predict(text: str, model: Pipeline, cut_off_probability: float=0.2) -> List[DomainPrediction]:
def predict(
text: str, model: Pipeline, cut_off_probability: float = 0.2
) -> List[DomainPrediction]:
assert 0 <= cut_off_probability <= 1
feature_names = model.named_steps['vectorizer'].get_feature_names_out()
feature_names = model.named_steps["vectorizer"].get_feature_names_out()
token_mapping = {
preprocess(original): original
for original in re.sub(r'[^a-zA-Z0-9]', ' ', text).split(' ')
for original in re.sub(r"[^a-zA-Z0-9]", " ", text).split(" ")
}
features = model.named_steps['vectorizer'].transform([text])
prediction = model.named_steps['classifier'].predict_proba(features)[0]
features = model.named_steps["vectorizer"].transform([text])
prediction = model.named_steps["classifier"].predict_proba(features)[0]
best_classes = sorted(enumerate(prediction), key=lambda v: v[1], reverse=True)
results: List[DomainPrediction] = []
for class_index, probability in best_classes:
weights = model.named_steps['classifier'].feature_log_prob_[class_index]
weights = model.named_steps["classifier"].feature_log_prob_[class_index]
results.append(DomainPrediction(
domain=model.named_steps['classifier'].classes_[class_index],
probability=round(probability * 100),
explanation=_get_explanation(
feature_names=feature_names,
features=features.A[0],
weights=weights,
token_mapping=token_mapping
results.append(
DomainPrediction(
domain=model.named_steps["classifier"].classes_[class_index],
probability=round(probability * 100),
explanation=_get_explanation(
feature_names=feature_names,
features=features.A[0],
weights=weights,
token_mapping=token_mapping,
),
)
))
)
if sum(r.probability for r in results) >= cut_off_probability:
break
@ -44,7 +48,12 @@ def predict(text: str, model: Pipeline, cut_off_probability: float=0.2) -> List[
return results
def _get_explanation(feature_names: Iterable[str], features: Iterable[float], weights: Iterable[float], token_mapping: Dict[str, str]) -> List[str]:
def _get_explanation(
feature_names: Iterable[str],
features: Iterable[float],
weights: Iterable[float],
token_mapping: Dict[str, str],
) -> List[str]:
influential = [
(value * weight, name)
for name, value, weight in zip(feature_names, features, weights)

62
example/train.ipynb
View file

@ -85,10 +85,7 @@
"source": [
"def preprocess(text: str) -> str:\n",
" cleaned = clean(text, convert_to_ascii=True)\n",
" lemmas = [\n",
" re.sub(r'\\d+', 'NUM', lemma)\n",
" for lemma in lemmatize_text(cleaned)\n",
" ]\n",
" lemmas = [re.sub(r\"\\d+\", \"NUM\", lemma) for lemma in lemmatize_text(cleaned)]\n",
" return \" \".join(lemmas)"
]
},
@ -157,14 +154,14 @@
"source": [
"corpora = list(SS_CORPUS_PATH.glob(f\"{PREFIX}*.json\"))\n",
"shuffle(corpora)\n",
"print(f'Found {len(corpora)} files')\n",
"print(f\"Found {len(corpora)} files\")\n",
"\n",
"data = []\n",
"for p in corpora[:MAX_FILE_COUNT]:\n",
" with open(p) as f:\n",
" data.extend(json.load(f).items())\n",
"\n",
"print(f'Found {len(data)} documents')"
"print(f\"Found {len(data)} documents\")"
]
},
{
@ -174,23 +171,12 @@
"outputs": [],
"source": [
"X_train, X_test, y_train, y_test = train_test_split(\n",
" [d[0] for d in data],\n",
" [d[1] for d in data],\n",
" test_size=0.1, \n",
" random_state=SEED\n",
" [d[0] for d in data], [d[1] for d in data], test_size=0.1, random_state=SEED\n",
")\n",
"\n",
"X_train = [\n",
" x\n",
" for x, y in zip(X_train, y_train)\n",
" for domain in y\n",
"]\n",
"X_train = [x for x, y in zip(X_train, y_train) for domain in y]\n",
"\n",
"y_train = [\n",
" domain\n",
" for x, y in zip(X_train, y_train)\n",
" for domain in y\n",
"]"
"y_train = [domain for x, y in zip(X_train, y_train) for domain in y]"
]
},
{
@ -207,23 +193,23 @@
"outputs": [],
"source": [
"classifier = GridSearchCV(\n",
" Pipeline(steps=[\n",
" ('vectorizer', TfidfVectorizer()),\n",
" ('classifier', ComplementNB())\n",
" ]),\n",
" Pipeline(steps=[(\"vectorizer\", TfidfVectorizer()), (\"classifier\", ComplementNB())]),\n",
" {\n",
" \"vectorizer__max_df\": [0.05, 0.1, 0.3],\n",
" \"vectorizer__min_df\": [5, 10, 30, 100],\n",
" \"vectorizer__sublinear_tf\": [True, False],\n",
" \"classifier__alpha\": [0.001, 0.1, 0.5, 1],\n",
" \"classifier__fit_prior\": [True, False]\n",
" \"classifier__fit_prior\": [True, False],\n",
" },\n",
" scoring=\"f1_macro\",\n",
" cv=3,\n",
" n_jobs=12,\n",
" verbose=7,\n",
" cv=3,\n",
" n_jobs=12,\n",
" verbose=7,\n",
")\n",
"classifier.fit(\n",
" X_train[:HYPERPARAMETER_OPTIMISATION_SIZE],\n",
" y_train[:HYPERPARAMETER_OPTIMISATION_SIZE],\n",
")\n",
"classifier.fit(X_train[:HYPERPARAMETER_OPTIMISATION_SIZE], y_train[:HYPERPARAMETER_OPTIMISATION_SIZE])\n",
"\n",
"results = pd.DataFrame(classifier.cv_results_)\n",
"results.sort_values(\"rank_test_score\")"
@ -247,10 +233,12 @@
}
],
"source": [
"classifier = Pipeline(steps=[\n",
" ('vectorizer', TfidfVectorizer(min_df=10, max_df=0.05)),\n",
" ('classifier', ComplementNB(alpha=0.5, fit_prior=False))\n",
"])\n",
"classifier = Pipeline(\n",
" steps=[\n",
" (\"vectorizer\", TfidfVectorizer(min_df=10, max_df=0.05)),\n",
" (\"classifier\", ComplementNB(alpha=0.5, fit_prior=False)),\n",
" ]\n",
")\n",
"\n",
"classifier.fit(X_train, y_train)"
]
@ -310,7 +298,11 @@
"\n",
"print(metrics.classification_report(y_test_aligned, predicted))\n",
"metrics.ConfusionMatrixDisplay.from_predictions(\n",
" y_true=y_test_aligned, y_pred=predicted, xticks_rotation=\"vertical\", normalize=\"pred\", values_format='.2f'\n",
" y_true=y_test_aligned,\n",
" y_pred=predicted,\n",
" xticks_rotation=\"vertical\",\n",
" normalize=\"pred\",\n",
" values_format=\".2f\",\n",
")\n",
"None"
]
@ -333,7 +325,7 @@
"outputs": [],
"source": [
"for X, y in zip(X_test[:50], y_test):\n",
" print(', '.join(y))\n",
" print(\", \".join(y))\n",
" pprint(predict(X))\n",
" print()"
]

20
format-python.sh Executable file
View file

@ -0,0 +1,20 @@
#!/bin/bash
set -e
echo "Installing dependencies if necessary"
python3 -m pip install --upgrade autoflake isort black black[jupyter] mypy
echo "Formatting and checking $1"
echo Running autoflake
python3 -m autoflake --expand-star-imports --remove-all-unused-imports --ignore-init-module-imports --remove-unused-variables --in-place -r $1
echo Running isort
python3 -m isort --profile black --skip .env $1
echo Running black
python3 -m black $1 --exclude .env
echo Running mypy
python3 -m mypy --namespace-packages --ignore-missing-imports --install-types --non-interactive --disallow-untyped-defs --disallow-incomplete-defs --pretty --follow-imports=silent --exclude=external/ --exclude=/build/ $1

2
good_ai/src/open_s3/helper/__init__.py
View file

@ -1,2 +1,2 @@
from .progress_bar import DownloadProgressBar, UploadProgressBar
from .human_readable_to_byte import human_readable_to_byte
from .progress_bar import DownloadProgressBar, UploadProgressBar

2
good_ai/src/open_s3/helper/human_readable_to_byte.py
View file

@ -27,5 +27,5 @@ def human_readable_to_byte(size: str) -> int:
scalar = float(results["scalar"])
idx = possible_units.index(results["unit"].upper())
factor = 1024 ** idx
factor = 1024**idx
return round(scalar * factor)

19
good_ai/src/open_s3/helper/progress_bar.py
View file

@ -1,40 +1,35 @@
import os
import threading
from logging import Logger
from typing import IO, Any, Optional
from tqdm.auto import tqdm
from pathlib import Path
import os
import sys
import threading
class ProgressBar:
def __init__(self, file_size: int, logger: Logger, prefix: str):
self._file_size = file_size
self._logger = logger
self._prefix=prefix
self._prefix = prefix
self._seen_so_far = 0
self._lock = threading.Lock()
def __call__(self, bytes_amount: int):
def __call__(self, bytes_amount: int) -> None:
with self._lock:
self._seen_so_far += bytes_amount
percentage = (self._seen_so_far / float(self._file_size)) * 100
size_length = len(str(self._file_size))
progress = str(self._seen_so_far).rjust(size_length)
self._logger.info(f"{self._prefix} {progress}/{self._file_size} bytes ({percentage:.1f}%)")
self._logger.info(
f"{self._prefix} {progress}/{self._file_size} bytes ({percentage:.1f}%)"
)
class DownloadProgressBar(ProgressBar):
def __init__(self, name: str, size: int, logger: Logger):
super().__init__(file_size=size, logger=logger, prefix=f'Downloading {name}')
super().__init__(file_size=size, logger=logger, prefix=f"Downloading {name}")
class UploadProgressBar(ProgressBar):
def __init__(self, path: Path, logger: Logger):
size = os.path.getsize(path)
super().__init__(file_size=size, logger=logger, prefix=f'Uploading {path.name}')
super().__init__(file_size=size, logger=logger, prefix=f"Uploading {path.name}")

52
good_ai/src/open_s3/large_file.py
View file

@ -6,9 +6,8 @@ import tempfile
from pathlib import Path
from types import TracebackType
from typing import IO, Any, Dict, List, Optional, Type, Union
import boto3
from botocore.exceptions import ClientError
import boto3
from helper import DownloadProgressBar, UploadProgressBar, human_readable_to_byte
logger = logging.getLogger("open_s3")
@ -62,7 +61,7 @@ class LargeFile:
newline: Optional[str] = None,
version: Optional[int] = None,
keep_last_n: Optional[int] = None,
offline_mode: bool = False
offline_mode: bool = False,
):
self._name: str = name
self._version = version
@ -175,8 +174,17 @@ class LargeFile:
with tempfile.TemporaryDirectory() as tmp:
tmp_file_archive = Path(tmp) / f"{self._local_name}.tar.gz"
size = self._client.head_object(Bucket=self.bucket_name, Key=key)['ContentLength']
self._client.download_file(Bucket=self.bucket_name, Key=key, Filename=str(tmp_file_archive), Callback=None if hide_progress else DownloadProgressBar(size=size, name=key, logger=logger))
size = self._client.head_object(Bucket=self.bucket_name, Key=key)[
"ContentLength"
]
self._client.download_file(
Bucket=self.bucket_name,
Key=key,
Filename=str(tmp_file_archive),
Callback=None
if hide_progress
else DownloadProgressBar(size=size, name=key, logger=logger),
)
logger.info(f"Decompressing {self._local_name}")
shutil.unpack_archive(str(tmp_file_archive), tmp, "gztar")
tmp_file = Path(tmp) / self._local_name
@ -193,7 +201,7 @@ class LargeFile:
with tempfile.TemporaryDirectory() as tmp:
if path.is_file():
logger.info(f"Copying file for {self._local_name}")
copy = shutil.copy
copy: Any = shutil.copy
else:
logger.info(f"Copying directory for {self._local_name}")
copy = shutil.copytree
@ -216,7 +224,14 @@ class LargeFile:
logger.info(f"Uploading {self._local_name} to S3 from {path}")
file_to_be_uploaded = Path(tmp) / f"{self._local_name}.tar.gz"
self._client.upload_file(Filename=str(file_to_be_uploaded), Bucket=self.bucket_name, Key=self._s3_name, Callback=None if hide_progress else UploadProgressBar(file_to_be_uploaded, logger=logger))
self._client.upload_file(
Filename=str(file_to_be_uploaded),
Bucket=self.bucket_name,
Key=self._s3_name,
Callback=None
if hide_progress
else UploadProgressBar(file_to_be_uploaded, logger=logger),
)
self.clean_up()
@ -239,7 +254,13 @@ class LargeFile:
"Please configure the S3 access options by calling LargeFile.configure_credentials or set offline_mode=True in the constructor."
)
self._client = boto3.client('s3', aws_access_key_id=self.access_key_id, aws_secret_access_key=self.secret_access_key, region_name=self.region_name, endpoint_url=self.endpoint_url)
self._client = boto3.client(
"s3",
aws_access_key_id=self.access_key_id,
aws_secret_access_key=self.secret_access_key,
region_name=self.region_name,
endpoint_url=self.endpoint_url,
)
def _find_versions(self) -> None:
if self._offline_mode:
@ -257,8 +278,7 @@ class LargeFile:
logger.info(f"Fetching offline versions of {self._name}")
self._versions = [
path
for path in self.cache_path.glob(f'{self._local_name}-*')
path for path in self.cache_path.glob(f"{self._local_name}-*")
]
def _fetch_versions_from_s3(self) -> None:
@ -267,10 +287,7 @@ class LargeFile:
Bucket=self.bucket_name, Prefix=self._name
)
self._versions = (
sorted(
o["Key"]
for o in found_objects["Contents"]
)
sorted(o["Key"] for o in found_objects["Contents"])
if "Contents" in found_objects
else []
)
@ -316,10 +333,13 @@ class LargeFile:
return int(key.name.split("-")[-1])
return int(key.split("/")[-1])
def _delete_old_versions_from_s3(self) -> None:
if self._keep_last_n is not None:
for key in (self._versions[: -self._keep_last_n] if self._keep_last_n > 0 else self._versions):
for key in (
self._versions[: -self._keep_last_n]
if self._keep_last_n > 0
else self._versions
):
logger.info(
f"Removing old version (keep_last_n={self._keep_last_n}): {key}"
)

8
good_ai/tests/open_s3/test_human_readable_to_byte.py
View file

@ -4,22 +4,22 @@ from src.open_s3.helper import human_readable_to_byte
class TestHumanReadableToByte(unittest.TestCase):
def test_simple_cases(self):
def test_simple_cases(self) -> None:
self.assertEqual(human_readable_to_byte("1KB"), 1024)
self.assertEqual(human_readable_to_byte("2KB"), 2048)
def test_fractions(self):
def test_fractions(self) -> None:
self.assertEqual(human_readable_to_byte("0.5KB"), 512)
self.assertEqual(human_readable_to_byte("20.5KB"), 1024 * 20 + 512)
def test_formating(self):
def test_formating(self) -> None:
self.assertEqual(human_readable_to_byte(" 1MB"), 1024 * 1024)
self.assertEqual(human_readable_to_byte(" 2 MB"), 1024 * 1024 * 2)
self.assertEqual(human_readable_to_byte(" 4 MB "), 1024 * 1024 * 4)
self.assertEqual(human_readable_to_byte("8MB "), 1024 * 1024 * 8)
self.assertEqual(human_readable_to_byte(" 1.5 MB "), 1024 * 1024 * 1.5)
def test_casing(self):
def test_casing(self) -> None:
self.assertEqual(human_readable_to_byte("0.5GB"), 0.5 * 1024 * 1024 * 1024)
self.assertEqual(human_readable_to_byte("0.5gB"), 0.5 * 1024 * 1024 * 1024)
self.assertEqual(human_readable_to_byte("0.5Gb"), 0.5 * 1024 * 1024 * 1024)

12
good_ai/tests/open_s3/test_large_file.py
View file

@ -1,5 +1,6 @@
from pathlib import Path
import unittest
from pathlib import Path
from typing import Any
from unittest.mock import Mock, create_autospec, patch
import botocore.session
@ -14,23 +15,22 @@ credentials = {
"aws_access_key_id": "YOUR_ACCESS_KEY_ID",
"aws_secret_access_key": "YOUR_VERY_SECRET_ACCESS_KEY",
"large_files_bucket_name": "create_a_bucket_and_put_its_name_here",
"other_key": 23,
"endpoint_url": "this is optional, for backblaze, use this: https://s3.us-west-002.backblazeb2.com",
}
class TestLargeFile(unittest.TestCase):
def test_uninitialized(self):
def test_uninitialized(self) -> None:
self.assertRaises(ValueError, LargeFile, "test-file")
def test_bad_file_modes(self):
def test_bad_file_modes(self) -> None:
self.assertRaises(ValueError, LargeFile, "test-file", "w", version=3)
self.assertRaises(ValueError, LargeFile, "test-file", "wb", version=3)
self.assertRaises(ValueError, LargeFile, "test-file", "w+r")
self.assertRaises(ValueError, LargeFile, "test-file", "test")
@patch("botocore.session")
def test_initialized_with_dict(self, session):
def test_initialized_with_dict(self, session) -> None:
session_mock = Mock()
session.get_session = create_autospec(
botocore.session.get_session, return_value=session_mock
@ -79,7 +79,7 @@ class TestLargeFile(unittest.TestCase):
self.assertEqual(lf._s3_name, "test-file/2")
@patch("botocore.session")
def test_initialized_with_file(self, session):
def test_initialized_with_file(self, session: Any) -> None:
session_mock = Mock()
session.get_session = create_autospec(
botocore.session.get_session, return_value=session_mock