Remove clutter
This commit is contained in:
parent
2c3f19e67c
commit
a19a39a3a0
7 changed files with 84 additions and 239 deletions
2
.vscode/settings.json
vendored
2
.vscode/settings.json
vendored
|
|
@ -5,8 +5,10 @@
|
|||
"fastapi",
|
||||
"iloc",
|
||||
"inplace",
|
||||
"ipynb",
|
||||
"levelno",
|
||||
"matplotlib",
|
||||
"nbconvert",
|
||||
"plotly",
|
||||
"psutil",
|
||||
"pydantic",
|
||||
|
|
|
|||
|
|
@ -1,12 +0,0 @@
|
|||
# Train Domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)
|
||||
|
||||
## Upload the dataset (or a part of it) to shared infrastructure
|
||||
|
||||
```sh
|
||||
mkdir ss-data && cd ss-data
|
||||
wget https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/manifest.txt
|
||||
wget -B https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/ -i manifest.txt
|
||||
cd -
|
||||
python3 -m great_ai.open_s3 --secrets s3.ini --push ss-data
|
||||
rm -rf ss-data
|
||||
```
|
||||
82
examples/simple/__deploy__.py
Normal file
82
examples/simple/__deploy__.py
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
|
||||
# # Train a domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)
|
||||
#
|
||||
# ## Part 3: Create production inference function
|
||||
#
|
||||
# In the [previous notebook](train.ipynb), we trained our AI model. Now, it's time to create **G**eneral **R**obust **E**nd-to-end **A**utomated **T**rustworthy deployment from it using the `GreatAI` Python package.
|
||||
|
||||
# In[1]:
|
||||
|
||||
|
||||
import re
|
||||
from typing import List
|
||||
|
||||
from great_ai import ClassificationOutput, GreatAI, use_model
|
||||
from great_ai.utilities.clean import clean
|
||||
from sklearn.pipeline import Pipeline
|
||||
|
||||
# In[2]:
|
||||
|
||||
|
||||
@GreatAI.deploy
|
||||
@use_model("small-domain-prediction-v2", version="latest")
|
||||
def predict_domain(
|
||||
text: str, model: Pipeline, target_confidence: int = 20
|
||||
) -> List[ClassificationOutput]:
|
||||
"""
|
||||
Predict the scientific domain of the input text.
|
||||
Return labels until their sum likelihood is larger than target_confidence.
|
||||
"""
|
||||
assert 0 <= target_confidence <= 100, "invalid argument"
|
||||
|
||||
preprocessed = re.sub(r"[^a-zA-Z ]", "", clean(text, convert_to_ascii=True))
|
||||
features = model.named_steps["vectorizer"].transform([preprocessed])
|
||||
prediction = model.named_steps["classifier"].predict_proba(features)[0]
|
||||
|
||||
best_classes = sorted(enumerate(prediction), key=lambda v: v[1], reverse=True)
|
||||
|
||||
results: List[ClassificationOutput] = []
|
||||
for class_index, probability in best_classes:
|
||||
results.append(
|
||||
ClassificationOutput(
|
||||
label=model.named_steps["classifier"].classes_[class_index],
|
||||
confidence=round(probability * 100),
|
||||
explanation=[
|
||||
word
|
||||
for _, word in sorted(
|
||||
(
|
||||
(weight, word)
|
||||
for weight, word, count in zip(
|
||||
model.named_steps["classifier"].feature_log_prob_[
|
||||
class_index
|
||||
],
|
||||
model.named_steps["vectorizer"].get_feature_names_out(),
|
||||
features.A[0],
|
||||
)
|
||||
if count > 0
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
][:5],
|
||||
)
|
||||
)
|
||||
|
||||
if sum(r.confidence for r in results) >= target_confidence:
|
||||
break
|
||||
|
||||
return results
|
||||
|
||||
|
||||
# In[3]:
|
||||
|
||||
|
||||
result = predict_domain(
|
||||
"""
|
||||
State-of-the-art methods for zero-shot visual recognition formulate learning as a joint embedding problem of images and side information. In these formulations the current best complement to visual features are attributes: manually encoded vectors describing shared characteristics among categories. Despite good performance, attributes have limitations: (1) finer-grained recognition requires commensurately more, and (2) attributes do not provide a natural language interface. We propose to overcome these limitations by training neural language models from scratch; i.e. without pre-training and only consuming words and characters. Our proposed models train end-to-end to align with the fine-grained and category-specific content of images. Natural language provides a flexible and compact way of encoding only the salient visual aspects for distinguishing categories. By training on raw text, our model can do inference on raw text as well, providing humans a familiar mode both for annotation and retrieval. Our model achieves strong performance on zero-shot text-based image retrieval and significantly outperforms the attribute-based state-of-the-art for zero-shot classification on the CaltechUCSD Birds 200-2011 dataset. """
|
||||
)
|
||||
|
||||
from pprint import pprint
|
||||
|
||||
pprint(result.dict(), width=120)
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
import re
|
||||
|
||||
from great_ai.utilities.clean import clean
|
||||
from great_ai.utilities.lemmatize_text import lemmatize_text
|
||||
|
||||
|
||||
def preprocess(text: str) -> str:
|
||||
text = clean(text, convert_to_ascii=True)
|
||||
text = re.sub(r"[^a-zA-Z0-9]", " ", text)
|
||||
return text
|
||||
|
||||
|
||||
def lemmatize(text: str) -> str:
|
||||
lemmatized = lemmatize_text(text)
|
||||
clean_lemmas = [re.sub(r"\d[\d.,]*", "NUM", lemma.lower()) for lemma in lemmatized]
|
||||
return " ".join(clean_lemmas)
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import json
|
||||
from random import shuffle
|
||||
|
||||
from devtools import debug
|
||||
from great_ai import process_batch
|
||||
|
||||
from predict_domain import predict_domain
|
||||
|
||||
if __name__ == "__main__":
|
||||
with open(".cache/data-1/s2-corpus-323.json") as f:
|
||||
raw = json.load(f)
|
||||
|
||||
shuffle(raw)
|
||||
data = {f'{r["title"]} {r["abstract"]}': r["domain"] for r in raw[:10]}
|
||||
|
||||
results = process_batch(predict_domain, data.keys())
|
||||
|
||||
for predicted, actual in zip(results, data.values()):
|
||||
print(", ".join(actual))
|
||||
debug(predicted)
|
||||
print()
|
||||
|
|
@ -1,62 +0,0 @@
|
|||
from typing import Iterable, List, Optional
|
||||
|
||||
from great_ai import ClassificationOutput, GreatAI, use_model
|
||||
from sklearn.pipeline import Pipeline
|
||||
|
||||
from helpers import lemmatize, preprocess
|
||||
|
||||
|
||||
@GreatAI.deploy
|
||||
@use_model("small-domain-prediction-v2", version="latest")
|
||||
def predict_domain(
|
||||
text: str, model: Pipeline, target_confidence: float = 20
|
||||
) -> List[ClassificationOutput]:
|
||||
"""
|
||||
Predict the scientific domain of the input text.
|
||||
Return labels until their sum likelihood is larger than target_confidence.
|
||||
"""
|
||||
assert 0 <= target_confidence <= 100, "invalid argument"
|
||||
|
||||
processed = [(word, lemmatize(word)) for word in preprocess(text).split(" ")]
|
||||
token_mapping = dict(processed)
|
||||
clean_input = " ".join(v[1] for v in processed)
|
||||
|
||||
feature_names = [
|
||||
token_mapping.get(name)
|
||||
for name in model.named_steps["vectorizer"].get_feature_names_out()
|
||||
]
|
||||
|
||||
prediction = model.predict_proba([clean_input])[0]
|
||||
best_classes = sorted(enumerate(prediction), key=lambda v: v[1], reverse=True)
|
||||
|
||||
results: List[ClassificationOutput] = []
|
||||
for class_index, probability in best_classes:
|
||||
results.append(
|
||||
ClassificationOutput(
|
||||
label=model.named_steps["classifier"].classes_[class_index],
|
||||
confidence=round(probability * 100),
|
||||
explanation=_get_explanation(
|
||||
weights=model.named_steps["classifier"].feature_log_prob_[
|
||||
class_index
|
||||
],
|
||||
words=feature_names,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
if sum(r.confidence for r in results) >= target_confidence:
|
||||
break
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def _get_explanation(
|
||||
weights: Iterable[float],
|
||||
words: Iterable[Optional[str]],
|
||||
) -> List[str]:
|
||||
most_influential = sorted(
|
||||
((weight, word) for weight, word in zip(weights, words) if word),
|
||||
reverse=True,
|
||||
)[:5]
|
||||
|
||||
return [word for _, word in most_influential]
|
||||
126
requirements.txt
126
requirements.txt
|
|
@ -1,126 +0,0 @@
|
|||
anyio==3.5.0
|
||||
appnope==0.1.2
|
||||
asgiref==3.5.0
|
||||
asttokens==2.0.5
|
||||
attrs==21.4.0
|
||||
autoflake==1.4
|
||||
backcall==0.2.0
|
||||
beautifulsoup4==4.10.0
|
||||
black==22.3.0
|
||||
blis==0.7.7
|
||||
boto3==1.21.32
|
||||
botocore==1.24.32
|
||||
Brotli==1.0.9
|
||||
catalogue==2.0.7
|
||||
certifi==2021.10.8
|
||||
charset-normalizer==2.0.12
|
||||
click==8.0.4
|
||||
cycler==0.11.0
|
||||
cymem==2.0.6
|
||||
dash==2.3.1
|
||||
dash-core-components==2.0.0
|
||||
dash-html-components==2.0.0
|
||||
dash-table==5.0.0
|
||||
debugpy==1.6.0
|
||||
decorator==5.1.1
|
||||
devtools==0.8.0
|
||||
dill==0.3.4
|
||||
en-core-web-lg @ https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.2.0/en_core_web_lg-3.2.0-py3-none-any.whl
|
||||
entrypoints==0.4
|
||||
executing==0.8.3
|
||||
fastapi==0.75.1
|
||||
flake8==4.0.1
|
||||
Flask==2.1.1
|
||||
Flask-Compress==1.11
|
||||
fonttools==4.31.2
|
||||
h11==0.13.0
|
||||
idna==3.3
|
||||
importlib-metadata==4.11.3
|
||||
iniconfig==1.1.1
|
||||
ipykernel==6.11.0
|
||||
ipython==8.2.0
|
||||
isort==5.10.1
|
||||
itsdangerous==2.1.2
|
||||
jedi==0.18.1
|
||||
Jinja2==3.1.1
|
||||
jmespath==1.0.0
|
||||
joblib==1.1.0
|
||||
jupyter-client==7.2.1
|
||||
jupyter-core==4.9.2
|
||||
kiwisolver==1.4.2
|
||||
langcodes==3.3.0
|
||||
langdetect==1.0.9
|
||||
language-data==1.1
|
||||
lxml==4.8.0
|
||||
marisa-trie==0.7.7
|
||||
MarkupSafe==2.1.1
|
||||
matplotlib==3.5.1
|
||||
matplotlib-inline==0.1.3
|
||||
mccabe==0.6.1
|
||||
multiprocess==0.70.12.2
|
||||
murmurhash==1.0.6
|
||||
mypy==0.942
|
||||
mypy-extensions==0.4.3
|
||||
nest-asyncio==1.5.4
|
||||
numpy==1.22.3
|
||||
packaging==21.3
|
||||
pandas==1.4.1
|
||||
parso==0.8.3
|
||||
pathspec==0.9.0
|
||||
pathy==0.6.1
|
||||
pexpect==4.8.0
|
||||
pickleshare==0.7.5
|
||||
Pillow==9.1.0
|
||||
platformdirs==2.5.1
|
||||
plotly==5.7.0
|
||||
pluggy==1.0.0
|
||||
preshed==3.0.6
|
||||
prompt-toolkit==3.0.28
|
||||
psutil==5.9.0
|
||||
ptyprocess==0.7.0
|
||||
pure-eval==0.2.2
|
||||
py==1.11.0
|
||||
pycodestyle==2.8.0
|
||||
pydantic==1.8.2
|
||||
pyflakes==2.4.0
|
||||
Pygments==2.11.2
|
||||
pyparsing==3.0.7
|
||||
pytest==7.1.1
|
||||
python-dateutil==2.8.2
|
||||
pytz==2022.1
|
||||
pyzmq==22.3.0
|
||||
requests==2.27.1
|
||||
s3transfer==0.5.2
|
||||
scikit-learn==1.0.2
|
||||
scipy==1.8.0
|
||||
six==1.16.0
|
||||
smart-open==5.2.1
|
||||
sniffio==1.2.0
|
||||
soupsieve==2.3.1
|
||||
spacy==3.2.4
|
||||
spacy-legacy==3.0.9
|
||||
spacy-loggers==1.0.2
|
||||
srsly==2.4.2
|
||||
stack-data==0.2.0
|
||||
starlette==0.17.1
|
||||
tenacity==8.0.1
|
||||
thinc==8.0.15
|
||||
threadpoolctl==3.1.0
|
||||
tinydb==4.7.0
|
||||
tokenize-rt==4.2.1
|
||||
tomli==2.0.1
|
||||
tornado==6.1
|
||||
tqdm==4.63.1
|
||||
traitlets==5.1.1
|
||||
typer==0.4.1
|
||||
types-requests==2.27.16
|
||||
types-ujson==4.2.1
|
||||
types-urllib3==1.26.11
|
||||
typing_extensions==4.1.1
|
||||
Unidecode==1.3.4
|
||||
urllib3==1.26.9
|
||||
uvicorn==0.17.6
|
||||
wasabi==0.9.1
|
||||
wcwidth==0.2.5
|
||||
Werkzeug==2.1.1
|
||||
zipp==3.8.0
|
||||
Loading…
Add table
Add a link
Reference in a new issue