Fix explanations
This commit is contained in:
parent
89661e8fbc
commit
75614f9022
2 changed files with 34 additions and 589 deletions
|
|
@ -1,5 +1,5 @@
|
||||||
import re
|
import re
|
||||||
from typing import Dict, Iterable, List
|
from typing import Dict, Iterable, List
|
||||||
|
|
||||||
from preprocess import preprocess
|
from preprocess import preprocess
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
|
|
@ -20,6 +20,8 @@ class DomainPrediction(BaseModel):
|
||||||
def predict_domain(
|
def predict_domain(
|
||||||
text: str, model: Pipeline, cut_off_probability: float = 0.2
|
text: str, model: Pipeline, cut_off_probability: float = 0.2
|
||||||
) -> List[DomainPrediction]:
|
) -> List[DomainPrediction]:
|
||||||
|
assert 0 <= cut_off_probability <= 1
|
||||||
|
|
||||||
"""
|
"""
|
||||||
Predict the scientific domain of the input text.
|
Predict the scientific domain of the input text.
|
||||||
Return labels until their sum likelihood is larger than cut_off_probability.
|
Return labels until their sum likelihood is larger than cut_off_probability.
|
||||||
|
|
@ -42,10 +44,11 @@ def predict_domain(
|
||||||
results: List[DomainPrediction] = []
|
results: List[DomainPrediction] = []
|
||||||
for class_index, probability in best_classes:
|
for class_index, probability in best_classes:
|
||||||
weights = model.named_steps["classifier"].feature_log_prob_[class_index]
|
weights = model.named_steps["classifier"].feature_log_prob_[class_index]
|
||||||
|
domain = model.named_steps["classifier"].classes_[class_index]
|
||||||
|
|
||||||
results.append(
|
results.append(
|
||||||
DomainPrediction(
|
DomainPrediction(
|
||||||
domain=model.named_steps["classifier"].classes_[class_index],
|
domain=domain,
|
||||||
probability=round(probability * 100),
|
probability=round(probability * 100),
|
||||||
explanation=_get_explanation(
|
explanation=_get_explanation(
|
||||||
feature_names=feature_names,
|
feature_names=feature_names,
|
||||||
|
|
@ -69,11 +72,11 @@ def _get_explanation(
|
||||||
token_mapping: Dict[str, str],
|
token_mapping: Dict[str, str],
|
||||||
) -> List[str]:
|
) -> List[str]:
|
||||||
influential = [
|
influential = [
|
||||||
(value * weight, name)
|
(weight, name)
|
||||||
for name, value, weight in zip(feature_names, features, weights)
|
for weight, value, name in zip(weights, features, feature_names)
|
||||||
if value
|
if value
|
||||||
]
|
]
|
||||||
|
|
||||||
most_influential = sorted(influential, reverse=True)[:5]
|
most_influential = sorted(influential, reverse=True)[:5]
|
||||||
|
|
||||||
return [token_mapping[v[1]] for v in most_influential]
|
return [token_mapping[name] for _, name in most_influential]
|
||||||
|
|
|
||||||
File diff suppressed because one or more lines are too long
Loading…
Add table
Add a link
Reference in a new issue