andras/great-ai
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases 4 Packages Wiki Activity Actions

Improve dashboard

Browse source
This commit is contained in:
Andras Schmelczer 2022-06-04 14:06:32 +02:00
parent 471c258cb0
commit 75a33544ad
30 changed files with 690 additions and 416 deletions
Download patch file Download diff file Expand all files Collapse all files

6
.vscode/settings.json vendored
View file

@ -2,6 +2,8 @@
"cSpell.words": [
"boto",
"botocore",
"datatable",
"displaylogo",
"fastapi",
"gridfs",
"iloc",
@ -10,6 +12,8 @@
"levelno",
"matplotlib",
"nbconvert",
"nbins",
"Parcoords",
"plotly",
"proba",
"psutil",
@ -21,6 +25,8 @@
"starlette",
"sublinear",
"Tfidf",
"ticktext",
"tickvals",
"tinydb",
"uvicorn",
"Vectorizer",

12
examples/simple/deploy.ipynb
View file

@ -133,14 +133,14 @@
}
],
"source": [
"result = predict_domain(\n",
" \"\"\"\n",
" State-of-the-art methods for zero-shot visual recognition formulate learning as a joint embedding problem of images and side information. In these formulations the current best complement to visual features are attributes: manually encoded vectors describing shared characteristics among categories. Despite good performance, attributes have limitations: (1) finer-grained recognition requires commensurately more, and (2) attributes do not provide a natural language interface. We propose to overcome these limitations by training neural language models from scratch; i.e. without pre-training and only consuming words and characters. Our proposed models train end-to-end to align with the fine-grained and category-specific content of images. Natural language provides a flexible and compact way of encoding only the salient visual aspects for distinguishing categories. By training on raw text, our model can do inference on raw text as well, providing humans a familiar mode both for annotation and retrieval. Our model achieves strong performance on zero-shot text-based image retrieval and significantly outperforms the attribute-based state-of-the-art for zero-shot classification on the CaltechUCSD Birds 200-2011 dataset. \"\"\"\n",
")\n",
"# result = predict_domain(\n",
"# \"\"\"\n",
"# State-of-the-art methods for zero-shot visual recognition formulate learning as a joint embedding problem of images and side information. In these formulations the current best complement to visual features are attributes: manually encoded vectors describing shared characteristics among categories. Despite good performance, attributes have limitations: (1) finer-grained recognition requires commensurately more, and (2) attributes do not provide a natural language interface. We propose to overcome these limitations by training neural language models from scratch; i.e. without pre-training and only consuming words and characters. Our proposed models train end-to-end to align with the fine-grained and category-specific content of images. Natural language provides a flexible and compact way of encoding only the salient visual aspects for distinguishing categories. By training on raw text, our model can do inference on raw text as well, providing humans a familiar mode both for annotation and retrieval. Our model achieves strong performance on zero-shot text-based image retrieval and significantly outperforms the attribute-based state-of-the-art for zero-shot classification on the CaltechUCSD Birds 200-2011 dataset. \"\"\"\n",
"# )\n",
"\n",
"from pprint import pprint\n",
"# from pprint import pprint\n",
"\n",
"# pprint(result.dict(), width=120)"
"# # pprint(result.dict(), width=120)"
]
}
],

4
great_ai/src/great_ai/great_ai/constants.py
View file

@ -5,10 +5,12 @@ from great_ai.large_file import LargeFileLocal, LargeFileMongo, LargeFileS3
ENV_VAR_KEY = "ENVIRONMENT"
PRODUCTION_KEY = "production"
DEFAULT_TRACING_DB_FILENAME = "tracing_database.json"
METRICS_PATH = "/metrics"
DASHBOARD_PATH = "/dashboard"
DEFAULT_LARGE_FILE_CONFIG_PATHS = {
LargeFileLocal: None,
LargeFileMongo: Path("mongodb.ini"),
LargeFileS3: Path("s3.ini"),
}
GITHUB_LINK = "https://github.com/ScoutinScience/great-ai"

2
great_ai/src/great_ai/great_ai/context/configure.py
View file

@ -18,7 +18,6 @@ from ..tracing.parallel_tinydb_driver import ParallelTinyDbDriver, TracingDataba
def configure(
version: str = "0.0.1",
log_level: int = DEBUG,
seed: int = 42,
tracing_database: TracingDatabase = ParallelTinyDbDriver(
@ -46,7 +45,6 @@ def configure(
)
context._context = context.Context(
version=version,
tracing_database=tracing_database,
large_file_implementation=large_file_implementation,
is_production=is_production,

4
great_ai/src/great_ai/great_ai/context/context.py
View file

@ -9,7 +9,6 @@ from ..tracing.tracing_database import TracingDatabase
class Context(BaseModel):
version: str
tracing_database: TracingDatabase
large_file_implementation: Type[LargeFile]
is_production: bool
@ -22,12 +21,11 @@ class Context(BaseModel):
def to_flat_dict(self) -> Dict[str, Any]:
return {
"version": self.version,
"tracing_database": type(self.tracing_database).__name__,
"large_file_implementation": self.large_file_implementation.__name__,
"is_production": self.is_production,
"logger": type(self.logger).__name__,
"should_log_exception_stack": self.should_log_exception_stack,
"prediction_cache_size": self.prediction_cache_size,
}

112
great_ai/src/great_ai/great_ai/dashboard/assets/index.css
View file

@ -1,112 +0,0 @@
:root {
--accent-color: #47c2d0;
--error-color: #a30808;
--background-color: #edf5f6;
--small-padding: 10px;
--medium-padding: 20px;
--large-padding: 40px;
--border-radius: 10px;
--shadow: 0 4px 6px -1px rgb(0 0 0 / 10%), 0 2px 4px -1px rgb(0 0 0 / 6%);
}
* {
margin: 0;
box-sizing: border-box;
}
body {
background-color: var(--background-color);
font-family: Arial, Helvetica, sans-serif;
}
h1,
h2,
h3,
h4,
h5,
h6 {
margin: var(--medium-padding) 0 var(--small-padding) 0;
}
.glance,
.table-container,
.parallel-coords {
padding: var(--large-padding);
margin: var(--large-padding);
border-radius: var(--border-radius);
box-shadow: var(--shadow);
background-color: white;
overflow: hidden;
}
.glance {
display: flex;
}
.glance .description {
width: 350px;
display: flex;
flex-direction: column;
justify-content: center;
}
.glance .dash-graph {
flex: 1;
}
.table-container {
padding: 0;
}
.table-container h2 {
padding: var(--small-padding);
margin: 0;
}
.table-container h2,
footer.watermark {
opacity: 0.35;
}
.dash-spreadsheet {
overflow: auto;
}
th,
td {
max-width: 350px;
}
.parallel-coords {
padding: 0;
}
footer.watermark {
display: flex;
justify-content: space-between;
align-items: center;
padding: var(--large-padding);
background-color: #ddd;
position: relative;
}
footer.watermark div {
display: flex;
flex-direction: column;
}
h6 {
font-size: 3rem;
margin-top: 0;
}
a img {
width: 64px;
height: 64px;
cursor: pointer;
transition: transform 300ms;
}
a img:hover {
transform: scale(1.1);
}

176
great_ai/src/great_ai/great_ai/dashboard/create_dash_app.py
View file

@ -1,176 +0,0 @@
from typing import Any, Dict, List
import pandas as pd
import plotly.express as px
import plotly.graph_objects as go
from dash import Dash, dash_table, dcc, html
from dash.dependencies import Input, Output
from flask import Flask
from great_ai.utilities.unique import unique
from ..constants import METRICS_PATH
from ..context import get_context
from ..helper import snake_case_to_text, text_to_hex_color
from ..views import SortBy
from .get_description import get_description
from .get_filter_from_datatable import get_filter_from_datatable
from .get_footer import get_footer
def create_dash_app(function_name: str, function_docs: str) -> Flask:
accent_color = text_to_hex_color(function_name)
flask_app = Flask(__name__)
app = Dash(
function_name,
requests_pathname_prefix=METRICS_PATH + "/",
server=flask_app,
title=snake_case_to_text(function_name),
update_title=None,
external_stylesheets=[
"/assets/index.css",
],
)
documents = get_context().tracing_database.query()
df = pd.DataFrame(documents)
execution_time_histogram = dcc.Graph(config={"displaylogo": False})
table = dash_table.DataTable(
columns=[{"name": i, "id": i} for i in df.columns],
page_current=0,
page_size=20,
page_action="custom",
filter_action="custom",
filter_query="",
sort_action="custom",
sort_mode="multi",
sort_by=[
{"column_id": "created", "direction": "desc"},
],
)
app.layout = html.Div(
[
html.Div(
[
get_description(
function_name=function_name,
function_docs=function_docs,
accent_color=accent_color,
),
execution_time_histogram,
],
className="glance",
),
html.Div([html.H2("Latest traces"), table], className="table-container"),
parallel_coords := dcc.Graph(
className="parallel-coords", config={"displaylogo": False}
),
get_footer(),
interval := dcc.Interval(
interval=4 * 1000, # in milliseconds
),
]
)
@app.callback(
Output(table, "data"),
Input(table, "page_current"),
Input(table, "page_size"),
Input(table, "sort_by"),
Input(table, "filter_query"),
Input(interval, "n_intervals"),
)
def update_table(
page_current: int,
page_size: int,
sort_by: List[SortBy],
filter: str,
n_intervals: int,
) -> List[Dict[str, Any]]:
conjunctive_filters = [
get_filter_from_datatable(f) for f in filter.split(" && ")
]
non_null_conjunctive_filters = [f for f in conjunctive_filters if f is not None]
return get_context().tracing_database.query(
skip=page_current * page_size,
take=page_size,
conjunctive_filters=non_null_conjunctive_filters,
sort_by=sort_by,
)
@app.callback(
Output(execution_time_histogram, "figure"),
Output(parallel_coords, "figure"),
Input(table, "filter_query"),
Input(interval, "n_intervals"),
)
def update_charts(filter: str, n_intervals: int) -> go.Figure:
conjunctive_filters = [
get_filter_from_datatable(f) for f in filter.split(" && ")
]
non_null_conjunctive_filters = [f for f in conjunctive_filters if f is not None]
rows = get_context().tracing_database.query(
conjunctive_filters=non_null_conjunctive_filters
)
if not rows:
return go.Figure(), go.Figure()
df = pd.DataFrame(rows)
fig = px.histogram(
df,
x="execution_time_ms",
labels={"execution_time_ms": "Execution time (ms)"},
nbins=20,
height=400,
log_y=True,
color_discrete_sequence=[accent_color],
)
fig.update_layout(
autosize=True,
margin=dict(l=0, r=0, b=0, t=0, pad=0),
)
return (
fig,
go.Figure(
go.Parcoords(
dimensions=[
get_dimension_descriptor(df, c)
for c in df.columns
if c not in {"id", "created", "output"}
],
line_color=accent_color,
)
),
)
return flask_app
def get_dimension_descriptor(df: pd.DataFrame, column: str) -> Dict[str, Any]:
dimension: Dict[str, Any] = {
"label": snake_case_to_text(column),
}
values = df[column]
try:
dimension["values"] = [float(v) for v in values]
except (TypeError, ValueError):
MAX_LENGTH = 40
unique_values = unique(values)
value_mapping = {str(v)[-MAX_LENGTH:]: i for i, v in enumerate(unique_values)}
dimension["values"] = [value_mapping[str(v)[-MAX_LENGTH:]] for v in values]
dimension["tickvals"] = list(value_mapping.values())
dimension["ticktext"] = [k[-MAX_LENGTH:] for k in value_mapping.keys()]
return dimension

52
great_ai/src/great_ai/great_ai/deploy/great_ai.py
View file

@ -5,12 +5,12 @@ from typing import Any, Callable, Iterable, Optional, Sequence, Type, Union, cas
from fastapi import APIRouter, FastAPI, status
from pydantic import BaseModel, create_model
from great_ai.great_ai.deploy.routes.bootstrap_dashboard import bootstrap_dashboard
from great_ai.great_ai.views.cache_statistics import CacheStatistics
from great_ai.utilities.parallel_map import parallel_map
from ..constants import METRICS_PATH
from ..constants import DASHBOARD_PATH
from ..context import get_context
from ..dashboard import create_dash_app
from ..helper import (
freeze_arguments,
get_function_metadata_store,
@ -28,25 +28,31 @@ from .routes import (
class GreatAI:
def __init__(self, func: Callable[..., Any]):
def __init__(self, func: Callable[..., Any], version: str):
self._func = automatically_decorate_parameters(func)
get_function_metadata_store(self._func).is_finalised = True
self._cached_func = lru_cache(get_context().prediction_cache_size)(
self._func
) # cannot put decorator on method, because it require the context to be setup
wraps(func)(self)
self._version = version
self.app = FastAPI(
title=self.name,
version=self.version,
description=self.documentation
+ f"\n\n Find out more on the [metrics page]({METRICS_PATH}).",
+ f"\n\nFind out more in the [dashboard]({DASHBOARD_PATH}).",
docs_url=None,
redoc_url=None,
)
@freeze_arguments
@lru_cache(get_context().prediction_cache_size)
def __call__(self, *args: Any, **kwargs: Any) -> Trace:
with TracingContext() as t:
result = self._func(*args, **kwargs)
result = self._cached_func(*args, **kwargs)
output = t.finalise(output=result)
return output
@ -54,9 +60,10 @@ class GreatAI:
def deploy(
func: Optional[Callable[..., Any]] = None,
*,
version: str = "0.0.1",
disable_rest_api: bool = False,
disable_docs: bool = False,
disable_metrics: bool = False,
disable_dashboard: bool = False,
) -> Union[Callable[[Callable[..., Any]], "GreatAI"], "GreatAI"]:
if func is None:
return cast(
@ -65,15 +72,15 @@ class GreatAI:
GreatAI.deploy,
disable_http=disable_rest_api,
disable_docs=disable_docs,
disable_metrics=disable_metrics,
disable_dashboard=disable_dashboard,
),
)
instance = GreatAI(func)
instance = GreatAI(func, version=version)
if not disable_rest_api:
instance._bootstrap_rest_api(
disable_docs=disable_docs, disable_metrics=disable_metrics
disable_docs=disable_docs, disable_dashboard=disable_dashboard
)
return instance
@ -95,7 +102,9 @@ class GreatAI:
@property
def version(self) -> str:
return f"{get_context().version}+{get_function_metadata_store(self._func).model_versions}"
return (
f"{self._version}+{get_function_metadata_store(self._func).model_versions}"
)
@property
def documentation(self) -> str:
@ -110,23 +119,26 @@ class GreatAI:
)
)
def _bootstrap_rest_api(self, disable_docs: bool, disable_metrics: bool) -> None:
self._bootstrap_prediction_endpoints()
def _bootstrap_rest_api(self, disable_docs: bool, disable_dashboard: bool) -> None:
self._bootstrap_prediction_endpoint()
if not disable_docs:
bootstrap_docs_endpoints(self.app)
if not disable_metrics:
dash_app = create_dash_app(self._func.__name__, self.documentation)
bootstrap_trace_endpoints(self.app, dash_app)
if not disable_dashboard:
bootstrap_dashboard(
self.app,
function_name=self._func.__name__,
documentation=self.documentation,
)
bootstrap_trace_endpoints(self.app)
bootstrap_feedback_endpoints(self.app)
self._bootstrap_meta_endpoints()
def _bootstrap_prediction_endpoints(self) -> None:
def _bootstrap_prediction_endpoint(self) -> None:
router = APIRouter(
prefix="/predictions",
prefix="/predict",
tags=["predictions"],
)
@ -160,7 +172,7 @@ class GreatAI:
@router.get("/health", status_code=status.HTTP_200_OK)
def check_health() -> HealthCheckResponse:
hits, misses, maxsize, cache_size = self.__call__.cache_info() # type: ignore
hits, misses, maxsize, cache_size = self._cached_func.cache_info()
cache_statistics = CacheStatistics(
hits=hits, misses=misses, size=cache_size, max_size=maxsize
)

1
great_ai/src/great_ai/great_ai/deploy/routes/__init__.py
View file

@ -1,3 +1,4 @@
from .bootstrap_dashboard import bootstrap_dashboard
from .bootstrap_docs_endpoints import bootstrap_docs_endpoints
from .bootstrap_feedback_endpoints import bootstrap_feedback_endpoints
from .bootstrap_trace_endpoints import bootstrap_trace_endpoints

27
great_ai/src/great_ai/great_ai/deploy/routes/bootstrap_dashboard.py Normal file
View file

@ -0,0 +1,27 @@
from pathlib import Path
from fastapi import FastAPI
from fastapi.middleware.wsgi import WSGIMiddleware
from fastapi.responses import RedirectResponse
from fastapi.staticfiles import StaticFiles
from ...constants import DASHBOARD_PATH
from .dashboard import create_dash_app
PATH = Path(__file__).parent.resolve()
def bootstrap_dashboard(app: FastAPI, function_name: str, documentation: str) -> None:
dash_app = create_dash_app(function_name, documentation)
app.mount(DASHBOARD_PATH, WSGIMiddleware(dash_app))
@app.get("/", include_in_schema=False)
def redirect_to_entrypoint() -> RedirectResponse:
return RedirectResponse(DASHBOARD_PATH)
app.mount(
"/assets",
StaticFiles(directory=PATH / "dashboard/assets"),
name="static",
)

9
great_ai/src/great_ai/great_ai/deploy/routes/bootstrap_feedback_endpoints.py
View file

@ -1,5 +1,6 @@
from typing import Any
import yaml
from fastapi import APIRouter, FastAPI, HTTPException, Response, status
from ...context import get_context
@ -17,7 +18,12 @@ def bootstrap_feedback_endpoints(app: FastAPI) -> None:
trace = get_context().tracing_database.get(trace_id)
if trace is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
trace.feedback = input.feedback
trace.feedback_flat = yaml.dump(
input.feedback, default_flow_style=False, indent=2
)
get_context().tracing_database.update(trace_id, trace)
return Response(status_code=status.HTTP_202_ACCEPTED)
@ -33,7 +39,10 @@ def bootstrap_feedback_endpoints(app: FastAPI) -> None:
trace = get_context().tracing_database.get(trace_id)
if trace is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
trace.feedback = None
trace.feedback_flat = None
get_context().tracing_database.update(trace_id, trace)
return Response(status_code=status.HTTP_204_NO_CONTENT)

30
great_ai/src/great_ai/great_ai/deploy/routes/bootstrap_trace_endpoints.py
View file

@ -1,38 +1,18 @@
from pathlib import Path
from typing import List
from fastapi import APIRouter, FastAPI, HTTPException, Response, status
from fastapi.middleware.wsgi import WSGIMiddleware
from fastapi.responses import RedirectResponse
from fastapi.staticfiles import StaticFiles
from flask import Flask
from ...constants import METRICS_PATH
from ...context import get_context
from ...views import Query, Trace
PATH = Path(__file__).parent.resolve()
from ...views import Query, Trace, TraceView
def bootstrap_trace_endpoints(app: FastAPI, dash_app: Flask) -> None:
app.mount(METRICS_PATH, WSGIMiddleware(dash_app))
@app.get("/", include_in_schema=False)
def redirect_to_entrypoint() -> RedirectResponse:
return RedirectResponse("/metrics")
app.mount(
"/assets",
StaticFiles(directory=PATH / "../../dashboard/assets"),
name="static",
)
def bootstrap_trace_endpoints(app: FastAPI) -> None:
router = APIRouter(
prefix="/traces",
tags=["traces"],
)
@router.post("/", status_code=status.HTTP_200_OK, response_model=List[Trace])
@router.post("/", status_code=status.HTTP_200_OK, response_model=List[TraceView])
def query_traces(
query: Query,
skip: int = 0,
@ -43,9 +23,9 @@ def bootstrap_trace_endpoints(app: FastAPI, dash_app: Flask) -> None:
sort_by=query.sort,
skip=skip,
take=take,
)
)[0]
@router.get("/{trace_id}", status_code=status.HTTP_200_OK, response_model=Trace)
@router.get("/{trace_id}", status_code=status.HTTP_200_OK, response_model=TraceView)
def get_trace(trace_id: str) -> Trace:
result = get_context().tracing_database.get(trace_id)
if result is None:

0
great_ai/src/great_ai/great_ai/dashboard/__init__.py → great_ai/src/great_ai/great_ai/deploy/routes/dashboard/__init__.py
View file

0
great_ai/src/great_ai/great_ai/dashboard/assets/github.png → great_ai/src/great_ai/great_ai/deploy/routes/dashboard/assets/github.png
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 4.2 KiB

After

Width:  |  Height:  |  Size: 4.2 KiB

Before After
Before After

227
great_ai/src/great_ai/great_ai/deploy/routes/dashboard/assets/index.css Normal file
View file

@ -0,0 +1,227 @@
:root {
--important-color: #a30808;
--background-color: #edf5f6;
--small-padding: 10px;
--medium-padding: 20px;
--large-padding: 40px;
--border-radius: 10px;
--shadow: 0 4px 6px -1px rgb(0 0 0 / 10%), 0 2px 4px -1px rgb(0 0 0 / 6%);
--disclaimer-width: 180px;
--disclaimer-height: 35px;
}
@media (max-width: 900px) {
body {
zoom: 0.8;
}
}
@media (max-width: 550px) {
:root {
--small-padding: 5px;
--medium-padding: 10px;
--large-padding: 20px;
--border-radius: 8px;
}
.environment {
margin-top: calc(-1 * var(--large-padding));
margin-bottom: var(--large-padding);
}
}
@media (min-width: 551px) {
.environment {
position: absolute;
width: var(--disclaimer-width);
height: var(--disclaimer-height);
transform: rotate(-45deg);
top: calc(
var(--disclaimer-width) / 1.4142 - var(--disclaimer-height) / 1.4142
);
left: calc(-1 * var(--disclaimer-height) / 1.4142);
transform-origin: top left;
z-index: 100;
}
}
* {
margin: 0;
box-sizing: border-box;
word-break: break-word;
}
body {
background-color: var(--background-color);
font-family: Arial, Helvetica, sans-serif;
}
h1,
h2,
h3,
h4,
h5,
h6 {
margin: var(--medium-padding) 0 var(--small-padding) 0;
}
h6 {
margin-top: 0;
font-size: 3rem;
}
html,
body,
#react-entry-point,
main {
height: 100%;
}
main {
padding-top: var(--large-padding);
display: flex;
flex-direction: column;
}
.environment {
background-color: var(--important-color);
color: white;
text-align: center;
display: flex;
align-items: center;
justify-content: center;
}
main > header,
.configuration-container,
.traces-table-container,
.parallel-coordinates,
main > footer {
padding: var(--large-padding);
flex-shrink: 0;
overflow: hidden;
}
main > header,
.configuration-container,
.traces-table-container,
.parallel-coordinates {
margin: 0 var(--large-padding) var(--large-padding) var(--large-padding);
border-radius: var(--border-radius);
box-shadow: var(--shadow);
background-color: white;
}
main > header {
display: flex;
align-items: center;
flex-wrap: wrap;
justify-content: space-between;
}
main > header > div {
min-width: 350px;
max-width: 450px;
margin-bottom: var(--large-padding);
flex: 1;
}
main > header > div > h1 {
margin-top: 0;
}
main > header > *:nth-child(2) {
min-width: 250px;
max-width: 550px;
flex: 1;
}
main > header .placeholder {
opacity: 0.35;
font-size: 1.5rem;
text-align: center;
display: block;
min-width: 250px;
width: 60%;
margin: auto;
}
.configuration-container {
display: flex;
justify-content: space-between;
flex-wrap: wrap;
}
.configuration-item {
border-left: 2px solid var(--important-color);
padding-left: var(--small-padding);
margin: var(--medium-padding);
}
.configuration-item h4 {
font-weight: bold;
margin: 0 0 var(--small-padding) 0;
}
.traces-table-container {
padding: 0;
}
.traces-table-container header {
padding: var(--large-padding);
}
.traces-table-container header h2 {
margin-top: 0;
}
.dash-filter--case {
display: none;
}
.traces-table-container td > div {
white-space: pre !important;
max-height: 150px !important;
overflow: auto !important;
display: inline-block !important;
text-align: left !important;
}
.traces-table-container th > div {
text-align: left !important;
}
.space-filler {
flex-grow: 1;
}
main > footer {
opacity: 0.35;
margin: 0;
}
main > footer {
display: flex;
justify-content: space-between;
align-items: center;
padding: var(--large-padding);
background-color: #ddd;
position: relative;
}
.parallel-coordinates {
padding: 0;
}
a img {
display: block;
margin-left: var(--large-padding);
width: 64px;
height: 64px;
cursor: pointer;
transition: transform 300ms;
}
a img:hover {
transform: scale(1.1);
}

254
great_ai/src/great_ai/great_ai/deploy/routes/dashboard/create_dash_app.py Normal file
View file

@ -0,0 +1,254 @@
from math import ceil
from typing import Any, Dict, List, Tuple
import pandas as pd
import plotly.express as px
import plotly.graph_objects as go
from dash import Dash, dcc, html
from dash.dependencies import Input, Output
from flask import Flask
from great_ai.utilities.unique import unique
from ....constants import DASHBOARD_PATH
from ....context import get_context
from ....helper import snake_case_to_text, text_to_hex_color
from ....views import SortBy
from .get_description import get_description
from .get_filter_from_datatable import get_filter_from_datatable
from .get_footer import get_footer
from .get_traces_table import get_traces_table
def create_dash_app(function_name: str, function_docs: str) -> Flask:
accent_color = text_to_hex_color(function_name)
flask_app = Flask(__name__)
app = Dash(
function_name,
requests_pathname_prefix=DASHBOARD_PATH + "/",
server=flask_app,
title=snake_case_to_text(function_name),
update_title=None,
external_stylesheets=[
"/assets/index.css",
],
)
app.layout = html.Main(
[
html.Div(
html.P("PRODUCTION" if get_context().is_production else "DEVELOPMENT"),
className="environment",
),
html.Header(
[
get_description(
function_name=function_name,
function_docs=function_docs,
accent_color=accent_color,
),
execution_time_histogram_container := html.Div(),
],
),
configuration_container := html.Div(
className="configuration-container",
),
traces_table_container := html.Div(
[
html.Header(
[
html.H2("Latest traces"),
html.P(
"Recent traces and aggregated metrics are presented below. Try filtering the table."
),
html.A(
"Filtering syntax.",
href="https://dash.plotly.com/datatable/filtering",
target="_blank",
),
]
),
table := get_traces_table(),
],
className="traces-table-container",
),
parallel_coordinates := dcc.Graph(
className="parallel-coordinates", config={"displaylogo": False}
),
html.Div(className="space-filler"),
get_footer(),
interval := dcc.Interval(
interval=4 * 1000, # in milliseconds
),
]
)
@app.callback(
Output(configuration_container, "children"),
Input(interval, "n_intervals"),
)
def update_configuration(
n_intervals: int,
) -> List[html.Div]:
config = get_context().to_flat_dict()
return [
html.Div(
[
html.H4(snake_case_to_text(key)),
html.P(str(value)),
],
className="configuration-item",
)
for key, value in config.items()
]
@app.callback(
Output(table, "data"),
Output(table, "page_count"),
Input(table, "page_current"),
Input(table, "page_size"),
Input(table, "sort_by"),
Input(table, "filter_query"),
Input(interval, "n_intervals"),
)
def update_table(
page_current: int,
page_size: int,
sort_by: List[SortBy],
filter_query: str,
n_intervals: int,
) -> Tuple[List[Dict[str, Any]], int]:
conjunctive_filters = (
[get_filter_from_datatable(f) for f in filter_query.split(" && ")]
if filter_query
else []
)
non_null_conjunctive_filters = [f for f in conjunctive_filters if f is not None]
elements, count = get_context().tracing_database.query(
skip=page_current * page_size,
take=page_size,
conjunctive_filters=non_null_conjunctive_filters,
sort_by=sort_by,
)
return (
[e.to_flat_dict() for e in elements],
max(1, ceil(count / page_size)),
)
@app.callback(
Output(table, "columns"),
Output(traces_table_container, "style"),
Input(interval, "n_intervals"),
)
def update_layout(
n_intervals: int,
) -> Tuple[List[Dict[str, str]], Dict[str, Any]]:
elements, count = get_context().tracing_database.query(take=1)
if elements:
keys = list(elements[0].to_flat_dict().keys())
header_height = max(len(i.split(":")) for i in keys)
columns = [
{
"name": [""] * (header_height - len(k.split(":")))
+ k.replace("_flat", "").split(":"),
"id": k,
}
for k in keys
]
else:
columns = []
return (
columns,
{"display": "block" if count > 0 else "none"},
)
@app.callback(
Output(execution_time_histogram_container, "children"),
Output(parallel_coordinates, "figure"),
Output(parallel_coordinates, "style"),
Input(table, "filter_query"),
Input(interval, "n_intervals"),
)
def update_charts(
filter_query: str, n_intervals: int
) -> Tuple[Any, go.Figure, Dict[str, Any]]:
conjunctive_filters = (
[get_filter_from_datatable(f) for f in filter_query.split(" && ")]
if filter_query
else []
)
non_null_conjunctive_filters = [f for f in conjunctive_filters if f is not None]
elements, count = get_context().tracing_database.query(
conjunctive_filters=non_null_conjunctive_filters
)
elements = [e.to_flat_dict() for e in elements]
if not elements:
return (
html.Span(
f"No traces yet: call your function ({function_name}) to create one.",
className="placeholder",
),
go.Figure(),
{"display": "none"},
)
execution_time_histogram = dcc.Graph(config={"displaylogo": False})
df = pd.DataFrame(elements)
fig = px.histogram(
df,
x="original_execution_time_ms",
labels={"original_execution_time_ms": "Execution time (ms)"},
nbins=20,
height=400,
log_y=True,
color_discrete_sequence=[accent_color],
)
fig.update_layout(
margin=dict(l=0, r=0, b=0, t=0, pad=0),
)
execution_time_histogram.figure = fig
parallel_coords_fig = go.Figure(
go.Parcoords(
dimensions=[
get_dimension_descriptor(df, c)
for c in df.columns
if c
not in {"trace_id", "created", "output", "exception", "feedback"}
and "_flat" not in c
],
line_color=accent_color,
)
)
return execution_time_histogram, parallel_coords_fig, {}
return flask_app
def get_dimension_descriptor(df: pd.DataFrame, column: str) -> Dict[str, Any]:
dimension: Dict[str, Any] = {
"label": snake_case_to_text(column),
}
values = df[column]
try:
dimension["values"] = [float(v) for v in values]
except (TypeError, ValueError):
MAX_LENGTH = 40
unique_values = unique(values)
value_mapping = {str(v)[-MAX_LENGTH:]: i for i, v in enumerate(unique_values)}
dimension["values"] = [value_mapping[str(v)[-MAX_LENGTH:]] for v in values]
dimension["tickvals"] = list(value_mapping.values())
dimension["ticktext"] = [k[-MAX_LENGTH:] for k in value_mapping.keys()]
return dimension

12
great_ai/src/great_ai/great_ai/dashboard/get_description.py → great_ai/src/great_ai/great_ai/deploy/routes/dashboard/get_description.py
View file

@ -1,6 +1,6 @@
from dash import dcc, html
from ..helper import snake_case_to_text, strip_lines
from ....helper import snake_case_to_text, strip_lines
def get_description(
@ -9,25 +9,21 @@ def get_description(
return html.Div(
[
html.H1(
f"{snake_case_to_text(function_name)} - metrics",
f"{snake_case_to_text(function_name)} - dashboard",
style={"color": accent_color},
),
dcc.Markdown(
strip_lines(
f"""
> View the live data of your deployments here.
> View the live data of your deployment here.
## Using the API
You can find the available endpoints at [/docs](/docs).
### Details
## Details
{function_docs}
## Metrics
Recent traces and aggregated metrics are presented below. Try filtering the table.
"""
),
className="description",

2
great_ai/src/great_ai/great_ai/dashboard/get_filter_from_datatable.py → great_ai/src/great_ai/great_ai/deploy/routes/dashboard/get_filter_from_datatable.py
View file

@ -1,6 +1,6 @@
from typing import Optional, Union
from ..views import Filter, operators
from ....views import Filter, operators
def get_filter_from_datatable(description: str) -> Optional[Filter]:

5
great_ai/src/great_ai/great_ai/dashboard/get_footer.py → great_ai/src/great_ai/great_ai/deploy/routes/dashboard/get_footer.py
View file

@ -1,5 +1,7 @@
from dash import html
from ....constants import GITHUB_LINK
def get_footer() -> html.Footer:
return html.Footer(
@ -14,9 +16,8 @@ def get_footer() -> html.Footer:
),
html.A(
html.Img(src="/assets/github.png"),
href="https://github.com/ScoutinScience/great-ai",
href=GITHUB_LINK,
target="_blank",
),
],
className="watermark",
)

32
great_ai/src/great_ai/great_ai/deploy/routes/dashboard/get_traces_table.py Normal file
View file

@ -0,0 +1,32 @@
from dash import dash_table
def get_traces_table() -> dash_table.DataTable:
return dash_table.DataTable(
page_current=0,
page_size=20,
page_action="custom",
filter_action="custom",
sort_action="custom",
sort_mode="multi",
sort_by=[
{"column_id": "created", "direction": "desc"},
],
style_data={
"white-space": "normal",
"height": "auto",
"max-height": "300px",
"overflow": "hidden",
"text-overflow": "ellipsis",
},
style_cell={"padding": "5px"},
style_header={
"background-color": "white",
"font-weight": "bold",
},
style_table={"max-height": "70vh", "overflow": "auto"},
merge_duplicate_headers=True,
style_cell_conditional=[
{"if": {"column_id": "output"}, "width": 1500},
],
)

1
great_ai/src/great_ai/great_ai/helper/freeze_arguments.py
View file

@ -22,5 +22,4 @@ def freeze_arguments(func: Callable[..., Any]) -> Callable[..., Any]:
}
return func(*args, **kwargs)
wrapper.cache_info = func.cache_info # type: ignore
return wrapper

4
great_ai/src/great_ai/great_ai/parameters/parameter.py
View file

@ -20,7 +20,7 @@ def parameter(
get_function_metadata_store(func).input_parameter_names.append(parameter_name)
assert_function_is_not_finalised(func)
actual_name = f"arg:{func.__name__}:{parameter_name}"
actual_name = f"arg:{parameter_name}"
@wraps(func)
def wrapper(*args: Any, **kwargs: Dict[str, Any]) -> Any:
@ -41,7 +41,7 @@ def parameter(
context = TracingContext.get_current_context()
if context and not disable_logging:
context.log_value(name=actual_name, value=argument)
context.log_value(name=f"{actual_name}:value", value=argument)
if isinstance(argument, str):
context.log_value(name=f"{actual_name}:length", value=len(argument))

35
great_ai/src/great_ai/great_ai/tracing/parallel_tinydb_driver.py
View file

@ -1,6 +1,6 @@
from multiprocessing import Lock
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional
from typing import Any, Callable, Optional, Sequence, Tuple
import pandas as pd
from tinydb import TinyDB
@ -34,39 +34,40 @@ class ParallelTinyDbDriver(TracingDatabase):
self,
skip: int = 0,
take: Optional[int] = None,
conjunctive_filters: List[Filter] = [],
sort_by: List[SortBy] = [],
) -> List[Dict[str, Any]]:
conjunctive_filters: Sequence[Filter] = [],
sort_by: Sequence[SortBy] = [],
) -> Tuple[Sequence[Trace], int]:
documents = [
d.to_flat_dict()
for d in self._safe_execute(
lambda db: [Trace.parse_obj(t) for t in db.all()]
)
Trace.parse_obj(t) for t in self._safe_execute(lambda db: db.all())
]
if not documents:
return []
return [], 0
df = pd.DataFrame(documents)
df = pd.DataFrame([d.to_flat_dict() for d in documents])
for f in conjunctive_filters:
if f.operator in operator_mapping:
operator = f.operator.lower()
if operator in operator_mapping:
df = df.loc[
getattr(df[f.property], operator_mapping[f.operator])(f.value)
]
elif f.operator == "contains":
df = df.loc[df[f.property].str.contains(f.value)]
elif operator == "contains":
df = df.loc[df[f.property].str.contains(f.value, case=False)]
if sort_by:
df = df.sort_values(
df.sort_values(
[col["column_id"] for col in sort_by],
ascending=[col["direction"] == "asc" for col in sort_by],
inplace=False,
inplace=True,
)
count = len(df)
result = df.iloc[skip:] if take is None else df.iloc[skip : skip + take]
return result.to_dict("records")
return [
next(d for d in documents if d.trace_id == trace_id)
for trace_id in result["trace_id"]
], count
def update(self, id: str, new_version: Trace) -> None:
self._safe_execute(

2
great_ai/src/great_ai/great_ai/tracing/tracing_context.py
View file

@ -29,7 +29,7 @@ class TracingContext:
delta_time = (datetime.utcnow() - self._start_time).microseconds / 1000
self._trace = Trace(
created=self._start_time.isoformat(),
execution_time_ms=delta_time,
original_execution_time_ms=delta_time,
logged_values=self._values,
models=self._models,
output=output,

8
great_ai/src/great_ai/great_ai/tracing/tracing_database.py
View file

@ -1,5 +1,5 @@
from abc import ABC, abstractmethod
from typing import List, Optional
from typing import Optional, Sequence, Tuple
from ..views import Filter, SortBy, Trace
@ -20,9 +20,9 @@ class TracingDatabase(ABC):
self,
skip: int = 0,
take: Optional[int] = None,
conjunctive_filters: List[Filter] = [],
sort_by: List[SortBy] = [],
) -> List[Trace]:
conjunctive_filters: Sequence[Filter] = [],
sort_by: Sequence[SortBy] = [],
) -> Tuple[Sequence[Trace], int]:
pass
@abstractmethod

1
great_ai/src/great_ai/great_ai/views/__init__.py
View file

@ -9,3 +9,4 @@ from .operators import operators
from .query import Query
from .sort_by import SortBy
from .trace import Trace
from .trace_view import TraceView

10
great_ai/src/great_ai/great_ai/views/operators.py
View file

@ -2,12 +2,4 @@ from typing import List, Literal
Operator = Literal[">=", "<=", "<", ">", "!=", "=", "contains"]
operators: List[Operator] = [
">=",
"<=",
"<",
">",
"!=",
"=",
"contains",
]
operators: List[Operator] = [">=", "<=", "<", ">", "!=", "=", "contains"]

8
great_ai/src/great_ai/great_ai/views/query.py
View file

@ -14,10 +14,14 @@ class Query(BaseModel):
schema_extra = {
"example": {
"filter": [
{"property": "execution_time_ms", "operator": ">", "value": 100}
{
"property": "original_execution_time_ms",
"operator": ">",
"value": 100,
}
],
"sort": [
{"column_id": "execution_time_ms", "direction": "asc"},
{"column_id": "original_execution_time_ms", "direction": "asc"},
{"column_id": "id", "direction": "desc"},
],
}

44
great_ai/src/great_ai/great_ai/views/trace.py
View file

@ -1,37 +1,33 @@
from json import dumps
from typing import Any, Dict, List, Optional
from uuid import uuid4
from typing import Any, Dict, Optional
from pydantic import BaseModel, validator
import yaml
from pydantic import validator
from .model import Model
from .trace_view import TraceView
class Trace(BaseModel):
trace_id: Optional[str]
created: str
execution_time_ms: float
logged_values: Dict[str, Any]
models: List[Model]
exception: Optional[str]
output: Any
feedback: Any = None
class Trace(TraceView):
models_flat: Optional[str]
output_flat: Optional[str]
feedback_flat: Optional[str]
@validator("trace_id", always=True)
def generate_id(cls, v: Optional[str], values: Dict[str, Any]) -> Optional[str]:
if not v:
return str(uuid4())
return v
@validator("models_flat", always=True)
def flatten_models(cls, v: Optional[str], values: Dict[str, Any]) -> str:
return ", ".join(f"{m.key}:{m.version}" for m in values["models"])
@validator("output_flat", always=True)
def flatten_output(cls, v: Optional[str], values: Dict[str, Any]) -> str:
return yaml.dump(values["output"], default_flow_style=False, indent=2)
def to_flat_dict(self) -> Dict[str, Any]:
return {
"id": self.trace_id,
"trace_id": self.trace_id,
"created": self.created,
"execution_time_ms": self.execution_time_ms,
"models": ", ".join(f"{m.key}:{m.version}" for m in self.models),
"output": dumps(self.output),
"original_execution_time_ms": self.original_execution_time_ms,
"models_flat": self.models_flat,
"output_flat": self.output_flat,
"exception": self.exception or "null",
"feedback": self.feedback,
"feedback_flat": self.feedback_flat or "null",
**self.logged_values,
}

26
great_ai/src/great_ai/great_ai/views/trace_view.py Normal file
View file

@ -0,0 +1,26 @@
from typing import Any, Dict, List, Optional
from uuid import uuid4
from pydantic import BaseModel, validator
from .model import Model
class TraceView(BaseModel):
trace_id: Optional[str]
created: str
original_execution_time_ms: float
logged_values: Dict[str, Any]
models: List[Model]
exception: Optional[str]
output: Any
feedback: Any = None
@validator("trace_id", always=True)
def generate_id(cls, v: Optional[str], values: Dict[str, Any]) -> Optional[str]:
if not v:
return str(uuid4())
return v
def __hash__(self) -> int:
return hash((type(self),) + tuple(self.__dict__.values()))