From 1ccd997324676eaed4fee252666e0297ab4705f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A1s=20Schmelczer?= Date: Sat, 4 Jun 2022 14:06:32 +0200 Subject: [PATCH] Improve dashboard --- .vscode/settings.json | 6 + examples/simple/deploy.ipynb | 12 +- great_ai/src/great_ai/great_ai/constants.py | 4 +- .../great_ai/great_ai/context/configure.py | 2 - .../src/great_ai/great_ai/context/context.py | 4 +- .../great_ai/dashboard/assets/index.css | 112 -------- .../great_ai/dashboard/create_dash_app.py | 176 ------------ .../src/great_ai/great_ai/deploy/great_ai.py | 52 ++-- .../great_ai/deploy/routes/__init__.py | 1 + .../deploy/routes/bootstrap_dashboard.py | 27 ++ .../routes/bootstrap_feedback_endpoints.py | 9 + .../routes/bootstrap_trace_endpoints.py | 30 +-- .../{ => deploy/routes}/dashboard/__init__.py | 0 .../routes}/dashboard/assets/github.png | Bin .../deploy/routes/dashboard/assets/index.css | 227 ++++++++++++++++ .../routes/dashboard/create_dash_app.py | 254 ++++++++++++++++++ .../routes}/dashboard/get_description.py | 12 +- .../dashboard/get_filter_from_datatable.py | 2 +- .../routes}/dashboard/get_footer.py | 5 +- .../routes/dashboard/get_traces_table.py | 32 +++ .../great_ai/helper/freeze_arguments.py | 1 - .../great_ai/great_ai/parameters/parameter.py | 4 +- .../tracing/parallel_tinydb_driver.py | 35 +-- .../great_ai/tracing/tracing_context.py | 2 +- .../great_ai/tracing/tracing_database.py | 8 +- .../src/great_ai/great_ai/views/__init__.py | 1 + .../src/great_ai/great_ai/views/operators.py | 10 +- great_ai/src/great_ai/great_ai/views/query.py | 8 +- great_ai/src/great_ai/great_ai/views/trace.py | 44 ++- .../src/great_ai/great_ai/views/trace_view.py | 26 ++ 30 files changed, 690 insertions(+), 416 deletions(-) delete mode 100644 great_ai/src/great_ai/great_ai/dashboard/assets/index.css delete mode 100644 great_ai/src/great_ai/great_ai/dashboard/create_dash_app.py create mode 100644 great_ai/src/great_ai/great_ai/deploy/routes/bootstrap_dashboard.py rename great_ai/src/great_ai/great_ai/{ => deploy/routes}/dashboard/__init__.py (100%) rename great_ai/src/great_ai/great_ai/{ => deploy/routes}/dashboard/assets/github.png (100%) create mode 100644 great_ai/src/great_ai/great_ai/deploy/routes/dashboard/assets/index.css create mode 100644 great_ai/src/great_ai/great_ai/deploy/routes/dashboard/create_dash_app.py rename great_ai/src/great_ai/great_ai/{ => deploy/routes}/dashboard/get_description.py (62%) rename great_ai/src/great_ai/great_ai/{ => deploy/routes}/dashboard/get_filter_from_datatable.py (95%) rename great_ai/src/great_ai/great_ai/{ => deploy/routes}/dashboard/get_footer.py (83%) create mode 100644 great_ai/src/great_ai/great_ai/deploy/routes/dashboard/get_traces_table.py create mode 100644 great_ai/src/great_ai/great_ai/views/trace_view.py diff --git a/.vscode/settings.json b/.vscode/settings.json index 55c37b8..ffe1417 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,6 +2,8 @@ "cSpell.words": [ "boto", "botocore", + "datatable", + "displaylogo", "fastapi", "gridfs", "iloc", @@ -10,6 +12,8 @@ "levelno", "matplotlib", "nbconvert", + "nbins", + "Parcoords", "plotly", "proba", "psutil", @@ -21,6 +25,8 @@ "starlette", "sublinear", "Tfidf", + "ticktext", + "tickvals", "tinydb", "uvicorn", "Vectorizer", diff --git a/examples/simple/deploy.ipynb b/examples/simple/deploy.ipynb index 02a5550..7805e61 100644 --- a/examples/simple/deploy.ipynb +++ b/examples/simple/deploy.ipynb @@ -133,14 +133,14 @@ } ], "source": [ - "result = predict_domain(\n", - " \"\"\"\n", - " State-of-the-art methods for zero-shot visual recognition formulate learning as a joint embedding problem of images and side information. In these formulations the current best complement to visual features are attributes: manually encoded vectors describing shared characteristics among categories. Despite good performance, attributes have limitations: (1) finer-grained recognition requires commensurately more, and (2) attributes do not provide a natural language interface. We propose to overcome these limitations by training neural language models from scratch; i.e. without pre-training and only consuming words and characters. Our proposed models train end-to-end to align with the fine-grained and category-specific content of images. Natural language provides a flexible and compact way of encoding only the salient visual aspects for distinguishing categories. By training on raw text, our model can do inference on raw text as well, providing humans a familiar mode both for annotation and retrieval. Our model achieves strong performance on zero-shot text-based image retrieval and significantly outperforms the attribute-based state-of-the-art for zero-shot classification on the CaltechUCSD Birds 200-2011 dataset. \"\"\"\n", - ")\n", + "# result = predict_domain(\n", + "# \"\"\"\n", + "# State-of-the-art methods for zero-shot visual recognition formulate learning as a joint embedding problem of images and side information. In these formulations the current best complement to visual features are attributes: manually encoded vectors describing shared characteristics among categories. Despite good performance, attributes have limitations: (1) finer-grained recognition requires commensurately more, and (2) attributes do not provide a natural language interface. We propose to overcome these limitations by training neural language models from scratch; i.e. without pre-training and only consuming words and characters. Our proposed models train end-to-end to align with the fine-grained and category-specific content of images. Natural language provides a flexible and compact way of encoding only the salient visual aspects for distinguishing categories. By training on raw text, our model can do inference on raw text as well, providing humans a familiar mode both for annotation and retrieval. Our model achieves strong performance on zero-shot text-based image retrieval and significantly outperforms the attribute-based state-of-the-art for zero-shot classification on the CaltechUCSD Birds 200-2011 dataset. \"\"\"\n", + "# )\n", "\n", - "from pprint import pprint\n", + "# from pprint import pprint\n", "\n", - "# pprint(result.dict(), width=120)" + "# # pprint(result.dict(), width=120)" ] } ], diff --git a/great_ai/src/great_ai/great_ai/constants.py b/great_ai/src/great_ai/great_ai/constants.py index adca62b..b3a68f6 100644 --- a/great_ai/src/great_ai/great_ai/constants.py +++ b/great_ai/src/great_ai/great_ai/constants.py @@ -5,10 +5,12 @@ from great_ai.large_file import LargeFileLocal, LargeFileMongo, LargeFileS3 ENV_VAR_KEY = "ENVIRONMENT" PRODUCTION_KEY = "production" DEFAULT_TRACING_DB_FILENAME = "tracing_database.json" -METRICS_PATH = "/metrics" +DASHBOARD_PATH = "/dashboard" DEFAULT_LARGE_FILE_CONFIG_PATHS = { LargeFileLocal: None, LargeFileMongo: Path("mongodb.ini"), LargeFileS3: Path("s3.ini"), } + +GITHUB_LINK = "https://github.com/ScoutinScience/great-ai" diff --git a/great_ai/src/great_ai/great_ai/context/configure.py b/great_ai/src/great_ai/great_ai/context/configure.py index 66fd999..559a33c 100644 --- a/great_ai/src/great_ai/great_ai/context/configure.py +++ b/great_ai/src/great_ai/great_ai/context/configure.py @@ -18,7 +18,6 @@ from ..tracing.parallel_tinydb_driver import ParallelTinyDbDriver, TracingDataba def configure( - version: str = "0.0.1", log_level: int = DEBUG, seed: int = 42, tracing_database: TracingDatabase = ParallelTinyDbDriver( @@ -46,7 +45,6 @@ def configure( ) context._context = context.Context( - version=version, tracing_database=tracing_database, large_file_implementation=large_file_implementation, is_production=is_production, diff --git a/great_ai/src/great_ai/great_ai/context/context.py b/great_ai/src/great_ai/great_ai/context/context.py index 8fd3726..05bac6f 100644 --- a/great_ai/src/great_ai/great_ai/context/context.py +++ b/great_ai/src/great_ai/great_ai/context/context.py @@ -9,7 +9,6 @@ from ..tracing.tracing_database import TracingDatabase class Context(BaseModel): - version: str tracing_database: TracingDatabase large_file_implementation: Type[LargeFile] is_production: bool @@ -22,12 +21,11 @@ class Context(BaseModel): def to_flat_dict(self) -> Dict[str, Any]: return { - "version": self.version, "tracing_database": type(self.tracing_database).__name__, "large_file_implementation": self.large_file_implementation.__name__, "is_production": self.is_production, - "logger": type(self.logger).__name__, "should_log_exception_stack": self.should_log_exception_stack, + "prediction_cache_size": self.prediction_cache_size, } diff --git a/great_ai/src/great_ai/great_ai/dashboard/assets/index.css b/great_ai/src/great_ai/great_ai/dashboard/assets/index.css deleted file mode 100644 index 5e48b88..0000000 --- a/great_ai/src/great_ai/great_ai/dashboard/assets/index.css +++ /dev/null @@ -1,112 +0,0 @@ -:root { - --accent-color: #47c2d0; - --error-color: #a30808; - --background-color: #edf5f6; - --small-padding: 10px; - --medium-padding: 20px; - --large-padding: 40px; - --border-radius: 10px; - --shadow: 0 4px 6px -1px rgb(0 0 0 / 10%), 0 2px 4px -1px rgb(0 0 0 / 6%); -} - -* { - margin: 0; - box-sizing: border-box; -} - -body { - background-color: var(--background-color); - font-family: Arial, Helvetica, sans-serif; -} - -h1, -h2, -h3, -h4, -h5, -h6 { - margin: var(--medium-padding) 0 var(--small-padding) 0; -} - -.glance, -.table-container, -.parallel-coords { - padding: var(--large-padding); - margin: var(--large-padding); - border-radius: var(--border-radius); - box-shadow: var(--shadow); - background-color: white; - overflow: hidden; -} - -.glance { - display: flex; -} - -.glance .description { - width: 350px; - display: flex; - flex-direction: column; - justify-content: center; -} - -.glance .dash-graph { - flex: 1; -} - -.table-container { - padding: 0; -} - -.table-container h2 { - padding: var(--small-padding); - margin: 0; -} - -.table-container h2, -footer.watermark { - opacity: 0.35; -} - -.dash-spreadsheet { - overflow: auto; -} - -th, -td { - max-width: 350px; -} - -.parallel-coords { - padding: 0; -} - -footer.watermark { - display: flex; - justify-content: space-between; - align-items: center; - padding: var(--large-padding); - background-color: #ddd; - position: relative; -} - -footer.watermark div { - display: flex; - flex-direction: column; -} - -h6 { - font-size: 3rem; - margin-top: 0; -} - -a img { - width: 64px; - height: 64px; - cursor: pointer; - transition: transform 300ms; -} - -a img:hover { - transform: scale(1.1); -} diff --git a/great_ai/src/great_ai/great_ai/dashboard/create_dash_app.py b/great_ai/src/great_ai/great_ai/dashboard/create_dash_app.py deleted file mode 100644 index 13578c3..0000000 --- a/great_ai/src/great_ai/great_ai/dashboard/create_dash_app.py +++ /dev/null @@ -1,176 +0,0 @@ -from typing import Any, Dict, List - -import pandas as pd -import plotly.express as px -import plotly.graph_objects as go -from dash import Dash, dash_table, dcc, html -from dash.dependencies import Input, Output -from flask import Flask - -from great_ai.utilities.unique import unique - -from ..constants import METRICS_PATH -from ..context import get_context -from ..helper import snake_case_to_text, text_to_hex_color -from ..views import SortBy -from .get_description import get_description -from .get_filter_from_datatable import get_filter_from_datatable -from .get_footer import get_footer - - -def create_dash_app(function_name: str, function_docs: str) -> Flask: - accent_color = text_to_hex_color(function_name) - - flask_app = Flask(__name__) - app = Dash( - function_name, - requests_pathname_prefix=METRICS_PATH + "/", - server=flask_app, - title=snake_case_to_text(function_name), - update_title=None, - external_stylesheets=[ - "/assets/index.css", - ], - ) - - documents = get_context().tracing_database.query() - df = pd.DataFrame(documents) - - execution_time_histogram = dcc.Graph(config={"displaylogo": False}) - table = dash_table.DataTable( - columns=[{"name": i, "id": i} for i in df.columns], - page_current=0, - page_size=20, - page_action="custom", - filter_action="custom", - filter_query="", - sort_action="custom", - sort_mode="multi", - sort_by=[ - {"column_id": "created", "direction": "desc"}, - ], - ) - - app.layout = html.Div( - [ - html.Div( - [ - get_description( - function_name=function_name, - function_docs=function_docs, - accent_color=accent_color, - ), - execution_time_histogram, - ], - className="glance", - ), - html.Div([html.H2("Latest traces"), table], className="table-container"), - parallel_coords := dcc.Graph( - className="parallel-coords", config={"displaylogo": False} - ), - get_footer(), - interval := dcc.Interval( - interval=4 * 1000, # in milliseconds - ), - ] - ) - - @app.callback( - Output(table, "data"), - Input(table, "page_current"), - Input(table, "page_size"), - Input(table, "sort_by"), - Input(table, "filter_query"), - Input(interval, "n_intervals"), - ) - def update_table( - page_current: int, - page_size: int, - sort_by: List[SortBy], - filter: str, - n_intervals: int, - ) -> List[Dict[str, Any]]: - conjunctive_filters = [ - get_filter_from_datatable(f) for f in filter.split(" && ") - ] - non_null_conjunctive_filters = [f for f in conjunctive_filters if f is not None] - - return get_context().tracing_database.query( - skip=page_current * page_size, - take=page_size, - conjunctive_filters=non_null_conjunctive_filters, - sort_by=sort_by, - ) - - @app.callback( - Output(execution_time_histogram, "figure"), - Output(parallel_coords, "figure"), - Input(table, "filter_query"), - Input(interval, "n_intervals"), - ) - def update_charts(filter: str, n_intervals: int) -> go.Figure: - conjunctive_filters = [ - get_filter_from_datatable(f) for f in filter.split(" && ") - ] - non_null_conjunctive_filters = [f for f in conjunctive_filters if f is not None] - - rows = get_context().tracing_database.query( - conjunctive_filters=non_null_conjunctive_filters - ) - - if not rows: - return go.Figure(), go.Figure() - - df = pd.DataFrame(rows) - - fig = px.histogram( - df, - x="execution_time_ms", - labels={"execution_time_ms": "Execution time (ms)"}, - nbins=20, - height=400, - log_y=True, - color_discrete_sequence=[accent_color], - ) - - fig.update_layout( - autosize=True, - margin=dict(l=0, r=0, b=0, t=0, pad=0), - ) - - return ( - fig, - go.Figure( - go.Parcoords( - dimensions=[ - get_dimension_descriptor(df, c) - for c in df.columns - if c not in {"id", "created", "output"} - ], - line_color=accent_color, - ) - ), - ) - - return flask_app - - -def get_dimension_descriptor(df: pd.DataFrame, column: str) -> Dict[str, Any]: - dimension: Dict[str, Any] = { - "label": snake_case_to_text(column), - } - - values = df[column] - - try: - dimension["values"] = [float(v) for v in values] - except (TypeError, ValueError): - MAX_LENGTH = 40 - unique_values = unique(values) - value_mapping = {str(v)[-MAX_LENGTH:]: i for i, v in enumerate(unique_values)} - - dimension["values"] = [value_mapping[str(v)[-MAX_LENGTH:]] for v in values] - dimension["tickvals"] = list(value_mapping.values()) - dimension["ticktext"] = [k[-MAX_LENGTH:] for k in value_mapping.keys()] - - return dimension diff --git a/great_ai/src/great_ai/great_ai/deploy/great_ai.py b/great_ai/src/great_ai/great_ai/deploy/great_ai.py index 0a511f2..2f51b14 100644 --- a/great_ai/src/great_ai/great_ai/deploy/great_ai.py +++ b/great_ai/src/great_ai/great_ai/deploy/great_ai.py @@ -5,12 +5,12 @@ from typing import Any, Callable, Iterable, Optional, Sequence, Type, Union, cas from fastapi import APIRouter, FastAPI, status from pydantic import BaseModel, create_model +from great_ai.great_ai.deploy.routes.bootstrap_dashboard import bootstrap_dashboard from great_ai.great_ai.views.cache_statistics import CacheStatistics from great_ai.utilities.parallel_map import parallel_map -from ..constants import METRICS_PATH +from ..constants import DASHBOARD_PATH from ..context import get_context -from ..dashboard import create_dash_app from ..helper import ( freeze_arguments, get_function_metadata_store, @@ -28,25 +28,31 @@ from .routes import ( class GreatAI: - def __init__(self, func: Callable[..., Any]): + def __init__(self, func: Callable[..., Any], version: str): self._func = automatically_decorate_parameters(func) get_function_metadata_store(self._func).is_finalised = True + + self._cached_func = lru_cache(get_context().prediction_cache_size)( + self._func + ) # cannot put decorator on method, because it require the context to be setup + wraps(func)(self) + self._version = version + self.app = FastAPI( title=self.name, version=self.version, description=self.documentation - + f"\n\n Find out more on the [metrics page]({METRICS_PATH}).", + + f"\n\nFind out more in the [dashboard]({DASHBOARD_PATH}).", docs_url=None, redoc_url=None, ) @freeze_arguments - @lru_cache(get_context().prediction_cache_size) def __call__(self, *args: Any, **kwargs: Any) -> Trace: with TracingContext() as t: - result = self._func(*args, **kwargs) + result = self._cached_func(*args, **kwargs) output = t.finalise(output=result) return output @@ -54,9 +60,10 @@ class GreatAI: def deploy( func: Optional[Callable[..., Any]] = None, *, + version: str = "0.0.1", disable_rest_api: bool = False, disable_docs: bool = False, - disable_metrics: bool = False, + disable_dashboard: bool = False, ) -> Union[Callable[[Callable[..., Any]], "GreatAI"], "GreatAI"]: if func is None: return cast( @@ -65,15 +72,15 @@ class GreatAI: GreatAI.deploy, disable_http=disable_rest_api, disable_docs=disable_docs, - disable_metrics=disable_metrics, + disable_dashboard=disable_dashboard, ), ) - instance = GreatAI(func) + instance = GreatAI(func, version=version) if not disable_rest_api: instance._bootstrap_rest_api( - disable_docs=disable_docs, disable_metrics=disable_metrics + disable_docs=disable_docs, disable_dashboard=disable_dashboard ) return instance @@ -95,7 +102,9 @@ class GreatAI: @property def version(self) -> str: - return f"{get_context().version}+{get_function_metadata_store(self._func).model_versions}" + return ( + f"{self._version}+{get_function_metadata_store(self._func).model_versions}" + ) @property def documentation(self) -> str: @@ -110,23 +119,26 @@ class GreatAI: ) ) - def _bootstrap_rest_api(self, disable_docs: bool, disable_metrics: bool) -> None: - self._bootstrap_prediction_endpoints() + def _bootstrap_rest_api(self, disable_docs: bool, disable_dashboard: bool) -> None: + self._bootstrap_prediction_endpoint() if not disable_docs: bootstrap_docs_endpoints(self.app) - if not disable_metrics: - dash_app = create_dash_app(self._func.__name__, self.documentation) - bootstrap_trace_endpoints(self.app, dash_app) + if not disable_dashboard: + bootstrap_dashboard( + self.app, + function_name=self._func.__name__, + documentation=self.documentation, + ) + bootstrap_trace_endpoints(self.app) bootstrap_feedback_endpoints(self.app) - self._bootstrap_meta_endpoints() - def _bootstrap_prediction_endpoints(self) -> None: + def _bootstrap_prediction_endpoint(self) -> None: router = APIRouter( - prefix="/predictions", + prefix="/predict", tags=["predictions"], ) @@ -160,7 +172,7 @@ class GreatAI: @router.get("/health", status_code=status.HTTP_200_OK) def check_health() -> HealthCheckResponse: - hits, misses, maxsize, cache_size = self.__call__.cache_info() # type: ignore + hits, misses, maxsize, cache_size = self._cached_func.cache_info() cache_statistics = CacheStatistics( hits=hits, misses=misses, size=cache_size, max_size=maxsize ) diff --git a/great_ai/src/great_ai/great_ai/deploy/routes/__init__.py b/great_ai/src/great_ai/great_ai/deploy/routes/__init__.py index a488a31..b5c2966 100644 --- a/great_ai/src/great_ai/great_ai/deploy/routes/__init__.py +++ b/great_ai/src/great_ai/great_ai/deploy/routes/__init__.py @@ -1,3 +1,4 @@ +from .bootstrap_dashboard import bootstrap_dashboard from .bootstrap_docs_endpoints import bootstrap_docs_endpoints from .bootstrap_feedback_endpoints import bootstrap_feedback_endpoints from .bootstrap_trace_endpoints import bootstrap_trace_endpoints diff --git a/great_ai/src/great_ai/great_ai/deploy/routes/bootstrap_dashboard.py b/great_ai/src/great_ai/great_ai/deploy/routes/bootstrap_dashboard.py new file mode 100644 index 0000000..01520a9 --- /dev/null +++ b/great_ai/src/great_ai/great_ai/deploy/routes/bootstrap_dashboard.py @@ -0,0 +1,27 @@ +from pathlib import Path + +from fastapi import FastAPI +from fastapi.middleware.wsgi import WSGIMiddleware +from fastapi.responses import RedirectResponse +from fastapi.staticfiles import StaticFiles + +from ...constants import DASHBOARD_PATH +from .dashboard import create_dash_app + +PATH = Path(__file__).parent.resolve() + + +def bootstrap_dashboard(app: FastAPI, function_name: str, documentation: str) -> None: + dash_app = create_dash_app(function_name, documentation) + + app.mount(DASHBOARD_PATH, WSGIMiddleware(dash_app)) + + @app.get("/", include_in_schema=False) + def redirect_to_entrypoint() -> RedirectResponse: + return RedirectResponse(DASHBOARD_PATH) + + app.mount( + "/assets", + StaticFiles(directory=PATH / "dashboard/assets"), + name="static", + ) diff --git a/great_ai/src/great_ai/great_ai/deploy/routes/bootstrap_feedback_endpoints.py b/great_ai/src/great_ai/great_ai/deploy/routes/bootstrap_feedback_endpoints.py index 82224d2..4848e4d 100644 --- a/great_ai/src/great_ai/great_ai/deploy/routes/bootstrap_feedback_endpoints.py +++ b/great_ai/src/great_ai/great_ai/deploy/routes/bootstrap_feedback_endpoints.py @@ -1,5 +1,6 @@ from typing import Any +import yaml from fastapi import APIRouter, FastAPI, HTTPException, Response, status from ...context import get_context @@ -17,7 +18,12 @@ def bootstrap_feedback_endpoints(app: FastAPI) -> None: trace = get_context().tracing_database.get(trace_id) if trace is None: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND) + trace.feedback = input.feedback + trace.feedback_flat = yaml.dump( + input.feedback, default_flow_style=False, indent=2 + ) + get_context().tracing_database.update(trace_id, trace) return Response(status_code=status.HTTP_202_ACCEPTED) @@ -33,7 +39,10 @@ def bootstrap_feedback_endpoints(app: FastAPI) -> None: trace = get_context().tracing_database.get(trace_id) if trace is None: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND) + trace.feedback = None + trace.feedback_flat = None + get_context().tracing_database.update(trace_id, trace) return Response(status_code=status.HTTP_204_NO_CONTENT) diff --git a/great_ai/src/great_ai/great_ai/deploy/routes/bootstrap_trace_endpoints.py b/great_ai/src/great_ai/great_ai/deploy/routes/bootstrap_trace_endpoints.py index 90fa6fa..9518291 100644 --- a/great_ai/src/great_ai/great_ai/deploy/routes/bootstrap_trace_endpoints.py +++ b/great_ai/src/great_ai/great_ai/deploy/routes/bootstrap_trace_endpoints.py @@ -1,38 +1,18 @@ -from pathlib import Path from typing import List from fastapi import APIRouter, FastAPI, HTTPException, Response, status -from fastapi.middleware.wsgi import WSGIMiddleware -from fastapi.responses import RedirectResponse -from fastapi.staticfiles import StaticFiles -from flask import Flask -from ...constants import METRICS_PATH from ...context import get_context -from ...views import Query, Trace - -PATH = Path(__file__).parent.resolve() +from ...views import Query, Trace, TraceView -def bootstrap_trace_endpoints(app: FastAPI, dash_app: Flask) -> None: - app.mount(METRICS_PATH, WSGIMiddleware(dash_app)) - - @app.get("/", include_in_schema=False) - def redirect_to_entrypoint() -> RedirectResponse: - return RedirectResponse("/metrics") - - app.mount( - "/assets", - StaticFiles(directory=PATH / "../../dashboard/assets"), - name="static", - ) - +def bootstrap_trace_endpoints(app: FastAPI) -> None: router = APIRouter( prefix="/traces", tags=["traces"], ) - @router.post("/", status_code=status.HTTP_200_OK, response_model=List[Trace]) + @router.post("/", status_code=status.HTTP_200_OK, response_model=List[TraceView]) def query_traces( query: Query, skip: int = 0, @@ -43,9 +23,9 @@ def bootstrap_trace_endpoints(app: FastAPI, dash_app: Flask) -> None: sort_by=query.sort, skip=skip, take=take, - ) + )[0] - @router.get("/{trace_id}", status_code=status.HTTP_200_OK, response_model=Trace) + @router.get("/{trace_id}", status_code=status.HTTP_200_OK, response_model=TraceView) def get_trace(trace_id: str) -> Trace: result = get_context().tracing_database.get(trace_id) if result is None: diff --git a/great_ai/src/great_ai/great_ai/dashboard/__init__.py b/great_ai/src/great_ai/great_ai/deploy/routes/dashboard/__init__.py similarity index 100% rename from great_ai/src/great_ai/great_ai/dashboard/__init__.py rename to great_ai/src/great_ai/great_ai/deploy/routes/dashboard/__init__.py diff --git a/great_ai/src/great_ai/great_ai/dashboard/assets/github.png b/great_ai/src/great_ai/great_ai/deploy/routes/dashboard/assets/github.png similarity index 100% rename from great_ai/src/great_ai/great_ai/dashboard/assets/github.png rename to great_ai/src/great_ai/great_ai/deploy/routes/dashboard/assets/github.png diff --git a/great_ai/src/great_ai/great_ai/deploy/routes/dashboard/assets/index.css b/great_ai/src/great_ai/great_ai/deploy/routes/dashboard/assets/index.css new file mode 100644 index 0000000..711fdc3 --- /dev/null +++ b/great_ai/src/great_ai/great_ai/deploy/routes/dashboard/assets/index.css @@ -0,0 +1,227 @@ +:root { + --important-color: #a30808; + --background-color: #edf5f6; + --small-padding: 10px; + --medium-padding: 20px; + --large-padding: 40px; + --border-radius: 10px; + --shadow: 0 4px 6px -1px rgb(0 0 0 / 10%), 0 2px 4px -1px rgb(0 0 0 / 6%); + --disclaimer-width: 180px; + --disclaimer-height: 35px; +} + +@media (max-width: 900px) { + body { + zoom: 0.8; + } +} + +@media (max-width: 550px) { + :root { + --small-padding: 5px; + --medium-padding: 10px; + --large-padding: 20px; + --border-radius: 8px; + } + + .environment { + margin-top: calc(-1 * var(--large-padding)); + margin-bottom: var(--large-padding); + } +} + +@media (min-width: 551px) { + .environment { + position: absolute; + width: var(--disclaimer-width); + height: var(--disclaimer-height); + transform: rotate(-45deg); + top: calc( + var(--disclaimer-width) / 1.4142 - var(--disclaimer-height) / 1.4142 + ); + left: calc(-1 * var(--disclaimer-height) / 1.4142); + transform-origin: top left; + z-index: 100; + } +} + +* { + margin: 0; + box-sizing: border-box; + word-break: break-word; +} + +body { + background-color: var(--background-color); + font-family: Arial, Helvetica, sans-serif; +} + +h1, +h2, +h3, +h4, +h5, +h6 { + margin: var(--medium-padding) 0 var(--small-padding) 0; +} + +h6 { + margin-top: 0; + font-size: 3rem; +} + +html, +body, +#react-entry-point, +main { + height: 100%; +} + +main { + padding-top: var(--large-padding); + display: flex; + flex-direction: column; +} + +.environment { + background-color: var(--important-color); + color: white; + text-align: center; + display: flex; + align-items: center; + justify-content: center; +} + +main > header, +.configuration-container, +.traces-table-container, +.parallel-coordinates, +main > footer { + padding: var(--large-padding); + flex-shrink: 0; + overflow: hidden; +} + +main > header, +.configuration-container, +.traces-table-container, +.parallel-coordinates { + margin: 0 var(--large-padding) var(--large-padding) var(--large-padding); + border-radius: var(--border-radius); + box-shadow: var(--shadow); + background-color: white; +} + +main > header { + display: flex; + align-items: center; + flex-wrap: wrap; + justify-content: space-between; +} + +main > header > div { + min-width: 350px; + max-width: 450px; + margin-bottom: var(--large-padding); + flex: 1; +} + +main > header > div > h1 { + margin-top: 0; +} + +main > header > *:nth-child(2) { + min-width: 250px; + max-width: 550px; + flex: 1; +} + +main > header .placeholder { + opacity: 0.35; + font-size: 1.5rem; + text-align: center; + display: block; + min-width: 250px; + width: 60%; + margin: auto; +} + +.configuration-container { + display: flex; + justify-content: space-between; + flex-wrap: wrap; +} + +.configuration-item { + border-left: 2px solid var(--important-color); + padding-left: var(--small-padding); + margin: var(--medium-padding); +} + +.configuration-item h4 { + font-weight: bold; + margin: 0 0 var(--small-padding) 0; +} + +.traces-table-container { + padding: 0; +} + +.traces-table-container header { + padding: var(--large-padding); +} + +.traces-table-container header h2 { + margin-top: 0; +} + +.dash-filter--case { + display: none; +} + +.traces-table-container td > div { + white-space: pre !important; + max-height: 150px !important; + overflow: auto !important; + display: inline-block !important; + text-align: left !important; +} + +.traces-table-container th > div { + text-align: left !important; +} + +.space-filler { + flex-grow: 1; +} + +main > footer { + opacity: 0.35; + margin: 0; +} + +main > footer { + display: flex; + justify-content: space-between; + align-items: center; + padding: var(--large-padding); + background-color: #ddd; + position: relative; +} + +.parallel-coordinates { + padding: 0; +} + +a img { + display: block; + margin-left: var(--large-padding); + width: 64px; + height: 64px; + cursor: pointer; + transition: transform 300ms; +} + +a img:hover { + transform: scale(1.1); +} diff --git a/great_ai/src/great_ai/great_ai/deploy/routes/dashboard/create_dash_app.py b/great_ai/src/great_ai/great_ai/deploy/routes/dashboard/create_dash_app.py new file mode 100644 index 0000000..1b54dfa --- /dev/null +++ b/great_ai/src/great_ai/great_ai/deploy/routes/dashboard/create_dash_app.py @@ -0,0 +1,254 @@ +from math import ceil +from typing import Any, Dict, List, Tuple + +import pandas as pd +import plotly.express as px +import plotly.graph_objects as go +from dash import Dash, dcc, html +from dash.dependencies import Input, Output +from flask import Flask + +from great_ai.utilities.unique import unique + +from ....constants import DASHBOARD_PATH +from ....context import get_context +from ....helper import snake_case_to_text, text_to_hex_color +from ....views import SortBy +from .get_description import get_description +from .get_filter_from_datatable import get_filter_from_datatable +from .get_footer import get_footer +from .get_traces_table import get_traces_table + + +def create_dash_app(function_name: str, function_docs: str) -> Flask: + accent_color = text_to_hex_color(function_name) + + flask_app = Flask(__name__) + app = Dash( + function_name, + requests_pathname_prefix=DASHBOARD_PATH + "/", + server=flask_app, + title=snake_case_to_text(function_name), + update_title=None, + external_stylesheets=[ + "/assets/index.css", + ], + ) + + app.layout = html.Main( + [ + html.Div( + html.P("PRODUCTION" if get_context().is_production else "DEVELOPMENT"), + className="environment", + ), + html.Header( + [ + get_description( + function_name=function_name, + function_docs=function_docs, + accent_color=accent_color, + ), + execution_time_histogram_container := html.Div(), + ], + ), + configuration_container := html.Div( + className="configuration-container", + ), + traces_table_container := html.Div( + [ + html.Header( + [ + html.H2("Latest traces"), + html.P( + "Recent traces and aggregated metrics are presented below. Try filtering the table." + ), + html.A( + "Filtering syntax.", + href="https://dash.plotly.com/datatable/filtering", + target="_blank", + ), + ] + ), + table := get_traces_table(), + ], + className="traces-table-container", + ), + parallel_coordinates := dcc.Graph( + className="parallel-coordinates", config={"displaylogo": False} + ), + html.Div(className="space-filler"), + get_footer(), + interval := dcc.Interval( + interval=4 * 1000, # in milliseconds + ), + ] + ) + + @app.callback( + Output(configuration_container, "children"), + Input(interval, "n_intervals"), + ) + def update_configuration( + n_intervals: int, + ) -> List[html.Div]: + config = get_context().to_flat_dict() + return [ + html.Div( + [ + html.H4(snake_case_to_text(key)), + html.P(str(value)), + ], + className="configuration-item", + ) + for key, value in config.items() + ] + + @app.callback( + Output(table, "data"), + Output(table, "page_count"), + Input(table, "page_current"), + Input(table, "page_size"), + Input(table, "sort_by"), + Input(table, "filter_query"), + Input(interval, "n_intervals"), + ) + def update_table( + page_current: int, + page_size: int, + sort_by: List[SortBy], + filter_query: str, + n_intervals: int, + ) -> Tuple[List[Dict[str, Any]], int]: + conjunctive_filters = ( + [get_filter_from_datatable(f) for f in filter_query.split(" && ")] + if filter_query + else [] + ) + non_null_conjunctive_filters = [f for f in conjunctive_filters if f is not None] + + elements, count = get_context().tracing_database.query( + skip=page_current * page_size, + take=page_size, + conjunctive_filters=non_null_conjunctive_filters, + sort_by=sort_by, + ) + + return ( + [e.to_flat_dict() for e in elements], + max(1, ceil(count / page_size)), + ) + + @app.callback( + Output(table, "columns"), + Output(traces_table_container, "style"), + Input(interval, "n_intervals"), + ) + def update_layout( + n_intervals: int, + ) -> Tuple[List[Dict[str, str]], Dict[str, Any]]: + elements, count = get_context().tracing_database.query(take=1) + + if elements: + keys = list(elements[0].to_flat_dict().keys()) + header_height = max(len(i.split(":")) for i in keys) + columns = [ + { + "name": [""] * (header_height - len(k.split(":"))) + + k.replace("_flat", "").split(":"), + "id": k, + } + for k in keys + ] + else: + columns = [] + + return ( + columns, + {"display": "block" if count > 0 else "none"}, + ) + + @app.callback( + Output(execution_time_histogram_container, "children"), + Output(parallel_coordinates, "figure"), + Output(parallel_coordinates, "style"), + Input(table, "filter_query"), + Input(interval, "n_intervals"), + ) + def update_charts( + filter_query: str, n_intervals: int + ) -> Tuple[Any, go.Figure, Dict[str, Any]]: + conjunctive_filters = ( + [get_filter_from_datatable(f) for f in filter_query.split(" && ")] + if filter_query + else [] + ) + non_null_conjunctive_filters = [f for f in conjunctive_filters if f is not None] + + elements, count = get_context().tracing_database.query( + conjunctive_filters=non_null_conjunctive_filters + ) + + elements = [e.to_flat_dict() for e in elements] + + if not elements: + return ( + html.Span( + f"No traces yet: call your function ({function_name}) to create one.", + className="placeholder", + ), + go.Figure(), + {"display": "none"}, + ) + + execution_time_histogram = dcc.Graph(config={"displaylogo": False}) + df = pd.DataFrame(elements) + fig = px.histogram( + df, + x="original_execution_time_ms", + labels={"original_execution_time_ms": "Execution time (ms)"}, + nbins=20, + height=400, + log_y=True, + color_discrete_sequence=[accent_color], + ) + fig.update_layout( + margin=dict(l=0, r=0, b=0, t=0, pad=0), + ) + execution_time_histogram.figure = fig + + parallel_coords_fig = go.Figure( + go.Parcoords( + dimensions=[ + get_dimension_descriptor(df, c) + for c in df.columns + if c + not in {"trace_id", "created", "output", "exception", "feedback"} + and "_flat" not in c + ], + line_color=accent_color, + ) + ) + return execution_time_histogram, parallel_coords_fig, {} + + return flask_app + + +def get_dimension_descriptor(df: pd.DataFrame, column: str) -> Dict[str, Any]: + dimension: Dict[str, Any] = { + "label": snake_case_to_text(column), + } + + values = df[column] + + try: + dimension["values"] = [float(v) for v in values] + except (TypeError, ValueError): + MAX_LENGTH = 40 + unique_values = unique(values) + value_mapping = {str(v)[-MAX_LENGTH:]: i for i, v in enumerate(unique_values)} + + dimension["values"] = [value_mapping[str(v)[-MAX_LENGTH:]] for v in values] + dimension["tickvals"] = list(value_mapping.values()) + dimension["ticktext"] = [k[-MAX_LENGTH:] for k in value_mapping.keys()] + + return dimension diff --git a/great_ai/src/great_ai/great_ai/dashboard/get_description.py b/great_ai/src/great_ai/great_ai/deploy/routes/dashboard/get_description.py similarity index 62% rename from great_ai/src/great_ai/great_ai/dashboard/get_description.py rename to great_ai/src/great_ai/great_ai/deploy/routes/dashboard/get_description.py index dce6a48..433442c 100644 --- a/great_ai/src/great_ai/great_ai/dashboard/get_description.py +++ b/great_ai/src/great_ai/great_ai/deploy/routes/dashboard/get_description.py @@ -1,6 +1,6 @@ from dash import dcc, html -from ..helper import snake_case_to_text, strip_lines +from ....helper import snake_case_to_text, strip_lines def get_description( @@ -9,25 +9,21 @@ def get_description( return html.Div( [ html.H1( - f"{snake_case_to_text(function_name)} - metrics", + f"{snake_case_to_text(function_name)} - dashboard", style={"color": accent_color}, ), dcc.Markdown( strip_lines( f""" - > View the live data of your deployments here. + > View the live data of your deployment here. ## Using the API You can find the available endpoints at [/docs](/docs). - ### Details + ## Details {function_docs} - - ## Metrics - - Recent traces and aggregated metrics are presented below. Try filtering the table. """ ), className="description", diff --git a/great_ai/src/great_ai/great_ai/dashboard/get_filter_from_datatable.py b/great_ai/src/great_ai/great_ai/deploy/routes/dashboard/get_filter_from_datatable.py similarity index 95% rename from great_ai/src/great_ai/great_ai/dashboard/get_filter_from_datatable.py rename to great_ai/src/great_ai/great_ai/deploy/routes/dashboard/get_filter_from_datatable.py index 8da7c35..8bb08e5 100644 --- a/great_ai/src/great_ai/great_ai/dashboard/get_filter_from_datatable.py +++ b/great_ai/src/great_ai/great_ai/deploy/routes/dashboard/get_filter_from_datatable.py @@ -1,6 +1,6 @@ from typing import Optional, Union -from ..views import Filter, operators +from ....views import Filter, operators def get_filter_from_datatable(description: str) -> Optional[Filter]: diff --git a/great_ai/src/great_ai/great_ai/dashboard/get_footer.py b/great_ai/src/great_ai/great_ai/deploy/routes/dashboard/get_footer.py similarity index 83% rename from great_ai/src/great_ai/great_ai/dashboard/get_footer.py rename to great_ai/src/great_ai/great_ai/deploy/routes/dashboard/get_footer.py index 44a0925..e6c2fe4 100644 --- a/great_ai/src/great_ai/great_ai/dashboard/get_footer.py +++ b/great_ai/src/great_ai/great_ai/deploy/routes/dashboard/get_footer.py @@ -1,5 +1,7 @@ from dash import html +from ....constants import GITHUB_LINK + def get_footer() -> html.Footer: return html.Footer( @@ -14,9 +16,8 @@ def get_footer() -> html.Footer: ), html.A( html.Img(src="/assets/github.png"), - href="https://github.com/ScoutinScience/great-ai", + href=GITHUB_LINK, target="_blank", ), ], - className="watermark", ) diff --git a/great_ai/src/great_ai/great_ai/deploy/routes/dashboard/get_traces_table.py b/great_ai/src/great_ai/great_ai/deploy/routes/dashboard/get_traces_table.py new file mode 100644 index 0000000..629de70 --- /dev/null +++ b/great_ai/src/great_ai/great_ai/deploy/routes/dashboard/get_traces_table.py @@ -0,0 +1,32 @@ +from dash import dash_table + + +def get_traces_table() -> dash_table.DataTable: + return dash_table.DataTable( + page_current=0, + page_size=20, + page_action="custom", + filter_action="custom", + sort_action="custom", + sort_mode="multi", + sort_by=[ + {"column_id": "created", "direction": "desc"}, + ], + style_data={ + "white-space": "normal", + "height": "auto", + "max-height": "300px", + "overflow": "hidden", + "text-overflow": "ellipsis", + }, + style_cell={"padding": "5px"}, + style_header={ + "background-color": "white", + "font-weight": "bold", + }, + style_table={"max-height": "70vh", "overflow": "auto"}, + merge_duplicate_headers=True, + style_cell_conditional=[ + {"if": {"column_id": "output"}, "width": 1500}, + ], + ) diff --git a/great_ai/src/great_ai/great_ai/helper/freeze_arguments.py b/great_ai/src/great_ai/great_ai/helper/freeze_arguments.py index cb78bea..3847ea5 100644 --- a/great_ai/src/great_ai/great_ai/helper/freeze_arguments.py +++ b/great_ai/src/great_ai/great_ai/helper/freeze_arguments.py @@ -22,5 +22,4 @@ def freeze_arguments(func: Callable[..., Any]) -> Callable[..., Any]: } return func(*args, **kwargs) - wrapper.cache_info = func.cache_info # type: ignore return wrapper diff --git a/great_ai/src/great_ai/great_ai/parameters/parameter.py b/great_ai/src/great_ai/great_ai/parameters/parameter.py index 2ffaa40..a29cc02 100644 --- a/great_ai/src/great_ai/great_ai/parameters/parameter.py +++ b/great_ai/src/great_ai/great_ai/parameters/parameter.py @@ -20,7 +20,7 @@ def parameter( get_function_metadata_store(func).input_parameter_names.append(parameter_name) assert_function_is_not_finalised(func) - actual_name = f"arg:{func.__name__}:{parameter_name}" + actual_name = f"arg:{parameter_name}" @wraps(func) def wrapper(*args: Any, **kwargs: Dict[str, Any]) -> Any: @@ -41,7 +41,7 @@ def parameter( context = TracingContext.get_current_context() if context and not disable_logging: - context.log_value(name=actual_name, value=argument) + context.log_value(name=f"{actual_name}:value", value=argument) if isinstance(argument, str): context.log_value(name=f"{actual_name}:length", value=len(argument)) diff --git a/great_ai/src/great_ai/great_ai/tracing/parallel_tinydb_driver.py b/great_ai/src/great_ai/great_ai/tracing/parallel_tinydb_driver.py index 159039b..84adcb4 100644 --- a/great_ai/src/great_ai/great_ai/tracing/parallel_tinydb_driver.py +++ b/great_ai/src/great_ai/great_ai/tracing/parallel_tinydb_driver.py @@ -1,6 +1,6 @@ from multiprocessing import Lock from pathlib import Path -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Callable, Optional, Sequence, Tuple import pandas as pd from tinydb import TinyDB @@ -34,39 +34,40 @@ class ParallelTinyDbDriver(TracingDatabase): self, skip: int = 0, take: Optional[int] = None, - conjunctive_filters: List[Filter] = [], - sort_by: List[SortBy] = [], - ) -> List[Dict[str, Any]]: + conjunctive_filters: Sequence[Filter] = [], + sort_by: Sequence[SortBy] = [], + ) -> Tuple[Sequence[Trace], int]: documents = [ - d.to_flat_dict() - for d in self._safe_execute( - lambda db: [Trace.parse_obj(t) for t in db.all()] - ) + Trace.parse_obj(t) for t in self._safe_execute(lambda db: db.all()) ] if not documents: - return [] + return [], 0 - df = pd.DataFrame(documents) + df = pd.DataFrame([d.to_flat_dict() for d in documents]) for f in conjunctive_filters: - if f.operator in operator_mapping: + operator = f.operator.lower() + if operator in operator_mapping: df = df.loc[ getattr(df[f.property], operator_mapping[f.operator])(f.value) ] - elif f.operator == "contains": - df = df.loc[df[f.property].str.contains(f.value)] + elif operator == "contains": + df = df.loc[df[f.property].str.contains(f.value, case=False)] if sort_by: - df = df.sort_values( + df.sort_values( [col["column_id"] for col in sort_by], ascending=[col["direction"] == "asc" for col in sort_by], - inplace=False, + inplace=True, ) + count = len(df) result = df.iloc[skip:] if take is None else df.iloc[skip : skip + take] - - return result.to_dict("records") + return [ + next(d for d in documents if d.trace_id == trace_id) + for trace_id in result["trace_id"] + ], count def update(self, id: str, new_version: Trace) -> None: self._safe_execute( diff --git a/great_ai/src/great_ai/great_ai/tracing/tracing_context.py b/great_ai/src/great_ai/great_ai/tracing/tracing_context.py index 7872c61..381ebab 100644 --- a/great_ai/src/great_ai/great_ai/tracing/tracing_context.py +++ b/great_ai/src/great_ai/great_ai/tracing/tracing_context.py @@ -29,7 +29,7 @@ class TracingContext: delta_time = (datetime.utcnow() - self._start_time).microseconds / 1000 self._trace = Trace( created=self._start_time.isoformat(), - execution_time_ms=delta_time, + original_execution_time_ms=delta_time, logged_values=self._values, models=self._models, output=output, diff --git a/great_ai/src/great_ai/great_ai/tracing/tracing_database.py b/great_ai/src/great_ai/great_ai/tracing/tracing_database.py index c48de0d..c61a6e7 100644 --- a/great_ai/src/great_ai/great_ai/tracing/tracing_database.py +++ b/great_ai/src/great_ai/great_ai/tracing/tracing_database.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import List, Optional +from typing import Optional, Sequence, Tuple from ..views import Filter, SortBy, Trace @@ -20,9 +20,9 @@ class TracingDatabase(ABC): self, skip: int = 0, take: Optional[int] = None, - conjunctive_filters: List[Filter] = [], - sort_by: List[SortBy] = [], - ) -> List[Trace]: + conjunctive_filters: Sequence[Filter] = [], + sort_by: Sequence[SortBy] = [], + ) -> Tuple[Sequence[Trace], int]: pass @abstractmethod diff --git a/great_ai/src/great_ai/great_ai/views/__init__.py b/great_ai/src/great_ai/great_ai/views/__init__.py index 65ef6f8..f2087fd 100644 --- a/great_ai/src/great_ai/great_ai/views/__init__.py +++ b/great_ai/src/great_ai/great_ai/views/__init__.py @@ -9,3 +9,4 @@ from .operators import operators from .query import Query from .sort_by import SortBy from .trace import Trace +from .trace_view import TraceView diff --git a/great_ai/src/great_ai/great_ai/views/operators.py b/great_ai/src/great_ai/great_ai/views/operators.py index 7f05343..071e266 100644 --- a/great_ai/src/great_ai/great_ai/views/operators.py +++ b/great_ai/src/great_ai/great_ai/views/operators.py @@ -2,12 +2,4 @@ from typing import List, Literal Operator = Literal[">=", "<=", "<", ">", "!=", "=", "contains"] -operators: List[Operator] = [ - ">=", - "<=", - "<", - ">", - "!=", - "=", - "contains", -] +operators: List[Operator] = [">=", "<=", "<", ">", "!=", "=", "contains"] diff --git a/great_ai/src/great_ai/great_ai/views/query.py b/great_ai/src/great_ai/great_ai/views/query.py index 13e233c..e46b567 100644 --- a/great_ai/src/great_ai/great_ai/views/query.py +++ b/great_ai/src/great_ai/great_ai/views/query.py @@ -14,10 +14,14 @@ class Query(BaseModel): schema_extra = { "example": { "filter": [ - {"property": "execution_time_ms", "operator": ">", "value": 100} + { + "property": "original_execution_time_ms", + "operator": ">", + "value": 100, + } ], "sort": [ - {"column_id": "execution_time_ms", "direction": "asc"}, + {"column_id": "original_execution_time_ms", "direction": "asc"}, {"column_id": "id", "direction": "desc"}, ], } diff --git a/great_ai/src/great_ai/great_ai/views/trace.py b/great_ai/src/great_ai/great_ai/views/trace.py index f6e0bd3..8ea922c 100644 --- a/great_ai/src/great_ai/great_ai/views/trace.py +++ b/great_ai/src/great_ai/great_ai/views/trace.py @@ -1,37 +1,33 @@ -from json import dumps -from typing import Any, Dict, List, Optional -from uuid import uuid4 +from typing import Any, Dict, Optional -from pydantic import BaseModel, validator +import yaml +from pydantic import validator -from .model import Model +from .trace_view import TraceView -class Trace(BaseModel): - trace_id: Optional[str] - created: str - execution_time_ms: float - logged_values: Dict[str, Any] - models: List[Model] - exception: Optional[str] - output: Any - feedback: Any = None +class Trace(TraceView): + models_flat: Optional[str] + output_flat: Optional[str] + feedback_flat: Optional[str] - @validator("trace_id", always=True) - def generate_id(cls, v: Optional[str], values: Dict[str, Any]) -> Optional[str]: - if not v: - return str(uuid4()) - return v + @validator("models_flat", always=True) + def flatten_models(cls, v: Optional[str], values: Dict[str, Any]) -> str: + return ", ".join(f"{m.key}:{m.version}" for m in values["models"]) + + @validator("output_flat", always=True) + def flatten_output(cls, v: Optional[str], values: Dict[str, Any]) -> str: + return yaml.dump(values["output"], default_flow_style=False, indent=2) def to_flat_dict(self) -> Dict[str, Any]: return { - "id": self.trace_id, + "trace_id": self.trace_id, "created": self.created, - "execution_time_ms": self.execution_time_ms, - "models": ", ".join(f"{m.key}:{m.version}" for m in self.models), - "output": dumps(self.output), + "original_execution_time_ms": self.original_execution_time_ms, + "models_flat": self.models_flat, + "output_flat": self.output_flat, "exception": self.exception or "null", - "feedback": self.feedback, + "feedback_flat": self.feedback_flat or "null", **self.logged_values, } diff --git a/great_ai/src/great_ai/great_ai/views/trace_view.py b/great_ai/src/great_ai/great_ai/views/trace_view.py new file mode 100644 index 0000000..90fa1ad --- /dev/null +++ b/great_ai/src/great_ai/great_ai/views/trace_view.py @@ -0,0 +1,26 @@ +from typing import Any, Dict, List, Optional +from uuid import uuid4 + +from pydantic import BaseModel, validator + +from .model import Model + + +class TraceView(BaseModel): + trace_id: Optional[str] + created: str + original_execution_time_ms: float + logged_values: Dict[str, Any] + models: List[Model] + exception: Optional[str] + output: Any + feedback: Any = None + + @validator("trace_id", always=True) + def generate_id(cls, v: Optional[str], values: Dict[str, Any]) -> Optional[str]: + if not v: + return str(uuid4()) + return v + + def __hash__(self) -> int: + return hash((type(self),) + tuple(self.__dict__.values()))