Improve dashboard

.vscode/settings.json

@@ -2,6 +2,8 @@
   "cSpell.words": [
     "boto",
     "botocore",
+    "datatable",
+    "displaylogo",
     "fastapi",
     "gridfs",
     "iloc",
@@ -10,6 +12,8 @@
     "levelno",
     "matplotlib",
     "nbconvert",
+    "nbins",
+    "Parcoords",
     "plotly",
     "proba",
     "psutil",
@@ -21,6 +25,8 @@
     "starlette",
     "sublinear",
     "Tfidf",
+    "ticktext",
+    "tickvals",
     "tinydb",
     "uvicorn",
     "Vectorizer",

examples/simple/deploy.ipynb

@@ -133,14 +133,14 @@
     }
    ],
    "source": [
-    "result = predict_domain(\n",
+    "# result = predict_domain(\n",
-    "    \"\"\"\n",
+    "#     \"\"\"\n",
-    "    State-of-the-art methods for zero-shot visual recognition formulate learning as a joint embedding problem of images and side information. In these formulations the current best complement to visual features are attributes: manually encoded vectors describing shared characteristics among categories. Despite good performance, attributes have limitations: (1) finer-grained recognition requires commensurately more, and (2) attributes do not provide a natural language interface. We propose to overcome these limitations by training neural language models from scratch; i.e. without pre-training and only consuming words and characters. Our proposed models train end-to-end to align with the fine-grained and category-specific content of images. Natural language provides a flexible and compact way of encoding only the salient visual aspects for distinguishing categories. By training on raw text, our model can do inference on raw text as well, providing humans a familiar mode both for annotation and retrieval. Our model achieves strong performance on zero-shot text-based image retrieval and significantly outperforms the attribute-based state-of-the-art for zero-shot classification on the CaltechUCSD Birds 200-2011 dataset. \"\"\"\n",
+    "#     State-of-the-art methods for zero-shot visual recognition formulate learning as a joint embedding problem of images and side information. In these formulations the current best complement to visual features are attributes: manually encoded vectors describing shared characteristics among categories. Despite good performance, attributes have limitations: (1) finer-grained recognition requires commensurately more, and (2) attributes do not provide a natural language interface. We propose to overcome these limitations by training neural language models from scratch; i.e. without pre-training and only consuming words and characters. Our proposed models train end-to-end to align with the fine-grained and category-specific content of images. Natural language provides a flexible and compact way of encoding only the salient visual aspects for distinguishing categories. By training on raw text, our model can do inference on raw text as well, providing humans a familiar mode both for annotation and retrieval. Our model achieves strong performance on zero-shot text-based image retrieval and significantly outperforms the attribute-based state-of-the-art for zero-shot classification on the CaltechUCSD Birds 200-2011 dataset. \"\"\"\n",
-    ")\n",
+    "# )\n",
     "\n",
-    "from pprint import pprint\n",
+    "# from pprint import pprint\n",
     "\n",
-    "# pprint(result.dict(), width=120)"
+    "# # pprint(result.dict(), width=120)"
    ]
   }
  ],

great_ai/src/great_ai/great_ai/constants.py

@@ -5,10 +5,12 @@ from great_ai.large_file import LargeFileLocal, LargeFileMongo, LargeFileS3
 ENV_VAR_KEY = "ENVIRONMENT"
 PRODUCTION_KEY = "production"
 DEFAULT_TRACING_DB_FILENAME = "tracing_database.json"
-METRICS_PATH = "/metrics"
+DASHBOARD_PATH = "/dashboard"
 
 DEFAULT_LARGE_FILE_CONFIG_PATHS = {
     LargeFileLocal: None,
     LargeFileMongo: Path("mongodb.ini"),
     LargeFileS3: Path("s3.ini"),
 }
+
+GITHUB_LINK = "https://github.com/ScoutinScience/great-ai"

great_ai/src/great_ai/great_ai/context/configure.py

@@ -18,7 +18,6 @@ from ..tracing.parallel_tinydb_driver import ParallelTinyDbDriver, TracingDataba
 
 
 def configure(
-    version: str = "0.0.1",
     log_level: int = DEBUG,
     seed: int = 42,
     tracing_database: TracingDatabase = ParallelTinyDbDriver(
@@ -46,7 +45,6 @@ def configure(
         )
 
     context._context = context.Context(
-        version=version,
         tracing_database=tracing_database,
         large_file_implementation=large_file_implementation,
         is_production=is_production,

great_ai/src/great_ai/great_ai/context/context.py

@@ -9,7 +9,6 @@ from ..tracing.tracing_database import TracingDatabase
 
 
 class Context(BaseModel):
-    version: str
     tracing_database: TracingDatabase
     large_file_implementation: Type[LargeFile]
     is_production: bool
@@ -22,12 +21,11 @@ class Context(BaseModel):
 
     def to_flat_dict(self) -> Dict[str, Any]:
         return {
-            "version": self.version,
             "tracing_database": type(self.tracing_database).__name__,
             "large_file_implementation": self.large_file_implementation.__name__,
             "is_production": self.is_production,
-            "logger": type(self.logger).__name__,
             "should_log_exception_stack": self.should_log_exception_stack,
+            "prediction_cache_size": self.prediction_cache_size,
         }
 
 

great_ai/src/great_ai/great_ai/dashboard/assets/index.css

@@ -1,112 +0,0 @@
-:root {
-  --accent-color: #47c2d0;
-  --error-color: #a30808;
-  --background-color: #edf5f6;
-  --small-padding: 10px;
-  --medium-padding: 20px;
-  --large-padding: 40px;
-  --border-radius: 10px;
-  --shadow: 0 4px 6px -1px rgb(0 0 0 / 10%), 0 2px 4px -1px rgb(0 0 0 / 6%);
-}
-
-* {
-  margin: 0;
-  box-sizing: border-box;
-}
-
-body {
-  background-color: var(--background-color);
-  font-family: Arial, Helvetica, sans-serif;
-}
-
-h1,
-h2,
-h3,
-h4,
-h5,
-h6 {
-  margin: var(--medium-padding) 0 var(--small-padding) 0;
-}
-
-.glance,
-.table-container,
-.parallel-coords {
-  padding: var(--large-padding);
-  margin: var(--large-padding);
-  border-radius: var(--border-radius);
-  box-shadow: var(--shadow);
-  background-color: white;
-  overflow: hidden;
-}
-
-.glance {
-  display: flex;
-}
-
-.glance .description {
-  width: 350px;
-  display: flex;
-  flex-direction: column;
-  justify-content: center;
-}
-
-.glance .dash-graph {
-  flex: 1;
-}
-
-.table-container {
-  padding: 0;
-}
-
-.table-container h2 {
-  padding: var(--small-padding);
-  margin: 0;
-}
-
-.table-container h2,
-footer.watermark {
-  opacity: 0.35;
-}
-
-.dash-spreadsheet {
-  overflow: auto;
-}
-
-th,
-td {
-  max-width: 350px;
-}
-
-.parallel-coords {
-  padding: 0;
-}
-
-footer.watermark {
-  display: flex;
-  justify-content: space-between;
-  align-items: center;
-  padding: var(--large-padding);
-  background-color: #ddd;
-  position: relative;
-}
-
-footer.watermark div {
-  display: flex;
-  flex-direction: column;
-}
-
-h6 {
-  font-size: 3rem;
-  margin-top: 0;
-}
-
-a img {
-  width: 64px;
-  height: 64px;
-  cursor: pointer;
-  transition: transform 300ms;
-}
-
-a img:hover {
-  transform: scale(1.1);
-}

great_ai/src/great_ai/great_ai/dashboard/create_dash_app.py

@@ -1,176 +0,0 @@
-from typing import Any, Dict, List
-
-import pandas as pd
-import plotly.express as px
-import plotly.graph_objects as go
-from dash import Dash, dash_table, dcc, html
-from dash.dependencies import Input, Output
-from flask import Flask
-
-from great_ai.utilities.unique import unique
-
-from ..constants import METRICS_PATH
-from ..context import get_context
-from ..helper import snake_case_to_text, text_to_hex_color
-from ..views import SortBy
-from .get_description import get_description
-from .get_filter_from_datatable import get_filter_from_datatable
-from .get_footer import get_footer
-
-
-def create_dash_app(function_name: str, function_docs: str) -> Flask:
-    accent_color = text_to_hex_color(function_name)
-
-    flask_app = Flask(__name__)
-    app = Dash(
-        function_name,
-        requests_pathname_prefix=METRICS_PATH + "/",
-        server=flask_app,
-        title=snake_case_to_text(function_name),
-        update_title=None,
-        external_stylesheets=[
-            "/assets/index.css",
-        ],
-    )
-
-    documents = get_context().tracing_database.query()
-    df = pd.DataFrame(documents)
-
-    execution_time_histogram = dcc.Graph(config={"displaylogo": False})
-    table = dash_table.DataTable(
-        columns=[{"name": i, "id": i} for i in df.columns],
-        page_current=0,
-        page_size=20,
-        page_action="custom",
-        filter_action="custom",
-        filter_query="",
-        sort_action="custom",
-        sort_mode="multi",
-        sort_by=[
-            {"column_id": "created", "direction": "desc"},
-        ],
-    )
-
-    app.layout = html.Div(
-        [
-            html.Div(
-                [
-                    get_description(
-                        function_name=function_name,
-                        function_docs=function_docs,
-                        accent_color=accent_color,
-                    ),
-                    execution_time_histogram,
-                ],
-                className="glance",
-            ),
-            html.Div([html.H2("Latest traces"), table], className="table-container"),
-            parallel_coords := dcc.Graph(
-                className="parallel-coords", config={"displaylogo": False}
-            ),
-            get_footer(),
-            interval := dcc.Interval(
-                interval=4 * 1000,  # in milliseconds
-            ),
-        ]
-    )
-
-    @app.callback(
-        Output(table, "data"),
-        Input(table, "page_current"),
-        Input(table, "page_size"),
-        Input(table, "sort_by"),
-        Input(table, "filter_query"),
-        Input(interval, "n_intervals"),
-    )
-    def update_table(
-        page_current: int,
-        page_size: int,
-        sort_by: List[SortBy],
-        filter: str,
-        n_intervals: int,
-    ) -> List[Dict[str, Any]]:
-        conjunctive_filters = [
-            get_filter_from_datatable(f) for f in filter.split(" && ")
-        ]
-        non_null_conjunctive_filters = [f for f in conjunctive_filters if f is not None]
-
-        return get_context().tracing_database.query(
-            skip=page_current * page_size,
-            take=page_size,
-            conjunctive_filters=non_null_conjunctive_filters,
-            sort_by=sort_by,
-        )
-
-    @app.callback(
-        Output(execution_time_histogram, "figure"),
-        Output(parallel_coords, "figure"),
-        Input(table, "filter_query"),
-        Input(interval, "n_intervals"),
-    )
-    def update_charts(filter: str, n_intervals: int) -> go.Figure:
-        conjunctive_filters = [
-            get_filter_from_datatable(f) for f in filter.split(" && ")
-        ]
-        non_null_conjunctive_filters = [f for f in conjunctive_filters if f is not None]
-
-        rows = get_context().tracing_database.query(
-            conjunctive_filters=non_null_conjunctive_filters
-        )
-
-        if not rows:
-            return go.Figure(), go.Figure()
-
-        df = pd.DataFrame(rows)
-
-        fig = px.histogram(
-            df,
-            x="execution_time_ms",
-            labels={"execution_time_ms": "Execution time (ms)"},
-            nbins=20,
-            height=400,
-            log_y=True,
-            color_discrete_sequence=[accent_color],
-        )
-
-        fig.update_layout(
-            autosize=True,
-            margin=dict(l=0, r=0, b=0, t=0, pad=0),
-        )
-
-        return (
-            fig,
-            go.Figure(
-                go.Parcoords(
-                    dimensions=[
-                        get_dimension_descriptor(df, c)
-                        for c in df.columns
-                        if c not in {"id", "created", "output"}
-                    ],
-                    line_color=accent_color,
-                )
-            ),
-        )
-
-    return flask_app
-
-
-def get_dimension_descriptor(df: pd.DataFrame, column: str) -> Dict[str, Any]:
-    dimension: Dict[str, Any] = {
-        "label": snake_case_to_text(column),
-    }
-
-    values = df[column]
-
-    try:
-        dimension["values"] = [float(v) for v in values]
-    except (TypeError, ValueError):
-        MAX_LENGTH = 40
-        unique_values = unique(values)
-        value_mapping = {str(v)[-MAX_LENGTH:]: i for i, v in enumerate(unique_values)}
-
-        dimension["values"] = [value_mapping[str(v)[-MAX_LENGTH:]] for v in values]
-        dimension["tickvals"] = list(value_mapping.values())
-        dimension["ticktext"] = [k[-MAX_LENGTH:] for k in value_mapping.keys()]
-
-    return dimension

great_ai/src/great_ai/great_ai/deploy/great_ai.py

@@ -5,12 +5,12 @@ from typing import Any, Callable, Iterable, Optional, Sequence, Type, Union, cas
 from fastapi import APIRouter, FastAPI, status
 from pydantic import BaseModel, create_model
 
+from great_ai.great_ai.deploy.routes.bootstrap_dashboard import bootstrap_dashboard
 from great_ai.great_ai.views.cache_statistics import CacheStatistics
 from great_ai.utilities.parallel_map import parallel_map
 
-from ..constants import METRICS_PATH
+from ..constants import DASHBOARD_PATH
 from ..context import get_context
-from ..dashboard import create_dash_app
 from ..helper import (
     freeze_arguments,
     get_function_metadata_store,
@@ -28,25 +28,31 @@ from .routes import (
 
 
 class GreatAI:
-    def __init__(self, func: Callable[..., Any]):
+    def __init__(self, func: Callable[..., Any], version: str):
         self._func = automatically_decorate_parameters(func)
         get_function_metadata_store(self._func).is_finalised = True
+
+        self._cached_func = lru_cache(get_context().prediction_cache_size)(
+            self._func
+        )  # cannot put decorator on method, because it require the context to be setup
+
         wraps(func)(self)
 
+        self._version = version
+
         self.app = FastAPI(
             title=self.name,
             version=self.version,
             description=self.documentation
-            + f"\n\n Find out more on the [metrics page]({METRICS_PATH}).",
+            + f"\n\nFind out more in the [dashboard]({DASHBOARD_PATH}).",
             docs_url=None,
             redoc_url=None,
         )
 
     @freeze_arguments
-    @lru_cache(get_context().prediction_cache_size)
     def __call__(self, *args: Any, **kwargs: Any) -> Trace:
         with TracingContext() as t:
-            result = self._func(*args, **kwargs)
+            result = self._cached_func(*args, **kwargs)
             output = t.finalise(output=result)
         return output
 
@@ -54,9 +60,10 @@ class GreatAI:
     def deploy(
         func: Optional[Callable[..., Any]] = None,
         *,
+        version: str = "0.0.1",
         disable_rest_api: bool = False,
         disable_docs: bool = False,
-        disable_metrics: bool = False,
+        disable_dashboard: bool = False,
     ) -> Union[Callable[[Callable[..., Any]], "GreatAI"], "GreatAI"]:
         if func is None:
             return cast(
@@ -65,15 +72,15 @@ class GreatAI:
                     GreatAI.deploy,
                     disable_http=disable_rest_api,
                     disable_docs=disable_docs,
-                    disable_metrics=disable_metrics,
+                    disable_dashboard=disable_dashboard,
                 ),
             )
 
-        instance = GreatAI(func)
+        instance = GreatAI(func, version=version)
 
         if not disable_rest_api:
             instance._bootstrap_rest_api(
-                disable_docs=disable_docs, disable_metrics=disable_metrics
+                disable_docs=disable_docs, disable_dashboard=disable_dashboard
             )
 
         return instance
@@ -95,7 +102,9 @@ class GreatAI:
 
     @property
     def version(self) -> str:
-        return f"{get_context().version}+{get_function_metadata_store(self._func).model_versions}"
+        return (
+            f"{self._version}+{get_function_metadata_store(self._func).model_versions}"
+        )
 
     @property
     def documentation(self) -> str:
@@ -110,23 +119,26 @@ class GreatAI:
             )
         )
 
-    def _bootstrap_rest_api(self, disable_docs: bool, disable_metrics: bool) -> None:
+    def _bootstrap_rest_api(self, disable_docs: bool, disable_dashboard: bool) -> None:
-        self._bootstrap_prediction_endpoints()
+        self._bootstrap_prediction_endpoint()
 
         if not disable_docs:
             bootstrap_docs_endpoints(self.app)
 
-        if not disable_metrics:
+        if not disable_dashboard:
-            dash_app = create_dash_app(self._func.__name__, self.documentation)
+            bootstrap_dashboard(
-            bootstrap_trace_endpoints(self.app, dash_app)
+                self.app,
+                function_name=self._func.__name__,
+                documentation=self.documentation,
+            )
+            bootstrap_trace_endpoints(self.app)
 
         bootstrap_feedback_endpoints(self.app)
-
         self._bootstrap_meta_endpoints()
 
-    def _bootstrap_prediction_endpoints(self) -> None:
+    def _bootstrap_prediction_endpoint(self) -> None:
         router = APIRouter(
-            prefix="/predictions",
+            prefix="/predict",
             tags=["predictions"],
         )
 
@@ -160,7 +172,7 @@ class GreatAI:
 
         @router.get("/health", status_code=status.HTTP_200_OK)
         def check_health() -> HealthCheckResponse:
-            hits, misses, maxsize, cache_size = self.__call__.cache_info()  # type: ignore
+            hits, misses, maxsize, cache_size = self._cached_func.cache_info()
             cache_statistics = CacheStatistics(
                 hits=hits, misses=misses, size=cache_size, max_size=maxsize
             )

great_ai/src/great_ai/great_ai/deploy/routes/__init__.py

@@ -1,3 +1,4 @@
+from .bootstrap_dashboard import bootstrap_dashboard
 from .bootstrap_docs_endpoints import bootstrap_docs_endpoints
 from .bootstrap_feedback_endpoints import bootstrap_feedback_endpoints
 from .bootstrap_trace_endpoints import bootstrap_trace_endpoints

great_ai/src/great_ai/great_ai/deploy/routes/bootstrap_dashboard.py

@@ -0,0 +1,27 @@
+from pathlib import Path
+
+from fastapi import FastAPI
+from fastapi.middleware.wsgi import WSGIMiddleware
+from fastapi.responses import RedirectResponse
+from fastapi.staticfiles import StaticFiles
+
+from ...constants import DASHBOARD_PATH
+from .dashboard import create_dash_app
+
+PATH = Path(__file__).parent.resolve()
+
+
+def bootstrap_dashboard(app: FastAPI, function_name: str, documentation: str) -> None:
+    dash_app = create_dash_app(function_name, documentation)
+
+    app.mount(DASHBOARD_PATH, WSGIMiddleware(dash_app))
+
+    @app.get("/", include_in_schema=False)
+    def redirect_to_entrypoint() -> RedirectResponse:
+        return RedirectResponse(DASHBOARD_PATH)
+
+    app.mount(
+        "/assets",
+        StaticFiles(directory=PATH / "dashboard/assets"),
+        name="static",
+    )

great_ai/src/great_ai/great_ai/deploy/routes/bootstrap_feedback_endpoints.py

@@ -1,5 +1,6 @@
 from typing import Any
 
+import yaml
 from fastapi import APIRouter, FastAPI, HTTPException, Response, status
 
 from ...context import get_context
@@ -17,7 +18,12 @@ def bootstrap_feedback_endpoints(app: FastAPI) -> None:
         trace = get_context().tracing_database.get(trace_id)
         if trace is None:
             raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
+
         trace.feedback = input.feedback
+        trace.feedback_flat = yaml.dump(
+            input.feedback, default_flow_style=False, indent=2
+        )
+
         get_context().tracing_database.update(trace_id, trace)
         return Response(status_code=status.HTTP_202_ACCEPTED)
 
@@ -33,7 +39,10 @@ def bootstrap_feedback_endpoints(app: FastAPI) -> None:
         trace = get_context().tracing_database.get(trace_id)
         if trace is None:
             raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
+
         trace.feedback = None
+        trace.feedback_flat = None
+
         get_context().tracing_database.update(trace_id, trace)
         return Response(status_code=status.HTTP_204_NO_CONTENT)
 

great_ai/src/great_ai/great_ai/deploy/routes/bootstrap_trace_endpoints.py

@@ -1,38 +1,18 @@
-from pathlib import Path
 from typing import List
 
 from fastapi import APIRouter, FastAPI, HTTPException, Response, status
-from fastapi.middleware.wsgi import WSGIMiddleware
-from fastapi.responses import RedirectResponse
-from fastapi.staticfiles import StaticFiles
-from flask import Flask
 
-from ...constants import METRICS_PATH
 from ...context import get_context
-from ...views import Query, Trace
+from ...views import Query, Trace, TraceView
-
-PATH = Path(__file__).parent.resolve()
 
 
-def bootstrap_trace_endpoints(app: FastAPI, dash_app: Flask) -> None:
+def bootstrap_trace_endpoints(app: FastAPI) -> None:
-    app.mount(METRICS_PATH, WSGIMiddleware(dash_app))
-
-    @app.get("/", include_in_schema=False)
-    def redirect_to_entrypoint() -> RedirectResponse:
-        return RedirectResponse("/metrics")
-
-    app.mount(
-        "/assets",
-        StaticFiles(directory=PATH / "../../dashboard/assets"),
-        name="static",
-    )
-
     router = APIRouter(
         prefix="/traces",
         tags=["traces"],
     )
 
-    @router.post("/", status_code=status.HTTP_200_OK, response_model=List[Trace])
+    @router.post("/", status_code=status.HTTP_200_OK, response_model=List[TraceView])
     def query_traces(
         query: Query,
         skip: int = 0,
@@ -43,9 +23,9 @@ def bootstrap_trace_endpoints(app: FastAPI, dash_app: Flask) -> None:
             sort_by=query.sort,
             skip=skip,
             take=take,
-        )
+        )[0]
 
-    @router.get("/{trace_id}", status_code=status.HTTP_200_OK, response_model=Trace)
+    @router.get("/{trace_id}", status_code=status.HTTP_200_OK, response_model=TraceView)
     def get_trace(trace_id: str) -> Trace:
         result = get_context().tracing_database.get(trace_id)
         if result is None:

great_ai/src/great_ai/great_ai/deploy/routes/dashboard/assets/index.css

@@ -0,0 +1,227 @@
+:root {
+  --important-color: #a30808;
+  --background-color: #edf5f6;
+  --small-padding: 10px;
+  --medium-padding: 20px;
+  --large-padding: 40px;
+  --border-radius: 10px;
+  --shadow: 0 4px 6px -1px rgb(0 0 0 / 10%), 0 2px 4px -1px rgb(0 0 0 / 6%);
+  --disclaimer-width: 180px;
+  --disclaimer-height: 35px;
+}
+
+@media (max-width: 900px) {
+  body {
+    zoom: 0.8;
+  }
+}
+
+@media (max-width: 550px) {
+  :root {
+    --small-padding: 5px;
+    --medium-padding: 10px;
+    --large-padding: 20px;
+    --border-radius: 8px;
+  }
+
+  .environment {
+    margin-top: calc(-1 * var(--large-padding));
+    margin-bottom: var(--large-padding);
+  }
+}
+
+@media (min-width: 551px) {
+  .environment {
+    position: absolute;
+    width: var(--disclaimer-width);
+    height: var(--disclaimer-height);
+    transform: rotate(-45deg);
+    top: calc(
+      var(--disclaimer-width) / 1.4142 - var(--disclaimer-height) / 1.4142
+    );
+    left: calc(-1 * var(--disclaimer-height) / 1.4142);
+    transform-origin: top left;
+    z-index: 100;
+  }
+}
+
+* {
+  margin: 0;
+  box-sizing: border-box;
+  word-break: break-word;
+}
+
+body {
+  background-color: var(--background-color);
+  font-family: Arial, Helvetica, sans-serif;
+}
+
+h1,
+h2,
+h3,
+h4,
+h5,
+h6 {
+  margin: var(--medium-padding) 0 var(--small-padding) 0;
+}
+
+h6 {
+  margin-top: 0;
+  font-size: 3rem;
+}
+
+html,
+body,
+#react-entry-point,
+main {
+  height: 100%;
+}
+
+main {
+  padding-top: var(--large-padding);
+  display: flex;
+  flex-direction: column;
+}
+
+.environment {
+  background-color: var(--important-color);
+  color: white;
+  text-align: center;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+}
+
+main > header,
+.configuration-container,
+.traces-table-container,
+.parallel-coordinates,
+main > footer {
+  padding: var(--large-padding);
+  flex-shrink: 0;
+  overflow: hidden;
+}
+
+main > header,
+.configuration-container,
+.traces-table-container,
+.parallel-coordinates {
+  margin: 0 var(--large-padding) var(--large-padding) var(--large-padding);
+  border-radius: var(--border-radius);
+  box-shadow: var(--shadow);
+  background-color: white;
+}
+
+main > header {
+  display: flex;
+  align-items: center;
+  flex-wrap: wrap;
+  justify-content: space-between;
+}
+
+main > header > div {
+  min-width: 350px;
+  max-width: 450px;
+  margin-bottom: var(--large-padding);
+  flex: 1;
+}
+
+main > header > div > h1 {
+  margin-top: 0;
+}
+
+main > header > *:nth-child(2) {
+  min-width: 250px;
+  max-width: 550px;
+  flex: 1;
+}
+
+main > header .placeholder {
+  opacity: 0.35;
+  font-size: 1.5rem;
+  text-align: center;
+  display: block;
+  min-width: 250px;
+  width: 60%;
+  margin: auto;
+}
+
+.configuration-container {
+  display: flex;
+  justify-content: space-between;
+  flex-wrap: wrap;
+}
+
+.configuration-item {
+  border-left: 2px solid var(--important-color);
+  padding-left: var(--small-padding);
+  margin: var(--medium-padding);
+}
+
+.configuration-item h4 {
+  font-weight: bold;
+  margin: 0 0 var(--small-padding) 0;
+}
+
+.traces-table-container {
+  padding: 0;
+}
+
+.traces-table-container header {
+  padding: var(--large-padding);
+}
+
+.traces-table-container header h2 {
+  margin-top: 0;
+}
+
+.dash-filter--case {
+  display: none;
+}
+
+.traces-table-container td > div {
+  white-space: pre !important;
+  max-height: 150px !important;
+  overflow: auto !important;
+  display: inline-block !important;
+  text-align: left !important;
+}
+
+.traces-table-container th > div {
+  text-align: left !important;
+}
+
+.space-filler {
+  flex-grow: 1;
+}
+
+main > footer {
+  opacity: 0.35;
+  margin: 0;
+}
+
+main > footer {
+  display: flex;
+  justify-content: space-between;
+  align-items: center;
+  padding: var(--large-padding);
+  background-color: #ddd;
+  position: relative;
+}
+
+.parallel-coordinates {
+  padding: 0;
+}
+
+a img {
+  display: block;
+  margin-left: var(--large-padding);
+  width: 64px;
+  height: 64px;
+  cursor: pointer;
+  transition: transform 300ms;
+}
+
+a img:hover {
+  transform: scale(1.1);
+}

great_ai/src/great_ai/great_ai/deploy/routes/dashboard/create_dash_app.py

@@ -0,0 +1,254 @@
+from math import ceil
+from typing import Any, Dict, List, Tuple
+
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
+from dash import Dash, dcc, html
+from dash.dependencies import Input, Output
+from flask import Flask
+
+from great_ai.utilities.unique import unique
+
+from ....constants import DASHBOARD_PATH
+from ....context import get_context
+from ....helper import snake_case_to_text, text_to_hex_color
+from ....views import SortBy
+from .get_description import get_description
+from .get_filter_from_datatable import get_filter_from_datatable
+from .get_footer import get_footer
+from .get_traces_table import get_traces_table
+
+
+def create_dash_app(function_name: str, function_docs: str) -> Flask:
+    accent_color = text_to_hex_color(function_name)
+
+    flask_app = Flask(__name__)
+    app = Dash(
+        function_name,
+        requests_pathname_prefix=DASHBOARD_PATH + "/",
+        server=flask_app,
+        title=snake_case_to_text(function_name),
+        update_title=None,
+        external_stylesheets=[
+            "/assets/index.css",
+        ],
+    )
+
+    app.layout = html.Main(
+        [
+            html.Div(
+                html.P("PRODUCTION" if get_context().is_production else "DEVELOPMENT"),
+                className="environment",
+            ),
+            html.Header(
+                [
+                    get_description(
+                        function_name=function_name,
+                        function_docs=function_docs,
+                        accent_color=accent_color,
+                    ),
+                    execution_time_histogram_container := html.Div(),
+                ],
+            ),
+            configuration_container := html.Div(
+                className="configuration-container",
+            ),
+            traces_table_container := html.Div(
+                [
+                    html.Header(
+                        [
+                            html.H2("Latest traces"),
+                            html.P(
+                                "Recent traces and aggregated metrics are presented below. Try filtering the table."
+                            ),
+                            html.A(
+                                "Filtering syntax.",
+                                href="https://dash.plotly.com/datatable/filtering",
+                                target="_blank",
+                            ),
+                        ]
+                    ),
+                    table := get_traces_table(),
+                ],
+                className="traces-table-container",
+            ),
+            parallel_coordinates := dcc.Graph(
+                className="parallel-coordinates", config={"displaylogo": False}
+            ),
+            html.Div(className="space-filler"),
+            get_footer(),
+            interval := dcc.Interval(
+                interval=4 * 1000,  # in milliseconds
+            ),
+        ]
+    )
+
+    @app.callback(
+        Output(configuration_container, "children"),
+        Input(interval, "n_intervals"),
+    )
+    def update_configuration(
+        n_intervals: int,
+    ) -> List[html.Div]:
+        config = get_context().to_flat_dict()
+        return [
+            html.Div(
+                [
+                    html.H4(snake_case_to_text(key)),
+                    html.P(str(value)),
+                ],
+                className="configuration-item",
+            )
+            for key, value in config.items()
+        ]
+
+    @app.callback(
+        Output(table, "data"),
+        Output(table, "page_count"),
+        Input(table, "page_current"),
+        Input(table, "page_size"),
+        Input(table, "sort_by"),
+        Input(table, "filter_query"),
+        Input(interval, "n_intervals"),
+    )
+    def update_table(
+        page_current: int,
+        page_size: int,
+        sort_by: List[SortBy],
+        filter_query: str,
+        n_intervals: int,
+    ) -> Tuple[List[Dict[str, Any]], int]:
+        conjunctive_filters = (
+            [get_filter_from_datatable(f) for f in filter_query.split(" && ")]
+            if filter_query
+            else []
+        )
+        non_null_conjunctive_filters = [f for f in conjunctive_filters if f is not None]
+
+        elements, count = get_context().tracing_database.query(
+            skip=page_current * page_size,
+            take=page_size,
+            conjunctive_filters=non_null_conjunctive_filters,
+            sort_by=sort_by,
+        )
+
+        return (
+            [e.to_flat_dict() for e in elements],
+            max(1, ceil(count / page_size)),
+        )
+
+    @app.callback(
+        Output(table, "columns"),
+        Output(traces_table_container, "style"),
+        Input(interval, "n_intervals"),
+    )
+    def update_layout(
+        n_intervals: int,
+    ) -> Tuple[List[Dict[str, str]], Dict[str, Any]]:
+        elements, count = get_context().tracing_database.query(take=1)
+
+        if elements:
+            keys = list(elements[0].to_flat_dict().keys())
+            header_height = max(len(i.split(":")) for i in keys)
+            columns = [
+                {
+                    "name": [""] * (header_height - len(k.split(":")))
+                    + k.replace("_flat", "").split(":"),
+                    "id": k,
+                }
+                for k in keys
+            ]
+        else:
+            columns = []
+
+        return (
+            columns,
+            {"display": "block" if count > 0 else "none"},
+        )
+
+    @app.callback(
+        Output(execution_time_histogram_container, "children"),
+        Output(parallel_coordinates, "figure"),
+        Output(parallel_coordinates, "style"),
+        Input(table, "filter_query"),
+        Input(interval, "n_intervals"),
+    )
+    def update_charts(
+        filter_query: str, n_intervals: int
+    ) -> Tuple[Any, go.Figure, Dict[str, Any]]:
+        conjunctive_filters = (
+            [get_filter_from_datatable(f) for f in filter_query.split(" && ")]
+            if filter_query
+            else []
+        )
+        non_null_conjunctive_filters = [f for f in conjunctive_filters if f is not None]
+
+        elements, count = get_context().tracing_database.query(
+            conjunctive_filters=non_null_conjunctive_filters
+        )
+
+        elements = [e.to_flat_dict() for e in elements]
+
+        if not elements:
+            return (
+                html.Span(
+                    f"No traces yet: call your function ({function_name}) to create one.",
+                    className="placeholder",
+                ),
+                go.Figure(),
+                {"display": "none"},
+            )
+
+        execution_time_histogram = dcc.Graph(config={"displaylogo": False})
+        df = pd.DataFrame(elements)
+        fig = px.histogram(
+            df,
+            x="original_execution_time_ms",
+            labels={"original_execution_time_ms": "Execution time (ms)"},
+            nbins=20,
+            height=400,
+            log_y=True,
+            color_discrete_sequence=[accent_color],
+        )
+        fig.update_layout(
+            margin=dict(l=0, r=0, b=0, t=0, pad=0),
+        )
+        execution_time_histogram.figure = fig
+
+        parallel_coords_fig = go.Figure(
+            go.Parcoords(
+                dimensions=[
+                    get_dimension_descriptor(df, c)
+                    for c in df.columns
+                    if c
+                    not in {"trace_id", "created", "output", "exception", "feedback"}
+                    and "_flat" not in c
+                ],
+                line_color=accent_color,
+            )
+        )
+        return execution_time_histogram, parallel_coords_fig, {}
+
+    return flask_app
+
+
+def get_dimension_descriptor(df: pd.DataFrame, column: str) -> Dict[str, Any]:
+    dimension: Dict[str, Any] = {
+        "label": snake_case_to_text(column),
+    }
+
+    values = df[column]
+
+    try:
+        dimension["values"] = [float(v) for v in values]
+    except (TypeError, ValueError):
+        MAX_LENGTH = 40
+        unique_values = unique(values)
+        value_mapping = {str(v)[-MAX_LENGTH:]: i for i, v in enumerate(unique_values)}
+
+        dimension["values"] = [value_mapping[str(v)[-MAX_LENGTH:]] for v in values]
+        dimension["tickvals"] = list(value_mapping.values())
+        dimension["ticktext"] = [k[-MAX_LENGTH:] for k in value_mapping.keys()]
+
+    return dimension

great_ai/src/great_ai/great_ai/deploy/routes/dashboard/get_description.py

@@ -1,6 +1,6 @@
 from dash import dcc, html
 
-from ..helper import snake_case_to_text, strip_lines
+from ....helper import snake_case_to_text, strip_lines
 
 
 def get_description(
@@ -9,25 +9,21 @@ def get_description(
     return html.Div(
         [
             html.H1(
-                f"{snake_case_to_text(function_name)} - metrics",
+                f"{snake_case_to_text(function_name)} - dashboard",
                 style={"color": accent_color},
             ),
             dcc.Markdown(
                 strip_lines(
                     f"""
-                > View the live data of your deployments here.
+                > View the live data of your deployment here.
 
                 ## Using the API
 
                 You can find the available endpoints at [/docs](/docs).
 
-                ### Details
+                ## Details
 
                 {function_docs}
-
-                ## Metrics
-
-                Recent traces and aggregated metrics are presented below. Try filtering the table.
                 """
                 ),
                 className="description",

great_ai/src/great_ai/great_ai/deploy/routes/dashboard/get_filter_from_datatable.py

@@ -1,6 +1,6 @@
 from typing import Optional, Union
 
-from ..views import Filter, operators
+from ....views import Filter, operators
 
 
 def get_filter_from_datatable(description: str) -> Optional[Filter]:

great_ai/src/great_ai/great_ai/deploy/routes/dashboard/get_footer.py

@@ -1,5 +1,7 @@
 from dash import html
 
+from ....constants import GITHUB_LINK
+
 
 def get_footer() -> html.Footer:
     return html.Footer(
@@ -14,9 +16,8 @@ def get_footer() -> html.Footer:
             ),
             html.A(
                 html.Img(src="/assets/github.png"),
-                href="https://github.com/ScoutinScience/great-ai",
+                href=GITHUB_LINK,
                 target="_blank",
             ),
         ],
-        className="watermark",
     )

great_ai/src/great_ai/great_ai/deploy/routes/dashboard/get_traces_table.py

@@ -0,0 +1,32 @@
+from dash import dash_table
+
+
+def get_traces_table() -> dash_table.DataTable:
+    return dash_table.DataTable(
+        page_current=0,
+        page_size=20,
+        page_action="custom",
+        filter_action="custom",
+        sort_action="custom",
+        sort_mode="multi",
+        sort_by=[
+            {"column_id": "created", "direction": "desc"},
+        ],
+        style_data={
+            "white-space": "normal",
+            "height": "auto",
+            "max-height": "300px",
+            "overflow": "hidden",
+            "text-overflow": "ellipsis",
+        },
+        style_cell={"padding": "5px"},
+        style_header={
+            "background-color": "white",
+            "font-weight": "bold",
+        },
+        style_table={"max-height": "70vh", "overflow": "auto"},
+        merge_duplicate_headers=True,
+        style_cell_conditional=[
+            {"if": {"column_id": "output"}, "width": 1500},
+        ],
+    )

great_ai/src/great_ai/great_ai/helper/freeze_arguments.py

@@ -22,5 +22,4 @@ def freeze_arguments(func: Callable[..., Any]) -> Callable[..., Any]:
         }
         return func(*args, **kwargs)
 
-    wrapper.cache_info = func.cache_info  # type: ignore
     return wrapper

great_ai/src/great_ai/great_ai/parameters/parameter.py

@@ -20,7 +20,7 @@ def parameter(
         get_function_metadata_store(func).input_parameter_names.append(parameter_name)
         assert_function_is_not_finalised(func)
 
-        actual_name = f"arg:{func.__name__}:{parameter_name}"
+        actual_name = f"arg:{parameter_name}"
 
         @wraps(func)
         def wrapper(*args: Any, **kwargs: Dict[str, Any]) -> Any:
@@ -41,7 +41,7 @@ def parameter(
 
             context = TracingContext.get_current_context()
             if context and not disable_logging:
-                context.log_value(name=actual_name, value=argument)
+                context.log_value(name=f"{actual_name}:value", value=argument)
                 if isinstance(argument, str):
                     context.log_value(name=f"{actual_name}:length", value=len(argument))
 

great_ai/src/great_ai/great_ai/tracing/parallel_tinydb_driver.py

@@ -1,6 +1,6 @@
 from multiprocessing import Lock
 from pathlib import Path
-from typing import Any, Callable, Dict, List, Optional
+from typing import Any, Callable, Optional, Sequence, Tuple
 
 import pandas as pd
 from tinydb import TinyDB
@@ -34,39 +34,40 @@ class ParallelTinyDbDriver(TracingDatabase):
         self,
         skip: int = 0,
         take: Optional[int] = None,
-        conjunctive_filters: List[Filter] = [],
+        conjunctive_filters: Sequence[Filter] = [],
-        sort_by: List[SortBy] = [],
+        sort_by: Sequence[SortBy] = [],
-    ) -> List[Dict[str, Any]]:
+    ) -> Tuple[Sequence[Trace], int]:
         documents = [
-            d.to_flat_dict()
+            Trace.parse_obj(t) for t in self._safe_execute(lambda db: db.all())
-            for d in self._safe_execute(
-                lambda db: [Trace.parse_obj(t) for t in db.all()]
-            )
         ]
 
         if not documents:
-            return []
+            return [], 0
 
-        df = pd.DataFrame(documents)
+        df = pd.DataFrame([d.to_flat_dict() for d in documents])
 
         for f in conjunctive_filters:
-            if f.operator in operator_mapping:
+            operator = f.operator.lower()
+            if operator in operator_mapping:
                 df = df.loc[
                     getattr(df[f.property], operator_mapping[f.operator])(f.value)
                 ]
-            elif f.operator == "contains":
+            elif operator == "contains":
-                df = df.loc[df[f.property].str.contains(f.value)]
+                df = df.loc[df[f.property].str.contains(f.value, case=False)]
 
         if sort_by:
-            df = df.sort_values(
+            df.sort_values(
                 [col["column_id"] for col in sort_by],
                 ascending=[col["direction"] == "asc" for col in sort_by],
-                inplace=False,
+                inplace=True,
             )
 
+        count = len(df)
         result = df.iloc[skip:] if take is None else df.iloc[skip : skip + take]
-
+        return [
-        return result.to_dict("records")
+            next(d for d in documents if d.trace_id == trace_id)
+            for trace_id in result["trace_id"]
+        ], count
 
     def update(self, id: str, new_version: Trace) -> None:
         self._safe_execute(

great_ai/src/great_ai/great_ai/tracing/tracing_context.py

@@ -29,7 +29,7 @@ class TracingContext:
         delta_time = (datetime.utcnow() - self._start_time).microseconds / 1000
         self._trace = Trace(
             created=self._start_time.isoformat(),
-            execution_time_ms=delta_time,
+            original_execution_time_ms=delta_time,
             logged_values=self._values,
             models=self._models,
             output=output,

great_ai/src/great_ai/great_ai/tracing/tracing_database.py

@@ -1,5 +1,5 @@
 from abc import ABC, abstractmethod
-from typing import List, Optional
+from typing import Optional, Sequence, Tuple
 
 from ..views import Filter, SortBy, Trace
 
@@ -20,9 +20,9 @@ class TracingDatabase(ABC):
         self,
         skip: int = 0,
         take: Optional[int] = None,
-        conjunctive_filters: List[Filter] = [],
+        conjunctive_filters: Sequence[Filter] = [],
-        sort_by: List[SortBy] = [],
+        sort_by: Sequence[SortBy] = [],
-    ) -> List[Trace]:
+    ) -> Tuple[Sequence[Trace], int]:
         pass
 
     @abstractmethod

great_ai/src/great_ai/great_ai/views/__init__.py

@@ -9,3 +9,4 @@ from .operators import operators
 from .query import Query
 from .sort_by import SortBy
 from .trace import Trace
+from .trace_view import TraceView

great_ai/src/great_ai/great_ai/views/operators.py

@@ -2,12 +2,4 @@ from typing import List, Literal
 
 Operator = Literal[">=", "<=", "<", ">", "!=", "=", "contains"]
 
-operators: List[Operator] = [
+operators: List[Operator] = [">=", "<=", "<", ">", "!=", "=", "contains"]
-    ">=",
-    "<=",
-    "<",
-    ">",
-    "!=",
-    "=",
-    "contains",
-]

great_ai/src/great_ai/great_ai/views/query.py

@@ -14,10 +14,14 @@ class Query(BaseModel):
         schema_extra = {
             "example": {
                 "filter": [
-                    {"property": "execution_time_ms", "operator": ">", "value": 100}
+                    {
+                        "property": "original_execution_time_ms",
+                        "operator": ">",
+                        "value": 100,
+                    }
                 ],
                 "sort": [
-                    {"column_id": "execution_time_ms", "direction": "asc"},
+                    {"column_id": "original_execution_time_ms", "direction": "asc"},
                     {"column_id": "id", "direction": "desc"},
                 ],
             }

great_ai/src/great_ai/great_ai/views/trace.py

@@ -1,37 +1,33 @@
-from json import dumps
+from typing import Any, Dict, Optional
-from typing import Any, Dict, List, Optional
-from uuid import uuid4
 
-from pydantic import BaseModel, validator
+import yaml
+from pydantic import validator
 
-from .model import Model
+from .trace_view import TraceView
 
 
-class Trace(BaseModel):
+class Trace(TraceView):
-    trace_id: Optional[str]
+    models_flat: Optional[str]
-    created: str
+    output_flat: Optional[str]
-    execution_time_ms: float
+    feedback_flat: Optional[str]
-    logged_values: Dict[str, Any]
-    models: List[Model]
-    exception: Optional[str]
-    output: Any
-    feedback: Any = None
 
-    @validator("trace_id", always=True)
+    @validator("models_flat", always=True)
-    def generate_id(cls, v: Optional[str], values: Dict[str, Any]) -> Optional[str]:
+    def flatten_models(cls, v: Optional[str], values: Dict[str, Any]) -> str:
-        if not v:
+        return ", ".join(f"{m.key}:{m.version}" for m in values["models"])
-            return str(uuid4())
+
-        return v
+    @validator("output_flat", always=True)
+    def flatten_output(cls, v: Optional[str], values: Dict[str, Any]) -> str:
+        return yaml.dump(values["output"], default_flow_style=False, indent=2)
 
     def to_flat_dict(self) -> Dict[str, Any]:
         return {
-            "id": self.trace_id,
+            "trace_id": self.trace_id,
             "created": self.created,
-            "execution_time_ms": self.execution_time_ms,
+            "original_execution_time_ms": self.original_execution_time_ms,
-            "models": ", ".join(f"{m.key}:{m.version}" for m in self.models),
+            "models_flat": self.models_flat,
-            "output": dumps(self.output),
+            "output_flat": self.output_flat,
             "exception": self.exception or "null",
-            "feedback": self.feedback,
+            "feedback_flat": self.feedback_flat or "null",
             **self.logged_values,
         }
 

great_ai/src/great_ai/great_ai/views/trace_view.py

@@ -0,0 +1,26 @@
+from typing import Any, Dict, List, Optional
+from uuid import uuid4
+
+from pydantic import BaseModel, validator
+
+from .model import Model
+
+
+class TraceView(BaseModel):
+    trace_id: Optional[str]
+    created: str
+    original_execution_time_ms: float
+    logged_values: Dict[str, Any]
+    models: List[Model]
+    exception: Optional[str]
+    output: Any
+    feedback: Any = None
+
+    @validator("trace_id", always=True)
+    def generate_id(cls, v: Optional[str], values: Dict[str, Any]) -> Optional[str]:
+        if not v:
+            return str(uuid4())
+        return v
+
+    def __hash__(self) -> int:
+        return hash((type(self),) + tuple(self.__dict__.values()))