Modernise

great_ai/context.py

@@ -4,7 +4,7 @@ from logging import DEBUG, Logger
 from pathlib import Path
 from typing import Any, Dict, Optional, Type, Union, cast
 
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 
 from great_ai import __version__
 
@@ -34,8 +34,7 @@ class Context(BaseModel):
     dashboard_table_size: int
     route_config: RouteConfig
 
-    class Config:
+    model_config = ConfigDict(arbitrary_types_allowed=True)
-        arbitrary_types_allowed = True
 
     def to_flat_dict(self) -> Dict[str, Any]:
         return {
@@ -137,9 +136,11 @@ def configure(
         ),
         is_production=is_production,
         logger=logger,
-        should_log_exception_stack=not is_production
+        should_log_exception_stack=(
+            not is_production
             if should_log_exception_stack is None
-        else should_log_exception_stack,
+            else should_log_exception_stack
+        ),
         prediction_cache_size=prediction_cache_size,
         dashboard_table_size=dashboard_table_size,
         route_config=route_config,

great_ai/deploy/great_ai.py

@@ -16,7 +16,7 @@ from typing import (
 )
 
 from fastapi import FastAPI
-from tqdm.cli import tqdm
+from tqdm import tqdm
 from typing_extensions import Literal  # <= Python 3.7
 
 from ..constants import DASHBOARD_PATH
@@ -103,15 +103,13 @@ class GreatAI(Generic[T, V]):
         # Overloaded function signatures 1 and 2 overlap with incompatible return types
         # https://github.com/python/mypy/issues/12759
         func: Callable[..., Awaitable[V]],
-    ) -> "GreatAI[Awaitable[Trace[V]], V]":
+    ) -> "GreatAI[Awaitable[Trace[V]], V]": ...
-        ...
 
     @overload
     @staticmethod
     def create(
         func: Callable[..., V],
-    ) -> "GreatAI[Trace[V], V]":
+    ) -> "GreatAI[Trace[V], V]": ...
-        ...
 
     @staticmethod
     def create(
@@ -146,7 +144,7 @@ class GreatAI(Generic[T, V]):
             >>> my_function('3').output
             Traceback (most recent call last):
                 ...
-            TypeError: type of a must be int; got str instead
+            TypeError: argument a is not of the expected type: ...
 
         Args:
             func: The prediction function that needs to be decorated.
@@ -170,8 +168,7 @@ class GreatAI(Generic[T, V]):
         concurrency: Optional[int] = None,
         unpack_arguments: Literal[True],
         do_not_persist_traces: bool = ...,
-    ) -> List[Trace[V]]:
+    ) -> List[Trace[V]]: ...
-        ...
 
     @overload
     def process_batch(
@@ -181,8 +178,7 @@ class GreatAI(Generic[T, V]):
         concurrency: Optional[int] = None,
         unpack_arguments: Literal[False] = ...,
         do_not_persist_traces: bool = ...,
-    ) -> List[Trace[V]]:
+    ) -> List[Trace[V]]: ...
-        ...
 
     def process_batch(
         self,
@@ -237,9 +233,11 @@ class GreatAI(Generic[T, V]):
         return list(
             tqdm(
                 parallel_map(
+                    (
                         inner_async
                         if get_function_metadata_store(self).is_asynchronous
-                    else inner,
+                        else inner
+                    ),
                     batch,
                     concurrency=concurrency,
                 ),
@@ -249,7 +247,7 @@ class GreatAI(Generic[T, V]):
 
     @staticmethod
     def _get_cached_traced_function(
-        func: Callable[..., Union[V, Awaitable[V]]]
+        func: Callable[..., Union[V, Awaitable[V]]],
     ) -> Callable[..., T]:
         @lru_cache(maxsize=get_context().prediction_cache_size)
         def func_in_tracing_context_sync(

great_ai/deploy/routes/bootstrap_feedback_endpoints.py

@@ -31,7 +31,7 @@ def bootstrap_feedback_endpoints(app: FastAPI) -> None:
         return trace.feedback
 
     @router.delete("/", status_code=status.HTTP_204_NO_CONTENT)
-    def delete_feedback(trace_id: str) -> Any:
+    def delete_feedback(trace_id: str) -> Response:
         trace = get_context().tracing_database.get(trace_id)
         if trace is None:
             raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)

great_ai/deploy/routes/bootstrap_prediction_endpoint.py

@@ -22,10 +22,10 @@ def bootstrap_prediction_endpoint(
         try:
             if inspect.iscoroutinefunction(func):
                 return await cast(Callable[..., Awaitable[Trace]], func)(
-                    **cast(BaseModel, input_value).dict()
+                    **cast(BaseModel, input_value).model_dump()
                 )
             return cast(Callable[..., Trace], func)(
-                **cast(BaseModel, input_value).dict()
+                **cast(BaseModel, input_value).model_dump()
             )
         except Exception as e:
             raise HTTPException(

great_ai/deploy/routes/dashboard/create_dash_app.py

@@ -26,9 +26,9 @@ def create_dash_app(function_name: str, version: str, function_docs: str) -> Fla
     app = Dash(
         function_name,
         requests_pathname_prefix=DASHBOARD_PATH + "/",
-        server=Flask(__name__),
+        server=Flask(__name__),  # type: ignore[arg-type]
         title=function_name,
-        update_title=None,
+        update_title=None,  # type: ignore[arg-type]
         external_stylesheets=[
             "/assets/index.css",
         ],
@@ -64,9 +64,9 @@ def create_dash_app(function_name: str, version: str, function_docs: str) -> Fla
     interval = dcc.Interval(
         interval=2 * 1000,  # in milliseconds
         n_intervals=0,
-        max_intervals=1
+        max_intervals=(
-        if get_context().is_production
+            1 if get_context().is_production else -1
-        else -1,  # will be incremented in production upon each successful request
+        ),  # will be incremented in production upon each successful request
     )
 
     app.layout = html.Main(
@@ -159,7 +159,7 @@ def create_dash_app(function_name: str, version: str, function_docs: str) -> Fla
             take=page_size,
             conjunctive_filters=non_null_conjunctive_filters,
             conjunctive_tags=[ONLINE_TAG_NAME],
-            sort_by=[SortBy.parse_obj(s) for s in sort_by],
+            sort_by=[SortBy.model_validate(s) for s in sort_by],
         )
 
         if non_null_conjunctive_filters:
@@ -244,7 +244,7 @@ def update_charts(
     fig.update_layout(
         margin=dict(l=0, r=0, b=0, t=0, pad=0),
     )
-    execution_time_histogram.figure = fig
+    execution_time_histogram.figure = fig  # type: ignore[attr-defined]
 
     parallel_coords_fig = go.Figure(
         go.Parcoords(

great_ai/deploy/routes/dashboard/get_description.py

@@ -16,8 +16,7 @@ def get_description(
                 style={"color": accent_color},
             ),
             dcc.Markdown(
-                strip_lines(
+                strip_lines(f"""
-                    f"""
                     > View the live data of your deployment here.
 
                     ## Using the API
@@ -27,8 +26,7 @@ def get_description(
                     ## Details
 
                     {function_docs}
-                """
+                """),
-                ),
                 className="description",
             ),
         ]

great_ai/deploy/routes/dashboard/get_traces_table.py

@@ -1,10 +1,12 @@
+from typing import Any
+
 from dash import dash_table
 
 from ....context import get_context
 
 
-def get_traces_table() -> dash_table.DataTable:
+def get_traces_table() -> Any:
-    return dash_table.DataTable(
+    return dash_table.DataTable(  # type: ignore[attr-defined]
         page_current=0,
         page_size=get_context().dashboard_table_size,
         page_action="custom",

great_ai/helper/freeze_arguments.py

@@ -1,12 +1,24 @@
 import inspect
-from functools import wraps
+from functools import lru_cache, wraps
-from typing import Any, Callable, Mapping, Sequence, Set, TypeVar, Union, cast
+from typing import Any, Callable, Mapping, Sequence, Set, Type, TypeVar, Union, cast
 
 from pydantic import BaseModel
 
 F = TypeVar("F", bound=Callable)
 
 
+@lru_cache(maxsize=None)
+def _hashable_model_type(model_type: Type[BaseModel]) -> Type[BaseModel]:
+    # The subclass is cached per model type: Pydantic v2's __eq__ requires both
+    # operands to share a type, so creating a fresh subclass on every freeze()
+    # call would make even identical models compare unequal.
+    class HashableValue(model_type):  # type: ignore
+        def __hash__(self) -> int:
+            return hash(frozenset((k, freeze(v)) for k, v in self.model_dump().items()))
+
+    return HashableValue
+
+
 class FrozenDict(dict):
     def __hash__(self) -> int:  # type: ignore
         return hash(frozenset((k, freeze(v)) for k, v in self.items()))
@@ -55,11 +67,6 @@ def freeze(value: Union[Sequence[Any], Mapping[str, Any], Set[Any], BaseModel])
         return FrozenSet(value)
 
     if isinstance(value, BaseModel):
-
+        return _hashable_model_type(type(value))(**value.model_dump())
-        class HashableValue(type(value)):  # type: ignore
-            def __hash__(self) -> int:
-                return hash(frozenset((k, freeze(v)) for k, v in self.dict().items()))
-
-        return HashableValue(**value.dict())
 
     return value

great_ai/large_file/helper/cached_property.py

@@ -28,7 +28,9 @@ class cached_property:
             )
         try:
             cache = instance.__dict__
-        except AttributeError:  # not all objects have __dict__ (e.g. class defines slots)
+        except (
+            AttributeError
+        ):  # not all objects have __dict__ (e.g. class defines slots)
             msg = (
                 f"No '__dict__' attribute on {type(instance).__name__!r} "
                 f"instance to cache {self.attrname!r} property."

great_ai/large_file/large_file/large_file_mongo.py

@@ -2,8 +2,9 @@ import re
 from pathlib import Path
 from typing import Any, List
 
-from gridfs import DEFAULT_CHUNK_SIZE, Database, GridFSBucket
+from gridfs import DEFAULT_CHUNK_SIZE, GridFSBucket
 from pymongo import MongoClient
+from pymongo.database import Database
 
 from ...utilities import get_logger
 from ..helper import DownloadProgressBar, UploadProgressBar, cached_property
@@ -60,7 +61,6 @@ class LargeFileMongo(LargeFileBase):
                 ),
                 version=int(f.name.split(MONGO_NAME_VERSION_SEPARATOR)[-1]),
                 remote_path=(f._id, f.length),
-                origin="mongodb",
             )
             for f in self._client.find(
                 {

great_ai/large_file/large_file/large_file_s3.py

@@ -100,9 +100,13 @@ class LargeFileS3(LargeFileBase):
             Bucket=self.bucket_name,
             Key=remote_path,
             Filename=str(local_path),
-            Callback=None
+            Callback=(
+                None
                 if hide_progress
-            else DownloadProgressBar(name=str(remote_path), size=size, logger=logger),
+                else DownloadProgressBar(
+                    name=str(remote_path), size=size, logger=logger
+                )
+            ),
         )
 
     def _upload(self, local_path: Path, hide_progress: bool) -> None:
@@ -113,9 +117,11 @@ class LargeFileS3(LargeFileBase):
             Filename=str(local_path),
             Bucket=self.bucket_name,
             Key=key,
-            Callback=None
+            Callback=(
+                None
                 if hide_progress
-            else UploadProgressBar(path=local_path, logger=logger),
+                else UploadProgressBar(path=local_path, logger=logger)
+            ),
         )
 
     def _delete_old_remote_versions(self) -> None:

great_ai/parameters/parameter.py

@@ -1,7 +1,7 @@
 from functools import wraps
 from typing import Any, Callable, Dict, TypeVar, cast
 
-from typeguard import check_type
+from typeguard import TypeCheckError, check_type
 
 from ..errors import ArgumentValidationError
 from ..helper import get_arguments, get_function_metadata_store
@@ -32,7 +32,7 @@ def parameter(
         >>> my_function('3')
         Traceback (most recent call last):
             ...
-        TypeError: type of a must be int; got str instead
+        TypeError: argument a is not of the expected type: ...
 
         >>> @parameter('positive_a', validate=lambda v: v > 0)
         ... def my_function(positive_a: int):
@@ -65,13 +65,16 @@ def parameter(
             expected_type = func.__annotations__.get(parameter_name)
 
             if expected_type is not None:
-                check_type(parameter_name, argument, expected_type)
+                try:
+                    check_type(argument, expected_type)
+                except TypeCheckError as error:
+                    raise TypeError(
+                        f"argument {parameter_name} is not of the expected type: {error}"
+                    ) from error
 
             if not validate(argument):
                 raise ArgumentValidationError(
-                    f"""Argument {parameter_name} in {
+                    f"Argument {parameter_name} in {func.__name__} did not pass validation"
-                        func.__name__
-                    } did not pass validation"""
                 )
 
             context = TracingContext.get_current_tracing_context()

great_ai/persistence/mongodb_driver.py

@@ -90,7 +90,7 @@ class MongoDbDriver(TracingDatabaseDriver):
             value = client[self.mongo_database].traces.find_one(id)
 
         if value:
-            value = Trace.parse_obj(value)
+            value = Trace.model_validate(value)
 
         return value
 
@@ -150,7 +150,7 @@ class MongoDbDriver(TracingDatabaseDriver):
             ]
 
             with client[self.mongo_database].traces.find(**query) as cursor:
-                documents = [Trace[Any].parse_obj(t) for t in cursor]
+                documents = [Trace[Any].model_validate(t) for t in cursor]
         return documents, count
 
     def update(self, id: str, new_version: Trace) -> None:

great_ai/persistence/parallel_tinydb_driver.py

@@ -29,16 +29,16 @@ class ParallelTinyDbDriver(TracingDatabaseDriver):
     path_to_db = Path(DEFAULT_TRACING_DB_FILENAME)
 
     def save(self, trace: Trace) -> str:
-        return self._safe_execute(lambda db: db.insert(trace.dict()))
+        return self._safe_execute(lambda db: db.insert(trace.model_dump()))
 
     def save_batch(self, documents: List[Trace]) -> List[str]:
-        traces = [d.dict() for d in documents]
+        traces = [d.model_dump() for d in documents]
         return self._safe_execute(lambda db: db.insert_multiple(traces))
 
     def get(self, id: str) -> Optional[Trace]:
         value = self._safe_execute(lambda db: db.get(lambda d: d["trace_id"] == id))
         if value:
-            value = Trace.parse_obj(value)
+            value = Trace.model_validate(value)
         return value
 
     def query(
@@ -51,7 +51,7 @@ class ParallelTinyDbDriver(TracingDatabaseDriver):
         since: Optional[datetime] = None,
         until: Optional[datetime] = None,
         has_feedback: Optional[bool] = None,
-        sort_by: Sequence[SortBy] = []
+        sort_by: Sequence[SortBy] = [],
     ) -> Tuple[List[Trace], int]:
         def does_match(d: Dict[str, Any]) -> bool:
             return (
@@ -67,7 +67,11 @@ class ParallelTinyDbDriver(TracingDatabaseDriver):
         if not documents:
             return [], 0
 
-        df = pd.DataFrame([Trace.parse_obj(d).to_flat_dict() for d in documents])
+        traces: List[Trace] = [Trace.model_validate(d) for d in documents]
+        # The DataFrame keeps its default 0..n-1 index, so filtering/sorting below
+        # preserve the row labels that index back into `traces`. This avoids
+        # reconstructing Traces from the lossy flattened rows.
+        df = pd.DataFrame([t.to_flat_dict() for t in traces])
 
         for f in conjunctive_filters:
             operator = f.operator.lower()
@@ -92,11 +96,13 @@ class ParallelTinyDbDriver(TracingDatabaseDriver):
 
         count = len(df)
         result = df.iloc[skip:] if take is None else df.iloc[skip : skip + take]
-        return [Trace.parse_obj(trace) for _, trace in result.iterrows()], count
+        return [traces[i] for i in result.index], count
 
     def update(self, id: str, new_version: Trace) -> None:
         self._safe_execute(
-            lambda db: db.update(new_version.dict(), lambda d: d["trace_id"] == id)
+            lambda db: db.update(
+                new_version.model_dump(), lambda d: d["trace_id"] == id
+            )
         )
 
     def delete(self, id: str) -> None:

great_ai/persistence/tracing_database_driver.py

@@ -54,7 +54,7 @@ class TracingDatabaseDriver(ABC):
         until: Optional[datetime] = None,
         since: Optional[datetime] = None,
         has_feedback: Optional[bool] = None,
-        sort_by: Sequence[SortBy] = []
+        sort_by: Sequence[SortBy] = [],
     ) -> Tuple[List[Trace], int]:
         pass
 

great_ai/remote/call_remote_great_ai_async.py

@@ -64,7 +64,7 @@ async def call_remote_great_ai_async(
         )
     try:
         if model_class is not None:
-            trace["output"] = model_class.parse_obj(trace["output"])
+            trace["output"] = model_class.model_validate(trace["output"])
-        return Trace.parse_obj(trace)
+        return Trace.model_validate(trace)
     except Exception:
         raise RemoteCallError("Could not parse Trace")

great_ai/tracing/add_ground_truth.py

@@ -23,7 +23,7 @@ def add_ground_truth(
     tags: Union[List[str], str] = [],
     train_split_ratio: float = 1,
     test_split_ratio: float = 0,
-    validation_split_ratio: float = 0
+    validation_split_ratio: float = 0,
 ) -> None:
     """Add training data (with optional train-test splitting).
 

great_ai/tracing/query_ground_truth.py

@@ -10,7 +10,7 @@ def query_ground_truth(
     *,
     since: Optional[datetime] = None,
     until: Optional[datetime] = None,
-    return_max_count: Optional[int] = None
+    return_max_count: Optional[int] = None,
 ) -> List[Trace]:
     """Return training samples.
 

great_ai/tracing/tracing_context.py

@@ -51,15 +51,19 @@ class TracingContext(Generic[T]):
                     logged_values=self._values,
                     models=self._models,
                     output=output,
-                    exception=None
+                    exception=(
+                        None
                         if exception is None
-                    else f"{type(exception).__name__}: {exception}",
+                        else f"{type(exception).__name__}: {exception}"
+                    ),
                     tags=[
                         self._name,
                         ONLINE_TAG_NAME,
+                        (
                             PRODUCTION_TAG_NAME
                             if get_context().is_production
-                        else DEVELOPMENT_TAG_NAME,
+                            else DEVELOPMENT_TAG_NAME
+                        ),
                     ],
                 ),
             )

great_ai/utilities/config_file/config_file.py

@@ -94,11 +94,9 @@ class ConfigFile(Mapping[str, str]):
             try:
                 value = next(v for v in values if v)
             except StopIteration:
-                raise ParseError(
+                raise ParseError(f"""Cannot parse config file ({
-                    f"""Cannot parse config file ({
                         self._path.absolute()
-                        }), error at key `{key}`"""
+                        }), error at key `{key}`""")
-                )
 
             already_exists = key in self._key_values
             if already_exists and not value.startswith(

great_ai/utilities/evaluate_ranking/evaluate_ranking.py

@@ -50,7 +50,7 @@ def evaluate_ranking(
         fig.patch.set_facecolor("white")
         ax = plt.axes()
     else:
-        ax = axes
+        ax = axes  # type: ignore[assignment]
 
     classes = sorted(unique(expected), reverse=reverse_order)
     str_classes = [str(c) for c in classes]

great_ai/utilities/external/pylatexenc/latex2text/__init__.py

@@ -1549,7 +1549,7 @@ def latex2text(
         "in favor of the `pylatexenc.latex2text.LatexNodes2Text` class.",
     )
 
-    (nodelist, tpos, tlen) = latexwalker.get_latex_nodes(
+    nodelist, tpos, tlen = latexwalker.get_latex_nodes(
         content, keep_inline_math=keep_inline_math, tolerant_parsing=tolerant_parsing
     )
 

great_ai/utilities/external/pylatexenc/latex2text/__main__.py

@@ -289,7 +289,7 @@ def main(argv=None):
         latex, tolerant_parsing=args.tolerant_parsing, strict_braces=args.strict_braces
     )
 
-    (nodelist, pos, len_) = lw.get_latex_nodes()
+    nodelist, pos, len_ = lw.get_latex_nodes()
 
     ln2t = LatexNodes2Text(
         math_mode=args.math_mode,

great_ai/utilities/external/pylatexenc/latex2text/_defaultspecs.py

@@ -1774,7 +1774,7 @@ def make_accented_char(node, combining, l2tobj):
 
 
 for u in unicode_accents_list:
-    (mname, mcombining) = u
+    mname, mcombining = u
     _latex_specs_base["macros"].append(
         MacroTextSpec(
             mname, lambda x, l2tobj, c=mcombining: make_accented_char(x, c, l2tobj)

great_ai/utilities/external/pylatexenc/latexencode/_partial_latex_encoder.py

@@ -71,7 +71,7 @@ class PartialLatexToLatexEncoder(UnicodeToLatexEncoder):
         # keyword arguments:
         keep_latex_chars=r"\${}^_",
         conversion_rules=None,
-        **kwargs
+        **kwargs,
     ):
 
         base_conversion_rules = conversion_rules
@@ -89,7 +89,7 @@ class PartialLatexToLatexEncoder(UnicodeToLatexEncoder):
                 )
             ]
             + base_conversion_rules,
-            **kwargs
+            **kwargs,
         )
 
         self.keep_latex_chars = keep_latex_chars

great_ai/utilities/external/pylatexenc/latexencode/_unicode_to_latex_encoder.py

@@ -617,7 +617,7 @@ class UnicodeToLatexEncoder(object):
         res = rulecallable(s, p.pos)
         if res is None:
             return None
-        (consumed, repl) = res
+        consumed, repl = res
         self._apply_replacement(p, repl, consumed, rule)
         return True
 

great_ai/utilities/external/pylatexenc/latexwalker/__init__.py

@@ -477,7 +477,7 @@ class LatexNode(object):
         parsing_state=None,
         pos=None,
         len=None,
-        **kwargs
+        **kwargs,
     ):
 
         # Important: subclasses must specify a list of fields they set in the
@@ -607,7 +607,7 @@ class LatexGroupNode(LatexNode):
                 "nodelist",
                 "delimiters",
             ),
-            **kwargs
+            **kwargs,
         )
         self.nodelist = nodelist
         self.delimiters = delimiters
@@ -639,7 +639,7 @@ class LatexCommentNode(LatexNode):
                 "comment",
                 "comment_post_space",
             ),
-            **kwargs
+            **kwargs,
         )
 
         self.comment = comment
@@ -715,7 +715,7 @@ class LatexMacroNode(LatexNode):
         super(LatexMacroNode, self).__init__(
             _fields=("macroname", "nodeargd", "macro_post_space"),
             _redundant_fields=("nodeoptarg", "nodeargs"),
-            **kwargs
+            **kwargs,
         )
 
         self.macroname = macroname
@@ -801,7 +801,7 @@ class LatexEnvironmentNode(LatexNode):
                 "optargs",
                 "args",
             ),
-            **kwargs
+            **kwargs,
         )
 
         self.environmentname = environmentname
@@ -1264,7 +1264,7 @@ class LatexWalker(object):
         environments=True,
         keep_inline_math=None,
         parsing_state=None,
-        **kwargs
+        **kwargs,
     ):
         r"""
         Parses the latex content given to the constructor (and stored in `self.s`),
@@ -1628,7 +1628,7 @@ class LatexWalker(object):
                             r"Expected expression, got \end",
                             self.s,
                             pos,
-                            **self.pos_to_lineno_colno(pos, as_dict=True)
+                            **self.pos_to_lineno_colno(pos, as_dict=True),
                         )
                     else:
                         return self._mknodeposlen(
@@ -1674,7 +1674,7 @@ class LatexWalker(object):
                         "Expected expression, got closing brace '{}'".format(tok.arg),
                         self.s,
                         pos,
-                        **self.pos_to_lineno_colno(pos, as_dict=True)
+                        **self.pos_to_lineno_colno(pos, as_dict=True),
                     )
                 return self._mknodeposlen(
                     LatexCharsNode,
@@ -1717,7 +1717,7 @@ class LatexWalker(object):
                 "Unknown token type: {}".format(tok.tok),
                 self.s,
                 pos,
-                **self.pos_to_lineno_colno(pos, as_dict=True)
+                **self.pos_to_lineno_colno(pos, as_dict=True),
             )
 
     def get_latex_maybe_optional_arg(self, pos, parsing_state=None):
@@ -1821,10 +1821,10 @@ class LatexWalker(object):
                 pos=pos,
                 msg="get_latex_braced_group: not an opening brace/bracket: %s"
                 % (self.s[pos]),
-                **self.pos_to_lineno_colno(pos, as_dict=True)
+                **self.pos_to_lineno_colno(pos, as_dict=True),
             )
 
-        (nodelist, npos, nlen) = self.get_latex_nodes(
+        nodelist, npos, nlen = self.get_latex_nodes(
             firsttok.pos + firsttok.len,
             stop_upon_closing_brace=(brace_type, closing_brace),
             parsing_state=parsing_state,
@@ -1883,12 +1883,14 @@ class LatexWalker(object):
                 pos=pos,
                 msg=r"get_latex_environment: expected \begin{%s}: %s"
                 % (
+                    (
                         environmentname
                         if environmentname is not None
-                    else "<environment name>",
+                        else "<environment name>"
+                    ),
                     firsttok.arg,
                 ),
-                **self.pos_to_lineno_colno(pos, as_dict=True)
+                **self.pos_to_lineno_colno(pos, as_dict=True),
             )
         if environmentname is None:
             environmentname = firsttok.arg
@@ -1915,9 +1917,9 @@ class LatexWalker(object):
             argsresult = (None, pos, 0, {})
 
         if len(argsresult) == 4:
-            (argd, apos, alen, adic) = argsresult
+            argd, apos, alen, adic = argsresult
         else:
-            (argd, apos, alen) = argsresult
+            argd, apos, alen = argsresult
             adic = {}
 
         pos = apos + alen
@@ -1930,7 +1932,7 @@ class LatexWalker(object):
                 math_mode_delimiter="{" + environmentname + "}",
             )
 
-        (nodelist, npos, nlen) = self.get_latex_nodes(
+        nodelist, npos, nlen = self.get_latex_nodes(
             pos,
             stop_upon_end_environment=environmentname,
             parsing_state=parsing_state_inner,
@@ -1975,7 +1977,7 @@ class LatexWalker(object):
                 s=self.s,
                 pos=tok.pos,
                 msg="End of input while parsing {}".format(what),
-                **self.pos_to_lineno_colno(tok.pos, as_dict=True)
+                **self.pos_to_lineno_colno(tok.pos, as_dict=True),
             )
 
         if getattr(e, "pos", None) is not None and e.lineno is None and e.colno is None:
@@ -2227,7 +2229,7 @@ class LatexWalker(object):
                         s=self.s,
                         pos=tok.pos,
                         msg="Unexpected mismatching closing brace: '%s'" % (tok.arg),
-                        **self.pos_to_lineno_colno(tok.pos, as_dict=True)
+                        **self.pos_to_lineno_colno(tok.pos, as_dict=True),
                     )
                 return True
 
@@ -2239,7 +2241,7 @@ class LatexWalker(object):
                         s=self.s,
                         pos=tok.pos,
                         msg=("Unexpected closing environment: '{}'".format(tok.arg)),
-                        **self.pos_to_lineno_colno(tok.pos, as_dict=True)
+                        **self.pos_to_lineno_colno(tok.pos, as_dict=True),
                     )
                 elif tok.arg != stop_upon_end_environment:
                     # p.push_lastchars(tok_to_pos_and_chars_from_ppos(tok))
@@ -2252,7 +2254,7 @@ class LatexWalker(object):
                                 tok.arg, stop_upon_end_environment
                             )
                         ),
-                        **self.pos_to_lineno_colno(tok.pos, as_dict=True)
+                        **self.pos_to_lineno_colno(tok.pos, as_dict=True),
                     )
                 return True
 
@@ -2276,7 +2278,7 @@ class LatexWalker(object):
                                 tok.arg,
                                 stop_upon_closing_mathmode,
                             ),
-                            **self.pos_to_lineno_colno(tok.pos, as_dict=True)
+                            **self.pos_to_lineno_colno(tok.pos, as_dict=True),
                         )
                     # all ok, this is a new math mode opening.  Keep an assert
                     # in case we forget to include some math-mode delimiters in
@@ -2291,7 +2293,7 @@ class LatexWalker(object):
                         s=self.s,
                         pos=tok.pos,
                         msg="Unexpected closing math mode: '{}'".format(tok.arg),
-                        **self.pos_to_lineno_colno(tok.pos, as_dict=True)
+                        **self.pos_to_lineno_colno(tok.pos, as_dict=True),
                     )
 
                 # we have encountered a new math inline, parse the math expression
@@ -2306,7 +2308,7 @@ class LatexWalker(object):
                 )
 
                 try:
-                    (mathinline_nodelist, mpos, mlen) = self.get_latex_nodes(
+                    mathinline_nodelist, mpos, mlen = self.get_latex_nodes(
                         p.pos,
                         stop_upon_closing_mathmode=corresponding_closing_mathmode,
                         parsing_state=parsing_state_inner,
@@ -2316,7 +2318,7 @@ class LatexWalker(object):
                         _maketuple(
                             'math mode "{}"'.format(tok.arg),
                             tok.pos,
-                            *self.pos_to_lineno_colno(tok.pos)
+                            *self.pos_to_lineno_colno(tok.pos),
                         )
                     )
                     raise
@@ -2354,7 +2356,7 @@ class LatexWalker(object):
             if tok.tok == "brace_open":
                 # another braced group to read.
                 try:
-                    (groupnode, bpos, blen) = self.get_latex_braced_group(
+                    groupnode, bpos, blen = self.get_latex_braced_group(
                         tok.pos, brace_type=tok.arg, parsing_state=p.parsing_state
                     )
                 # except LatexWalkerEndOfStream as e:
@@ -2376,7 +2378,7 @@ class LatexWalker(object):
             if tok.tok == "begin_environment":
                 # an environment to read.
                 try:
-                    (envnode, epos, elen) = self.get_latex_environment(
+                    envnode, epos, elen = self.get_latex_environment(
                         tok.pos, environmentname=tok.arg, parsing_state=p.parsing_state
                     )
                 except LatexWalkerParseError as e:
@@ -2384,7 +2386,7 @@ class LatexWalker(object):
                         _maketuple(
                             'begin environment "{}"'.format(tok.arg),
                             tok.pos,
-                            *self.pos_to_lineno_colno(tok.pos)
+                            *self.pos_to_lineno_colno(tok.pos),
                         )
                     )
                     raise
@@ -2415,9 +2417,9 @@ class LatexWalker(object):
                     margsresult = (None, tok.pos + tok.len, 0, {})
 
                 if len(margsresult) == 4:
-                    (nodeargd, mapos, malen, mdic) = margsresult
+                    nodeargd, mapos, malen, mdic = margsresult
                 else:
-                    (nodeargd, mapos, malen) = margsresult
+                    nodeargd, mapos, malen = margsresult
                     mdic = {}
 
                 p.pos = mapos + malen
@@ -2473,9 +2475,9 @@ class LatexWalker(object):
                 if res is not None:
                     # specials expects arguments, read them
                     if len(res) == 4:
-                        (nodeargd, mapos, malen, spdic) = res
+                        nodeargd, mapos, malen, spdic = res
                     else:
-                        (nodeargd, mapos, malen) = res
+                        nodeargd, mapos, malen = res
                         spdic = {}
 
                     p.pos = mapos + malen
@@ -2505,7 +2507,7 @@ class LatexWalker(object):
                 s=self.s,
                 pos=p.pos,
                 msg="Unknown token: {!r}".format(tok),
-                **self.pos_to_lineno_colno(p.pos, as_dict=True)
+                **self.pos_to_lineno_colno(p.pos, as_dict=True),
             )
 
         while True:
@@ -2531,7 +2533,7 @@ class LatexWalker(object):
                         msg="Unexpected end of stream, was expecting {}".format(
                             expecting
                         ),
-                        **self.pos_to_lineno_colno(len(self.s), as_dict=True)
+                        **self.pos_to_lineno_colno(len(self.s), as_dict=True),
                     )
                     if self.tolerant_parsing:
                         r_endnow = True
@@ -2651,7 +2653,7 @@ def get_latex_nodes(
     stop_upon_closing_brace=None,
     stop_upon_end_environment=None,
     stop_upon_closing_mathmode=None,
-    **parse_flags
+    **parse_flags,
 ):
     """
     Parses latex content `s`.

great_ai/utilities/external/pylatexenc/latexwalker/__main__.py

@@ -184,7 +184,7 @@ def main(argv=None):
         latex, tolerant_parsing=args.tolerant_parsing, strict_braces=args.strict_braces
     )
 
-    (nodelist, pos, len_) = latexwalker.get_latex_nodes()
+    nodelist, pos, len_ = latexwalker.get_latex_nodes()
 
     if args.output_format == "human":
         print("\n--- NODES ---\n")

great_ai/utilities/external/pylatexenc/macrospec/__init__.py

@@ -34,7 +34,6 @@ macros and environments, specifying how they should be parsed by
    `pylatexenc 2.0`.
 """
 
-
 import sys
 
 if sys.version_info.major > 2:
@@ -154,7 +153,7 @@ class EnvironmentSpec(object):
         environmentname,
         args_parser=MacroStandardArgsParser(),
         is_math_mode=False,
-        **kwargs
+        **kwargs,
     ):
         super(EnvironmentSpec, self).__init__(**kwargs)
         self.environmentname = environmentname
@@ -776,9 +775,9 @@ class LatexContextDb(object):
             new_context.add_context_category(
                 cat,
                 macros=self.d[cat]["macros"].values() if keep_macros else [],
-                environments=self.d[cat]["environments"].values()
+                environments=(
-                if keep_environments
+                    self.d[cat]["environments"].values() if keep_environments else []
-                else [],
+                ),
                 specials=self.d[cat]["specials"].values() if keep_specials else [],
             )
 

great_ai/utilities/external/pylatexenc/macrospec/_argparsers.py

@@ -296,7 +296,7 @@ class MacroStandardArgsParser(object):
 
         for j, argt in enumerate(self.argspec):
             if argt == "{":
-                (node, np, nl) = w.get_latex_expression(
+                node, np, nl = w.get_latex_expression(
                     p, strict_braces=False, parsing_state=get_inner_parsing_state(j)
                 )
                 p = np + nl
@@ -315,7 +315,7 @@ class MacroStandardArgsParser(object):
                 if optarginfotuple is None:
                     argnlist.append(None)
                     continue
-                (node, np, nl) = optarginfotuple
+                node, np, nl = optarginfotuple
                 p = np + nl
                 argnlist.append(node)
 

great_ai/utilities/parallel_map/manage_communication.py

@@ -39,11 +39,9 @@ def manage_communication(
                 is_iteration_over = True
             except Exception as e:
                 if ignore_exceptions:
-                    logger.error(
+                    logger.error(f"""Exception {e} encountered in input, traceback:\n{
-                        f"""Exception {e} encountered in input, traceback:\n{
                             traceback.format_exc()
-                        }"""
+                        }""")
-                    )
                 else:
                     raise
 

great_ai/utilities/parallel_map/parallel_map.py

@@ -31,8 +31,7 @@ def parallel_map(
     chunk_size: Optional[int] = ...,
     concurrency: Optional[int] = ...,
     unordered: bool = ...,
-) -> Iterable[Optional[V]]:
+) -> Iterable[Optional[V]]: ...
-    ...
 
 
 @overload
@@ -44,8 +43,7 @@ def parallel_map(
     ignore_exceptions: Literal[True],
     concurrency: Optional[int] = ...,
     unordered: bool = ...,
-) -> Iterable[Optional[V]]:
+) -> Iterable[Optional[V]]: ...
-    ...
 
 
 @overload
@@ -57,8 +55,7 @@ def parallel_map(
     ignore_exceptions: Literal[False] = ...,
     concurrency: Optional[int] = ...,
     unordered: bool = ...,
-) -> Iterable[V]:
+) -> Iterable[V]: ...
-    ...
 
 
 @overload
@@ -70,8 +67,7 @@ def parallel_map(
     ignore_exceptions: Literal[False] = ...,
     concurrency: Optional[int] = ...,
     unordered: bool = ...,
-) -> Iterable[V]:
+) -> Iterable[V]: ...
-    ...
 
 
 def parallel_map(

great_ai/utilities/parallel_map/simple_parallel_map.py

@@ -1,6 +1,6 @@
 from typing import Awaitable, Callable, List, Optional, Sequence, TypeVar, Union
 
-from tqdm.cli import tqdm
+from tqdm import tqdm
 
 from .parallel_map import parallel_map
 

great_ai/utilities/parallel_map/threaded_parallel_map.py

@@ -30,8 +30,7 @@ def threaded_parallel_map(
     chunk_size: Optional[int] = ...,
     concurrency: Optional[int] = ...,
     unordered: bool = ...,
-) -> Iterable[Optional[V]]:
+) -> Iterable[Optional[V]]: ...
-    ...
 
 
 @overload
@@ -43,8 +42,7 @@ def threaded_parallel_map(
     ignore_exceptions: Literal[True],
     concurrency: Optional[int] = ...,
     unordered: bool = ...,
-) -> Iterable[Optional[V]]:
+) -> Iterable[Optional[V]]: ...
-    ...
 
 
 @overload
@@ -56,8 +54,7 @@ def threaded_parallel_map(
     ignore_exceptions: Literal[False] = ...,
     concurrency: Optional[int] = ...,
     unordered: bool = ...,
-) -> Iterable[V]:
+) -> Iterable[V]: ...
-    ...
 
 
 @overload
@@ -69,8 +66,7 @@ def threaded_parallel_map(
     ignore_exceptions: Literal[False] = ...,
     concurrency: Optional[int] = ...,
     unordered: bool = ...,
-) -> Iterable[V]:
+) -> Iterable[V]: ...
-    ...
 
 
 def threaded_parallel_map(

great_ai/views/outputs/classification_output.py

@@ -6,4 +6,4 @@ from ..hashable_base_model import HashableBaseModel
 class ClassificationOutput(HashableBaseModel):
     label: Union[str, int]
     confidence: float
-    explanation: Optional[Any]
+    explanation: Optional[Any] = None

great_ai/views/outputs/regression_output.py

@@ -5,4 +5,4 @@ from ..hashable_base_model import HashableBaseModel
 
 class RegressionOutput(HashableBaseModel):
     value: Union[int, float]
-    explanation: Optional[Any]
+    explanation: Optional[Any] = None

great_ai/views/outputs/sequence_labeling_output.py

@@ -9,9 +9,9 @@ class LabeledToken(HashableBaseModel):
     token: str
     tag: Literal["B", "I", "O", "E", "S"]
     confidence: float
-    explanation: Optional[Any]
+    explanation: Optional[Any] = None
 
 
 class SequenceLabelingOutput(HashableBaseModel):
     labeled_tokens: List[LabeledToken]
-    explanation: Optional[Any]
+    explanation: Optional[Any] = None

great_ai/views/query.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 from typing import List, Optional, Sequence
 
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 
 from .filter import Filter
 from .sort_by import SortBy
@@ -15,8 +15,8 @@ class Query(BaseModel):
     until: Optional[datetime] = None
     has_feedback: Optional[bool] = None
 
-    class Config:
+    model_config = ConfigDict(
-        schema_extra = {
+        json_schema_extra={
             "example": {
                 "filter": [
                     {
@@ -33,3 +33,4 @@ class Query(BaseModel):
                 "has_feedback": False,
             }
         }
+    )

great_ai/views/trace.py

@@ -1,7 +1,7 @@
 from pprint import pformat
 from typing import Any, Dict, Generic, List, Optional, TypeVar
 
-from pydantic import Extra
+from pydantic import ConfigDict
 
 from .hashable_base_model import HashableBaseModel
 from .model import Model
@@ -32,13 +32,12 @@ class Trace(Generic[T], HashableBaseModel):
     original_execution_time_ms: float
     logged_values: Dict[str, Any]
     models: List[Model]
-    exception: Optional[str]
+    exception: Optional[str] = None
-    output: Optional[T]
+    output: Optional[T] = None
     feedback: Any = None
     tags: List[str]
 
-    class Config:
+    model_config = ConfigDict(extra="ignore")
-        extra = Extra.ignore
 
     @property
     def input(self) -> Any:
@@ -79,7 +78,7 @@ class Trace(Generic[T], HashableBaseModel):
     def to_flat_dict(self, include_original: bool = True) -> Dict[str, Any]:
         return {
             **(
-                self.dict()
+                self.model_dump()
                 if include_original
                 else {
                     "trace_id": self.trace_id,
@@ -88,9 +87,11 @@ class Trace(Generic[T], HashableBaseModel):
                 }
             ),
             **{
-                k: v
+                k: (
+                    v
                     if (isinstance(v, float) or isinstance(v, int))
                     else pformat(v, indent=2, compact=True)
+                )
                 for k, v in self.logged_values.items()
             },
             "models_flat": self.models_flat,
@@ -101,6 +102,7 @@ class Trace(Generic[T], HashableBaseModel):
         }
 
     def __repr__(self) -> str:
-        return f"""Trace[{type(self.output).__name__}]({
+        formatted = pformat(self.model_dump(), indent=2, compact=True).replace(
-            pformat(self.dict(), indent=2, compact=True).replace('{ ', '{', 1)
+            "{ ", "{", 1
-        })"""
+        )
+        return f"Trace[{type(self.output).__name__}]({formatted})"

pyproject.toml

@@ -66,6 +66,11 @@ dev = [
     "pytest-asyncio",
 ]
 
+[tool.pytest.ini_options]
+# `...` in doctests (e.g. third-party exception messages that drift between
+# dependency versions) is treated as a wildcard rather than a literal.
+doctest_optionflags = ["ELLIPSIS"]
+
 [project.urls]
 Documentation = "https://great-ai.scoutinscience.com"
 GitHub = "https://github.com/schmelczer/great-ai"

scripts/check-python.sh

@@ -16,10 +16,10 @@ for dir in "$@"; do
 
     if find . -name "*.py" 2>/dev/null | grep -q .; then
         yes | python3 -m mypy . --install-types > /dev/null || true
-        python3 -m mypy --namespace-packages --ignore-missing-imports --install-types --non-interactive --disallow-untyped-defs --disallow-incomplete-defs --follow-imports=silent --exclude=external/ --exclude=/build/ --pretty .
+        python3 -m mypy --namespace-packages --ignore-missing-imports --install-types --non-interactive --disallow-untyped-defs --disallow-incomplete-defs --follow-imports=silent --exclude=external/ --exclude=/build/ --exclude='(^|/)__main__\.py$' --pretty .
     fi
 
-    python3 -m flake8 . --count --show-source --statistics --exclude=__init__.py,.env,external --ignore=E501,E402,F821,W503,E722,E203
+    python3 -m flake8 . --count --show-source --statistics --exclude=__init__.py,.env,external --ignore=E501,E402,F821,W503,E722,E203,E704
 
     cd -
 done

tests/external/test_internals.py

@@ -24,7 +24,8 @@ class Wrapped:
 
 
 @pytest.mark.asyncio
-async def test_done_callback_cancelled(loop):
+async def test_done_callback_cancelled():
+    loop = asyncio.get_running_loop()
     task = loop.create_future()
     fut = loop.create_future()
 
@@ -38,7 +39,8 @@ async def test_done_callback_cancelled(loop):
 
 
 @pytest.mark.asyncio
-async def test_done_callback_exception(loop):
+async def test_done_callback_exception():
+    loop = asyncio.get_running_loop()
     task = loop.create_future()
     fut = loop.create_future()
 
@@ -60,7 +62,8 @@ async def test_done_callback_exception(loop):
 
 
 @pytest.mark.asyncio
-async def test_done_callback(loop):
+async def test_done_callback():
+    loop = asyncio.get_running_loop()
     task = loop.create_future()
     fut = loop.create_future()
 
@@ -228,7 +231,8 @@ def test_close(loop):
 
 
 @pytest.mark.asyncio
-async def test_wait_closed(loop):
+async def test_wait_closed():
+    loop = asyncio.get_running_loop()
     wrapped = Wrapped()
     wrapped.tasks = set()
 

tests/utilities/test_config_file.py

@@ -14,14 +14,11 @@ def test_simple() -> None:
     assert c.first_key == "András"
     assert c.second_key == "test 2"
     assert c.third_key == "test= 2=="
-    assert (
+    assert c.fourth_key == """
-        c.fourth_key
-        == """
 this#
 is
 multiline
 """
-    )
     assert c.whitespace == "hardly  matters"
     with pytest.raises(KeyError):
         c.this
@@ -34,14 +31,11 @@ def test_simple_dict() -> None:
     assert c["my_hashtag"] == "#great_ai"
     assert c["second_key"] == "test 2"
     assert c["third_key"] == "test= 2=="
-    assert (
+    assert c["fourth_key"] == """
-        c["fourth_key"]
-        == """
 this#
 is
 multiline
 """
-    )
     assert c["whitespace"] == "hardly  matters"
     with pytest.raises(KeyError):
         c["#this"]
@@ -55,14 +49,11 @@ def test_string_path() -> None:
 
     assert c.second_key == "test 2"
     assert c.third_key == "test= 2=="
-    assert (
+    assert c.fourth_key == """
-        c.fourth_key
-        == """
 this#
 is
 multiline
 """
-    )
     assert c.whitespace == "hardly  matters"
 
 

tests/utilities/test_parallel_map.py

@@ -23,9 +23,7 @@ def test_with_iterable() -> None:
 
     expected = [v**3 for v in range(10)]
 
-    assert (
+    assert list(parallel_map(lambda x: x**3, my_generator(), chunk_size=1)) == expected
-        list(parallel_map(lambda x: x**3, my_generator(), chunk_size=1)) == expected
-    )
 
 
 def test_simple_case_invalid_values() -> None:
@@ -44,14 +42,10 @@ def test_this_process_exception() -> None:
         assert False
 
     with pytest.raises(AssertionError):
-        list(
+        list(parallel_map(lambda v: v**2, my_generator(), concurrency=2, chunk_size=2))
-            parallel_map(lambda v: v**2, my_generator(), concurrency=2, chunk_size=2)
-        )
 
     with pytest.raises(AssertionError):
-        list(
+        list(parallel_map(lambda v: v**2, my_generator(), concurrency=1, chunk_size=2))
-            parallel_map(lambda v: v**2, my_generator(), concurrency=1, chunk_size=2)
-        )
 
 
 def test_ignore_this_process_exception() -> None: