Add max deps and bump old ones

.forgejo/workflows/docker.yml

@@ -0,0 +1,57 @@
+name: Publish on DockerHub
+
+on:
+  push:
+    tags: ['*']
+  workflow_dispatch:
+
+jobs:
+  publish:
+    runs-on: docker
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      # The `docker` runner image is node-based and may not ship the docker CLI
+      # that the docker/* actions below drive; install it idempotently, as the
+      # sibling repos do for their Docker jobs.
+      - name: Ensure Docker CLI
+        run: |
+          set -eux
+          if ! command -v docker >/dev/null 2>&1; then
+            ARCH=$(uname -m)
+            curl -fsSL --retry 3 --retry-connrefused \
+              "https://download.docker.com/linux/static/stable/${ARCH}/docker-27.5.1.tgz" \
+              | tar xz --strip-components=1 -C /usr/local/bin docker/docker
+          fi
+          docker --version
+
+      # Marketplace actions must be referenced by full URL: this instance's
+      # DEFAULT_ACTIONS_URL points at code.forgejo.org, so only bare `actions/*`
+      # names resolve to GitHub automatically.
+      - name: Set up QEMU
+        uses: https://github.com/docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: https://github.com/docker/setup-buildx-action@v3
+
+      - name: Log in to Docker Hub
+        uses: https://github.com/docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Extract metadata for the Docker image
+        id: meta
+        uses: https://github.com/docker/metadata-action@v5
+        with:
+          images: schmelczera/great-ai
+
+      - name: Build and push
+        uses: https://github.com/docker/build-push-action@v6
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64,linux/arm64

.forgejo/workflows/docs.yml

@@ -0,0 +1,52 @@
+name: Publish documentation
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'docs/**'
+      - 'great_ai/**'
+      - 'mkdocs.yaml'
+      - '.forgejo/workflows/docs.yml'
+  workflow_dispatch:
+
+concurrency:
+  group: pages
+  cancel-in-progress: false
+
+jobs:
+  publish:
+    runs-on: docker
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # The mkdocs git-revision-date plugin reads each page's git history.
+          fetch-depth: 0
+
+      - name: Install uv
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+
+      - name: Set up Python
+        run: uv python install 3.10
+
+      - name: Install dependencies
+        run: |
+          uv venv --python 3.10
+          uv pip install '.[dev]'
+
+      - name: Build documentation
+        run: |
+          . .venv/bin/activate
+          mkdocs build
+
+      # The old workflow ran `mkdocs gh-deploy` (GitHub Pages); on this instance
+      # built sites are rsynced into the host /pages mount that nginx serves,
+      # via the shared ci-actions/deploy-pages action — same as photos/reconcile.
+      - name: Deploy to pages mount
+        if: github.ref == 'refs/heads/main'
+        uses: http://forgejo:3000/andras/ci-actions/deploy-pages@main
+        with:
+          source: site
+          target: great-ai

.forgejo/workflows/publish.yml

@@ -0,0 +1,28 @@
+name: Publish on PyPI
+
+on:
+  push:
+    tags: ['*']
+  workflow_dispatch:
+
+jobs:
+  publish:
+    runs-on: docker
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+
+      - name: Set up Python
+        run: uv python install 3.10
+
+      # Forgejo cannot use PyPI trusted publishing (OIDC), so authenticate with
+      # an API token, the same way the sibling repos publish their packages.
+      - name: Build and publish
+        env:
+          FLIT_USERNAME: __token__
+          FLIT_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+        run: uvx flit publish

.forgejo/workflows/test.yml

@@ -0,0 +1,73 @@
+name: Check
+
+on:
+  push:
+    branches: [main, dev]
+  pull_request:
+    branches: [main, dev]
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ gitea.workflow }}-${{ gitea.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lint:
+    name: Lint, format & type checks
+    runs-on: docker
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+
+      - name: Set up Python
+        run: uv python install 3.10
+
+      - name: Install dependencies
+        # --seed gives the venv its own pip, which scripts/check-python.sh shells
+        # out to for its linters; '.[dev]' provides the rest (mypy resolves the
+        # package's own imports against the installed tree).
+        run: |
+          uv venv --seed --python 3.10
+          uv pip install '.[dev]'
+
+      - name: Check code and formatting
+        run: |
+          . .venv/bin/activate
+          scripts/check-python.sh great_ai tests
+
+  test:
+    name: Test on Python ${{ matrix.python-version }}
+    runs-on: docker
+    strategy:
+      fail-fast: false
+      matrix:
+        # pyproject declares >= 3.7, but uv's standalone CPython and the current
+        # dependency floors (scikit-learn/numpy now require >= 3.9) no longer
+        # build on the older interpreters, so we test the current supported set.
+        # The windows leg of the old GitHub matrix is dropped: this Forgejo
+        # instance only has a Linux `docker` runner.
+        python-version: ['3.10', '3.11', '3.12', '3.13']
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          uv venv --python ${{ matrix.python-version }}
+          uv pip install '.[dev]'
+
+      - name: Run tests
+        run: |
+          . .venv/bin/activate
+          pytest --doctest-modules --asyncio-mode=strict

.github/dependabot.yml

@@ -1,6 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: "github-actions"
-    directory: "/"
-    schedule:
-      interval: "daily"

.github/workflows/codeql-analysis.yml

@@ -1,38 +0,0 @@
-name: analyse with CodeQL
-
-on:
-  push:
-    branches: 
-      - main
-  pull_request:
-    branches: 
-      - main
-  schedule:
-    - cron: '45 13 * * 5'
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language:
-          - python
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
-
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
-      with:
-        languages: ${{ matrix.language }}
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2

.github/workflows/docker.yaml

@@ -1,39 +0,0 @@
-name: publish on DockerHub
-
-on:
-  push:
-    tags:
-      - '*'
-
-jobs:
-  publish:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Log in to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Extract metadata for the Docker image
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: schmelczera/great-ai
-
-      - name: Build and push
-        uses: docker/build-push-action@v3
-        with:
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          platforms: linux/amd64,linux/arm64

.github/workflows/docs.yaml

@@ -1,27 +0,0 @@
-name: publish documentation
-
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - main
-      - dev
-
-jobs:
-  publish:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.9
-
-      - name: Install dependencies
-        run: pip install --upgrade './[dev]'
-
-      - name: Build documentation
-        run: mkdocs gh-deploy

.github/workflows/publish.yaml

@@ -1,26 +0,0 @@
-name: publish on PyPI
-
-on:
-  push:
-    tags:
-      - '*'
-
-jobs:
-  publish:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.9
-
-      - name: Install Flit
-        run: pip install --upgrade flit --user
-
-      - name: Build and publish
-        run: flit publish --setup-py 
-        env:
-          FLIT_USERNAME: __token__
-          FLIT_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/test.yml

@@ -1,96 +0,0 @@
-name: tests
-
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - main
-      - dev
-
-jobs:
-  test:
-      name: Build and test on ${{ matrix.operating-system }} with Python ${{ matrix.python-version }}
-
-      strategy:
-        matrix:
-          python-version: ["3.7", "3.8", "3.9", "3.10"]
-          operating-system: [windows-latest, ubuntu-latest]
-
-      runs-on: ${{ matrix.operating-system }}
-
-      steps:
-        - name: Checkout
-          uses: actions/checkout@v3
-
-        - name: Setup Python ${{ matrix.python-version }}
-          uses: actions/setup-python@v4
-          with:
-            python-version: ${{ matrix.python-version }}
-
-        - name: Install dependencies
-          run: |
-            python3 -m pip install --upgrade pip
-            pip install --upgrade './[dev]'
-
-        - name: Check code and formatting
-          run: scripts/check-python.sh great_ai tests
-
-        - name: Run tests
-          run: python3 -m pytest --doctest-modules --cov=. --cov-report=xml --junit-xml pytest.xml --asyncio-mode=strict || true
-
-        - name: Upload results
-          if: always()
-          uses: actions/upload-artifact@v3
-          with:
-            name: Unit test coverage (Python ${{ matrix.python-version }} - ${{ matrix.operating-system }})
-            path: "*.xml"
-
-  sonar:
-    name: Analyse Python project with SonarQube
-    needs: test
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install --upgrade .
-          rm -rf build
-
-      - name: Download test results
-        uses: actions/download-artifact@v3
-        with:
-          path: artifacts
-
-      - uses: sonarsource/sonarqube-scan-action@master
-        env:
-          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
-          SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
-        with:
-          args: >
-            -Dsonar.projectKey=great-ai
-            -Dsonar.login=${{ secrets.SONAR_TOKEN }}
-            -Dsonar.verbose=true
-            -Dsonar.python.coverage.reportPaths=artifacts/*/coverage.xml
-            -Dsonar.coverage.exclusions=**/external/**/*
-            -Dsonar.exclusions=**/external/**/*,artifacts/**/*
-
-  publish-test-results:
-    name: Publish unit test results
-    needs: test
-    runs-on: ubuntu-latest
-    if: always()
-    steps:
-      - name: Download artifacts
-        uses: actions/download-artifact@v3
-        with:
-          path: artifacts
-
-      - name: Publish results
-        uses: EnricoMi/publish-unit-test-result-action@v2
-        with:
-          files: artifacts/*/pytest.xml

great_ai/context.py

@@ -4,7 +4,7 @@ from logging import DEBUG, Logger
 from pathlib import Path
 from typing import Any, Dict, Optional, Type, Union, cast
 
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 
 from great_ai import __version__
 
@@ -34,8 +34,7 @@ class Context(BaseModel):
     dashboard_table_size: int
     route_config: RouteConfig
 
-    class Config:
+    model_config = ConfigDict(arbitrary_types_allowed=True)
-        arbitrary_types_allowed = True
 
     def to_flat_dict(self) -> Dict[str, Any]:
         return {
@@ -137,9 +136,11 @@ def configure(
         ),
         is_production=is_production,
         logger=logger,
-        should_log_exception_stack=not is_production
+        should_log_exception_stack=(
+            not is_production
             if should_log_exception_stack is None
-        else should_log_exception_stack,
+            else should_log_exception_stack
+        ),
         prediction_cache_size=prediction_cache_size,
         dashboard_table_size=dashboard_table_size,
         route_config=route_config,

great_ai/deploy/great_ai.py

@@ -6,6 +6,7 @@ from typing import (
     Callable,
     Generic,
     List,
+    Literal,
     Optional,
     Sequence,
     Tuple,
@@ -16,8 +17,7 @@ from typing import (
 )
 
 from fastapi import FastAPI
-from tqdm.cli import tqdm
+from tqdm import tqdm
-from typing_extensions import Literal  # <= Python 3.7
 
 from ..constants import DASHBOARD_PATH
 from ..context import get_context
@@ -103,15 +103,13 @@ class GreatAI(Generic[T, V]):
         # Overloaded function signatures 1 and 2 overlap with incompatible return types
         # https://github.com/python/mypy/issues/12759
         func: Callable[..., Awaitable[V]],
-    ) -> "GreatAI[Awaitable[Trace[V]], V]":
+    ) -> "GreatAI[Awaitable[Trace[V]], V]": ...
-        ...
 
     @overload
     @staticmethod
     def create(
         func: Callable[..., V],
-    ) -> "GreatAI[Trace[V], V]":
+    ) -> "GreatAI[Trace[V], V]": ...
-        ...
 
     @staticmethod
     def create(
@@ -146,7 +144,7 @@ class GreatAI(Generic[T, V]):
             >>> my_function('3').output
             Traceback (most recent call last):
                 ...
-            TypeError: type of a must be int; got str instead
+            TypeError: argument a is not of the expected type: ...
 
         Args:
             func: The prediction function that needs to be decorated.
@@ -170,8 +168,7 @@ class GreatAI(Generic[T, V]):
         concurrency: Optional[int] = None,
         unpack_arguments: Literal[True],
         do_not_persist_traces: bool = ...,
-    ) -> List[Trace[V]]:
+    ) -> List[Trace[V]]: ...
-        ...
 
     @overload
     def process_batch(
@@ -181,8 +178,7 @@ class GreatAI(Generic[T, V]):
         concurrency: Optional[int] = None,
         unpack_arguments: Literal[False] = ...,
         do_not_persist_traces: bool = ...,
-    ) -> List[Trace[V]]:
+    ) -> List[Trace[V]]: ...
-        ...
 
     def process_batch(
         self,
@@ -234,22 +230,24 @@ class GreatAI(Generic[T, V]):
                 ),
             )
 
+        # inner/inner_async return the class' T (bound to Trace | Awaitable[Trace]);
+        # in this method T resolves to Trace[V], but that is not provable to mypy, so
+        # cast to the concrete shape parallel_map expects.
+        map_function = cast(
+            Callable[[Any], Union[Trace[V], Awaitable[Trace[V]]]],
+            inner_async if get_function_metadata_store(self).is_asynchronous else inner,
+        )
+
         return list(
             tqdm(
-                parallel_map(
+                parallel_map(map_function, batch, concurrency=concurrency),
-                    inner_async
-                    if get_function_metadata_store(self).is_asynchronous
-                    else inner,
-                    batch,
-                    concurrency=concurrency,
-                ),
                 total=len(batch),
             )
         )
 
     @staticmethod
     def _get_cached_traced_function(
-        func: Callable[..., Union[V, Awaitable[V]]]
+        func: Callable[..., Union[V, Awaitable[V]]],
     ) -> Callable[..., T]:
         @lru_cache(maxsize=get_context().prediction_cache_size)
         def func_in_tracing_context_sync(

great_ai/deploy/routes/bootstrap_dashboard.py

@@ -1,7 +1,7 @@
 from pathlib import Path
 
+from a2wsgi import WSGIMiddleware
 from fastapi import FastAPI
-from fastapi.middleware.wsgi import WSGIMiddleware
 from fastapi.responses import FileResponse, RedirectResponse
 from fastapi.staticfiles import StaticFiles
 
@@ -19,7 +19,10 @@ def bootstrap_dashboard(app: FastAPI, function_name: str, documentation: str) ->
     def get_favicon() -> FileResponse:
         return FileResponse(PATH / "dashboard/assets/favicon.ico")
 
-    app.mount(DASHBOARD_PATH, WSGIMiddleware(dash_app))
+    # a2wsgi types its WSGI input (Flask) and ASGI output more strictly than
+    # Starlette's loose WSGIApp/ASGIApp aliases, so mypy flags the mount despite
+    # this being the canonical, runtime-correct way to bridge the Dash app.
+    app.mount(DASHBOARD_PATH, WSGIMiddleware(dash_app))  # type: ignore[arg-type]
 
     @app.get("/", include_in_schema=False)
     def redirect_to_entrypoint() -> RedirectResponse:

great_ai/deploy/routes/bootstrap_feedback_endpoints.py

@@ -31,7 +31,7 @@ def bootstrap_feedback_endpoints(app: FastAPI) -> None:
         return trace.feedback
 
     @router.delete("/", status_code=status.HTTP_204_NO_CONTENT)
-    def delete_feedback(trace_id: str) -> Any:
+    def delete_feedback(trace_id: str) -> Response:
         trace = get_context().tracing_database.get(trace_id)
         if trace is None:
             raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)

great_ai/deploy/routes/bootstrap_prediction_endpoint.py

@@ -22,10 +22,10 @@ def bootstrap_prediction_endpoint(
         try:
             if inspect.iscoroutinefunction(func):
                 return await cast(Callable[..., Awaitable[Trace]], func)(
-                    **cast(BaseModel, input_value).dict()
+                    **cast(BaseModel, input_value).model_dump()
                 )
             return cast(Callable[..., Trace], func)(
-                **cast(BaseModel, input_value).dict()
+                **cast(BaseModel, input_value).model_dump()
             )
         except Exception as e:
             raise HTTPException(

great_ai/deploy/routes/dashboard/create_dash_app.py

@@ -26,9 +26,9 @@ def create_dash_app(function_name: str, version: str, function_docs: str) -> Fla
     app = Dash(
         function_name,
         requests_pathname_prefix=DASHBOARD_PATH + "/",
-        server=Flask(__name__),
+        server=Flask(__name__),  # type: ignore[arg-type]
         title=function_name,
-        update_title=None,
+        update_title=None,  # type: ignore[arg-type]
         external_stylesheets=[
             "/assets/index.css",
         ],
@@ -64,9 +64,9 @@ def create_dash_app(function_name: str, version: str, function_docs: str) -> Fla
     interval = dcc.Interval(
         interval=2 * 1000,  # in milliseconds
         n_intervals=0,
-        max_intervals=1
+        max_intervals=(
-        if get_context().is_production
+            1 if get_context().is_production else -1
-        else -1,  # will be incremented in production upon each successful request
+        ),  # will be incremented in production upon each successful request
     )
 
     app.layout = html.Main(
@@ -159,7 +159,7 @@ def create_dash_app(function_name: str, version: str, function_docs: str) -> Fla
             take=page_size,
             conjunctive_filters=non_null_conjunctive_filters,
             conjunctive_tags=[ONLINE_TAG_NAME],
-            sort_by=[SortBy.parse_obj(s) for s in sort_by],
+            sort_by=[SortBy.model_validate(s) for s in sort_by],
         )
 
         if non_null_conjunctive_filters:
@@ -244,7 +244,7 @@ def update_charts(
     fig.update_layout(
         margin=dict(l=0, r=0, b=0, t=0, pad=0),
     )
-    execution_time_histogram.figure = fig
+    execution_time_histogram.figure = fig  # type: ignore[attr-defined]
 
     parallel_coords_fig = go.Figure(
         go.Parcoords(

great_ai/deploy/routes/dashboard/get_traces_table.py

@@ -1,10 +1,12 @@
+from typing import Any
+
 from dash import dash_table
 
 from ....context import get_context
 
 
-def get_traces_table() -> dash_table.DataTable:
+def get_traces_table() -> Any:
-    return dash_table.DataTable(
+    return dash_table.DataTable(  # type: ignore[attr-defined]
         page_current=0,
         page_size=get_context().dashboard_table_size,
         page_action="custom",

great_ai/helper/freeze_arguments.py

@@ -1,12 +1,24 @@
 import inspect
-from functools import wraps
+from functools import lru_cache, wraps
-from typing import Any, Callable, Mapping, Sequence, Set, TypeVar, Union, cast
+from typing import Any, Callable, Mapping, Sequence, Set, Type, TypeVar, Union, cast
 
 from pydantic import BaseModel
 
 F = TypeVar("F", bound=Callable)
 
 
+@lru_cache(maxsize=None)
+def _hashable_model_type(model_type: Type[BaseModel]) -> Type[BaseModel]:
+    # The subclass is cached per model type: Pydantic v2's __eq__ requires both
+    # operands to share a type, so creating a fresh subclass on every freeze()
+    # call would make even identical models compare unequal.
+    class HashableValue(model_type):  # type: ignore
+        def __hash__(self) -> int:
+            return hash(frozenset((k, freeze(v)) for k, v in self.model_dump().items()))
+
+    return HashableValue
+
+
 class FrozenDict(dict):
     def __hash__(self) -> int:  # type: ignore
         return hash(frozenset((k, freeze(v)) for k, v in self.items()))
@@ -55,11 +67,6 @@ def freeze(value: Union[Sequence[Any], Mapping[str, Any], Set[Any], BaseModel])
         return FrozenSet(value)
 
     if isinstance(value, BaseModel):
-
+        return _hashable_model_type(type(value))(**value.model_dump())
-        class HashableValue(type(value)):  # type: ignore
-            def __hash__(self) -> int:
-                return hash(frozenset((k, freeze(v)) for k, v in self.dict().items()))
-
-        return HashableValue(**value.dict())
 
     return value

great_ai/large_file/helper/cached_property.py

@@ -28,7 +28,9 @@ class cached_property:
             )
         try:
             cache = instance.__dict__
-        except AttributeError:  # not all objects have __dict__ (e.g. class defines slots)
+        except (
+            AttributeError
+        ):  # not all objects have __dict__ (e.g. class defines slots)
             msg = (
                 f"No '__dict__' attribute on {type(instance).__name__!r} "
                 f"instance to cache {self.attrname!r} property."

great_ai/large_file/large_file/large_file_base.py

@@ -7,9 +7,7 @@ from abc import ABC, abstractmethod
 from functools import lru_cache
 from pathlib import Path
 from types import TracebackType
-from typing import IO, Any, List, Optional, Type, Union, cast
+from typing import IO, Any, List, Literal, Optional, Type, Union, cast
-
-from typing_extensions import Literal  # <= Python 3.7
 
 from great_ai.utilities import ConfigFile, get_logger
 

great_ai/large_file/large_file/large_file_mongo.py

@@ -2,8 +2,9 @@ import re
 from pathlib import Path
 from typing import Any, List
 
-from gridfs import DEFAULT_CHUNK_SIZE, Database, GridFSBucket
+from gridfs import DEFAULT_CHUNK_SIZE, GridFSBucket
 from pymongo import MongoClient
+from pymongo.database import Database
 
 from ...utilities import get_logger
 from ..helper import DownloadProgressBar, UploadProgressBar, cached_property
@@ -60,7 +61,6 @@ class LargeFileMongo(LargeFileBase):
                 ),
                 version=int(f.name.split(MONGO_NAME_VERSION_SEPARATOR)[-1]),
                 remote_path=(f._id, f.length),
-                origin="mongodb",
             )
             for f in self._client.find(
                 {

great_ai/large_file/large_file/large_file_s3.py

@@ -100,9 +100,13 @@ class LargeFileS3(LargeFileBase):
             Bucket=self.bucket_name,
             Key=remote_path,
             Filename=str(local_path),
-            Callback=None
+            Callback=(
+                None
                 if hide_progress
-            else DownloadProgressBar(name=str(remote_path), size=size, logger=logger),
+                else DownloadProgressBar(
+                    name=str(remote_path), size=size, logger=logger
+                )
+            ),
         )
 
     def _upload(self, local_path: Path, hide_progress: bool) -> None:
@@ -113,9 +117,11 @@ class LargeFileS3(LargeFileBase):
             Filename=str(local_path),
             Bucket=self.bucket_name,
             Key=key,
-            Callback=None
+            Callback=(
+                None
                 if hide_progress
-            else UploadProgressBar(path=local_path, logger=logger),
+                else UploadProgressBar(path=local_path, logger=logger)
+            ),
         )
 
     def _delete_old_remote_versions(self) -> None:

great_ai/models/use_model.py

@@ -1,8 +1,19 @@
 from functools import wraps
-from typing import Any, Callable, Dict, List, Optional, Set, Tuple, TypeVar, Union, cast
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    List,
+    Literal,
+    Optional,
+    Set,
+    Tuple,
+    TypeVar,
+    Union,
+    cast,
+)
 
 from dill import load
-from typing_extensions import Literal  # <= Python 3.7
 
 from ..context import get_context
 from ..helper import get_function_metadata_store

great_ai/parameters/parameter.py

@@ -1,7 +1,7 @@
 from functools import wraps
 from typing import Any, Callable, Dict, TypeVar, cast
 
-from typeguard import check_type
+from typeguard import TypeCheckError, check_type
 
 from ..errors import ArgumentValidationError
 from ..helper import get_arguments, get_function_metadata_store
@@ -32,7 +32,7 @@ def parameter(
         >>> my_function('3')
         Traceback (most recent call last):
             ...
-        TypeError: type of a must be int; got str instead
+        TypeError: argument a is not of the expected type: ...
 
         >>> @parameter('positive_a', validate=lambda v: v > 0)
         ... def my_function(positive_a: int):
@@ -65,13 +65,16 @@ def parameter(
             expected_type = func.__annotations__.get(parameter_name)
 
             if expected_type is not None:
-                check_type(parameter_name, argument, expected_type)
+                try:
+                    check_type(argument, expected_type)
+                except TypeCheckError as error:
+                    raise TypeError(
+                        f"argument {parameter_name} is not of the expected type: {error}"
+                    ) from error
 
             if not validate(argument):
                 raise ArgumentValidationError(
-                    f"""Argument {parameter_name} in {
+                    f"Argument {parameter_name} in {func.__name__} did not pass validation"
-                        func.__name__
-                    } did not pass validation"""
                 )
 
             context = TracingContext.get_current_tracing_context()

great_ai/persistence/mongodb_driver.py

@@ -90,7 +90,7 @@ class MongoDbDriver(TracingDatabaseDriver):
             value = client[self.mongo_database].traces.find_one(id)
 
         if value:
-            value = Trace.parse_obj(value)
+            value = Trace.model_validate(value)
 
         return value
 
@@ -150,7 +150,7 @@ class MongoDbDriver(TracingDatabaseDriver):
             ]
 
             with client[self.mongo_database].traces.find(**query) as cursor:
-                documents = [Trace[Any].parse_obj(t) for t in cursor]
+                documents = [Trace[Any].model_validate(t) for t in cursor]
         return documents, count
 
     def update(self, id: str, new_version: Trace) -> None:

great_ai/persistence/parallel_tinydb_driver.py

@@ -29,16 +29,16 @@ class ParallelTinyDbDriver(TracingDatabaseDriver):
     path_to_db = Path(DEFAULT_TRACING_DB_FILENAME)
 
     def save(self, trace: Trace) -> str:
-        return self._safe_execute(lambda db: db.insert(trace.dict()))
+        return self._safe_execute(lambda db: db.insert(trace.model_dump()))
 
     def save_batch(self, documents: List[Trace]) -> List[str]:
-        traces = [d.dict() for d in documents]
+        traces = [d.model_dump() for d in documents]
         return self._safe_execute(lambda db: db.insert_multiple(traces))
 
     def get(self, id: str) -> Optional[Trace]:
         value = self._safe_execute(lambda db: db.get(lambda d: d["trace_id"] == id))
         if value:
-            value = Trace.parse_obj(value)
+            value = Trace.model_validate(value)
         return value
 
     def query(
@@ -51,7 +51,7 @@ class ParallelTinyDbDriver(TracingDatabaseDriver):
         since: Optional[datetime] = None,
         until: Optional[datetime] = None,
         has_feedback: Optional[bool] = None,
-        sort_by: Sequence[SortBy] = []
+        sort_by: Sequence[SortBy] = [],
     ) -> Tuple[List[Trace], int]:
         def does_match(d: Dict[str, Any]) -> bool:
             return (
@@ -67,7 +67,11 @@ class ParallelTinyDbDriver(TracingDatabaseDriver):
         if not documents:
             return [], 0
 
-        df = pd.DataFrame([Trace.parse_obj(d).to_flat_dict() for d in documents])
+        traces: List[Trace] = [Trace.model_validate(d) for d in documents]
+        # The DataFrame keeps its default 0..n-1 index, so filtering/sorting below
+        # preserve the row labels that index back into `traces`. This avoids
+        # reconstructing Traces from the lossy flattened rows.
+        df = pd.DataFrame([t.to_flat_dict() for t in traces])
 
         for f in conjunctive_filters:
             operator = f.operator.lower()
@@ -92,11 +96,13 @@ class ParallelTinyDbDriver(TracingDatabaseDriver):
 
         count = len(df)
         result = df.iloc[skip:] if take is None else df.iloc[skip : skip + take]
-        return [Trace.parse_obj(trace) for _, trace in result.iterrows()], count
+        return [traces[i] for i in result.index], count
 
     def update(self, id: str, new_version: Trace) -> None:
         self._safe_execute(
-            lambda db: db.update(new_version.dict(), lambda d: d["trace_id"] == id)
+            lambda db: db.update(
+                new_version.model_dump(), lambda d: d["trace_id"] == id
+            )
         )
 
     def delete(self, id: str) -> None:

great_ai/persistence/tracing_database_driver.py

@@ -54,7 +54,7 @@ class TracingDatabaseDriver(ABC):
         until: Optional[datetime] = None,
         since: Optional[datetime] = None,
         has_feedback: Optional[bool] = None,
-        sort_by: Sequence[SortBy] = []
+        sort_by: Sequence[SortBy] = [],
     ) -> Tuple[List[Trace], int]:
         pass
 

great_ai/remote/call_remote_great_ai_async.py

@@ -64,7 +64,7 @@ async def call_remote_great_ai_async(
         )
     try:
         if model_class is not None:
-            trace["output"] = model_class.parse_obj(trace["output"])
+            trace["output"] = model_class.model_validate(trace["output"])
-        return Trace.parse_obj(trace)
+        return Trace.model_validate(trace)
     except Exception:
         raise RemoteCallError("Could not parse Trace")

great_ai/tracing/add_ground_truth.py

@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import datetime, timezone
 from math import ceil
 from random import shuffle
 from typing import Any, Iterable, List, TypeVar, Union, cast
@@ -23,7 +23,7 @@ def add_ground_truth(
     tags: Union[List[str], str] = [],
     train_split_ratio: float = 1,
     test_split_ratio: float = 0,
-    validation_split_ratio: float = 0
+    validation_split_ratio: float = 0,
 ) -> None:
     """Add training data (with optional train-test splitting).
 
@@ -94,7 +94,7 @@ def add_ground_truth(
     )
     shuffle(split_tags)
 
-    created = datetime.utcnow().isoformat()
+    created = datetime.now(timezone.utc).replace(tzinfo=None).isoformat()
     traces = [
         cast(
             Trace[T],

great_ai/tracing/query_ground_truth.py

@@ -10,7 +10,7 @@ def query_ground_truth(
     *,
     since: Optional[datetime] = None,
     until: Optional[datetime] = None,
-    return_max_count: Optional[int] = None
+    return_max_count: Optional[int] = None,
 ) -> List[Trace]:
     """Return training samples.
 

great_ai/tracing/tracing_context.py

@@ -1,12 +1,10 @@
 from contextvars import ContextVar
-from datetime import datetime
+from datetime import datetime, timezone
 from time import perf_counter
 from types import TracebackType
-from typing import Any, Dict, Generic, List, Optional, Type, TypeVar, cast
+from typing import Any, Dict, Generic, List, Literal, Optional, Type, TypeVar, cast
 from uuid import uuid4
 
-from typing_extensions import Literal  # <= Python 3.7
-
 from ..constants import DEVELOPMENT_TAG_NAME, ONLINE_TAG_NAME, PRODUCTION_TAG_NAME
 from ..context import get_context
 from ..views import Model, Trace
@@ -25,7 +23,7 @@ class TracingContext(Generic[T]):
         self._models: List[Model] = []
         self._values: Dict[str, Any] = {}
         self._trace: Optional[Trace[T]] = None
-        self._start_datetime = datetime.utcnow()
+        self._start_datetime = datetime.now(timezone.utc).replace(tzinfo=None)
         self._start_time = perf_counter()
         self._name = function_name
 
@@ -51,15 +49,19 @@ class TracingContext(Generic[T]):
                     logged_values=self._values,
                     models=self._models,
                     output=output,
-                    exception=None
+                    exception=(
+                        None
                         if exception is None
-                    else f"{type(exception).__name__}: {exception}",
+                        else f"{type(exception).__name__}: {exception}"
+                    ),
                     tags=[
                         self._name,
                         ONLINE_TAG_NAME,
+                        (
                             PRODUCTION_TAG_NAME
                             if get_context().is_production
-                        else DEVELOPMENT_TAG_NAME,
+                            else DEVELOPMENT_TAG_NAME
+                        ),
                     ],
                 ),
             )

great_ai/utilities/evaluate_ranking/evaluate_ranking.py

@@ -50,7 +50,7 @@ def evaluate_ranking(
         fig.patch.set_facecolor("white")
         ax = plt.axes()
     else:
-        ax = axes
+        ax = axes  # type: ignore[assignment]
 
     classes = sorted(unique(expected), reverse=reverse_order)
     str_classes = [str(c) for c in classes]

great_ai/utilities/external/pylatexenc/latex2text/__init__.py

@@ -1549,7 +1549,7 @@ def latex2text(
         "in favor of the `pylatexenc.latex2text.LatexNodes2Text` class.",
     )
 
-    (nodelist, tpos, tlen) = latexwalker.get_latex_nodes(
+    nodelist, tpos, tlen = latexwalker.get_latex_nodes(
         content, keep_inline_math=keep_inline_math, tolerant_parsing=tolerant_parsing
     )
 

great_ai/utilities/external/pylatexenc/latex2text/__main__.py

@@ -289,7 +289,7 @@ def main(argv=None):
         latex, tolerant_parsing=args.tolerant_parsing, strict_braces=args.strict_braces
     )
 
-    (nodelist, pos, len_) = lw.get_latex_nodes()
+    nodelist, pos, len_ = lw.get_latex_nodes()
 
     ln2t = LatexNodes2Text(
         math_mode=args.math_mode,

great_ai/utilities/external/pylatexenc/latex2text/_defaultspecs.py

@@ -1774,7 +1774,7 @@ def make_accented_char(node, combining, l2tobj):
 
 
 for u in unicode_accents_list:
-    (mname, mcombining) = u
+    mname, mcombining = u
     _latex_specs_base["macros"].append(
         MacroTextSpec(
             mname, lambda x, l2tobj, c=mcombining: make_accented_char(x, c, l2tobj)

great_ai/utilities/external/pylatexenc/latexencode/_partial_latex_encoder.py

@@ -71,7 +71,7 @@ class PartialLatexToLatexEncoder(UnicodeToLatexEncoder):
         # keyword arguments:
         keep_latex_chars=r"\${}^_",
         conversion_rules=None,
-        **kwargs
+        **kwargs,
     ):
 
         base_conversion_rules = conversion_rules
@@ -89,7 +89,7 @@ class PartialLatexToLatexEncoder(UnicodeToLatexEncoder):
                 )
             ]
             + base_conversion_rules,
-            **kwargs
+            **kwargs,
         )
 
         self.keep_latex_chars = keep_latex_chars

great_ai/utilities/external/pylatexenc/latexencode/_unicode_to_latex_encoder.py

@@ -617,7 +617,7 @@ class UnicodeToLatexEncoder(object):
         res = rulecallable(s, p.pos)
         if res is None:
             return None
-        (consumed, repl) = res
+        consumed, repl = res
         self._apply_replacement(p, repl, consumed, rule)
         return True
 

great_ai/utilities/external/pylatexenc/latexwalker/__init__.py

@@ -477,7 +477,7 @@ class LatexNode(object):
         parsing_state=None,
         pos=None,
         len=None,
-        **kwargs
+        **kwargs,
     ):
 
         # Important: subclasses must specify a list of fields they set in the
@@ -607,7 +607,7 @@ class LatexGroupNode(LatexNode):
                 "nodelist",
                 "delimiters",
             ),
-            **kwargs
+            **kwargs,
         )
         self.nodelist = nodelist
         self.delimiters = delimiters
@@ -639,7 +639,7 @@ class LatexCommentNode(LatexNode):
                 "comment",
                 "comment_post_space",
             ),
-            **kwargs
+            **kwargs,
         )
 
         self.comment = comment
@@ -715,7 +715,7 @@ class LatexMacroNode(LatexNode):
         super(LatexMacroNode, self).__init__(
             _fields=("macroname", "nodeargd", "macro_post_space"),
             _redundant_fields=("nodeoptarg", "nodeargs"),
-            **kwargs
+            **kwargs,
         )
 
         self.macroname = macroname
@@ -801,7 +801,7 @@ class LatexEnvironmentNode(LatexNode):
                 "optargs",
                 "args",
             ),
-            **kwargs
+            **kwargs,
         )
 
         self.environmentname = environmentname
@@ -1264,7 +1264,7 @@ class LatexWalker(object):
         environments=True,
         keep_inline_math=None,
         parsing_state=None,
-        **kwargs
+        **kwargs,
     ):
         r"""
         Parses the latex content given to the constructor (and stored in `self.s`),
@@ -1628,7 +1628,7 @@ class LatexWalker(object):
                             r"Expected expression, got \end",
                             self.s,
                             pos,
-                            **self.pos_to_lineno_colno(pos, as_dict=True)
+                            **self.pos_to_lineno_colno(pos, as_dict=True),
                         )
                     else:
                         return self._mknodeposlen(
@@ -1674,7 +1674,7 @@ class LatexWalker(object):
                         "Expected expression, got closing brace '{}'".format(tok.arg),
                         self.s,
                         pos,
-                        **self.pos_to_lineno_colno(pos, as_dict=True)
+                        **self.pos_to_lineno_colno(pos, as_dict=True),
                     )
                 return self._mknodeposlen(
                     LatexCharsNode,
@@ -1717,7 +1717,7 @@ class LatexWalker(object):
                 "Unknown token type: {}".format(tok.tok),
                 self.s,
                 pos,
-                **self.pos_to_lineno_colno(pos, as_dict=True)
+                **self.pos_to_lineno_colno(pos, as_dict=True),
             )
 
     def get_latex_maybe_optional_arg(self, pos, parsing_state=None):
@@ -1821,10 +1821,10 @@ class LatexWalker(object):
                 pos=pos,
                 msg="get_latex_braced_group: not an opening brace/bracket: %s"
                 % (self.s[pos]),
-                **self.pos_to_lineno_colno(pos, as_dict=True)
+                **self.pos_to_lineno_colno(pos, as_dict=True),
             )
 
-        (nodelist, npos, nlen) = self.get_latex_nodes(
+        nodelist, npos, nlen = self.get_latex_nodes(
             firsttok.pos + firsttok.len,
             stop_upon_closing_brace=(brace_type, closing_brace),
             parsing_state=parsing_state,
@@ -1883,12 +1883,14 @@ class LatexWalker(object):
                 pos=pos,
                 msg=r"get_latex_environment: expected \begin{%s}: %s"
                 % (
+                    (
                         environmentname
                         if environmentname is not None
-                    else "<environment name>",
+                        else "<environment name>"
+                    ),
                     firsttok.arg,
                 ),
-                **self.pos_to_lineno_colno(pos, as_dict=True)
+                **self.pos_to_lineno_colno(pos, as_dict=True),
             )
         if environmentname is None:
             environmentname = firsttok.arg
@@ -1915,9 +1917,9 @@ class LatexWalker(object):
             argsresult = (None, pos, 0, {})
 
         if len(argsresult) == 4:
-            (argd, apos, alen, adic) = argsresult
+            argd, apos, alen, adic = argsresult
         else:
-            (argd, apos, alen) = argsresult
+            argd, apos, alen = argsresult
             adic = {}
 
         pos = apos + alen
@@ -1930,7 +1932,7 @@ class LatexWalker(object):
                 math_mode_delimiter="{" + environmentname + "}",
             )
 
-        (nodelist, npos, nlen) = self.get_latex_nodes(
+        nodelist, npos, nlen = self.get_latex_nodes(
             pos,
             stop_upon_end_environment=environmentname,
             parsing_state=parsing_state_inner,
@@ -1975,7 +1977,7 @@ class LatexWalker(object):
                 s=self.s,
                 pos=tok.pos,
                 msg="End of input while parsing {}".format(what),
-                **self.pos_to_lineno_colno(tok.pos, as_dict=True)
+                **self.pos_to_lineno_colno(tok.pos, as_dict=True),
             )
 
         if getattr(e, "pos", None) is not None and e.lineno is None and e.colno is None:
@@ -2227,7 +2229,7 @@ class LatexWalker(object):
                         s=self.s,
                         pos=tok.pos,
                         msg="Unexpected mismatching closing brace: '%s'" % (tok.arg),
-                        **self.pos_to_lineno_colno(tok.pos, as_dict=True)
+                        **self.pos_to_lineno_colno(tok.pos, as_dict=True),
                     )
                 return True
 
@@ -2239,7 +2241,7 @@ class LatexWalker(object):
                         s=self.s,
                         pos=tok.pos,
                         msg=("Unexpected closing environment: '{}'".format(tok.arg)),
-                        **self.pos_to_lineno_colno(tok.pos, as_dict=True)
+                        **self.pos_to_lineno_colno(tok.pos, as_dict=True),
                     )
                 elif tok.arg != stop_upon_end_environment:
                     # p.push_lastchars(tok_to_pos_and_chars_from_ppos(tok))
@@ -2252,7 +2254,7 @@ class LatexWalker(object):
                                 tok.arg, stop_upon_end_environment
                             )
                         ),
-                        **self.pos_to_lineno_colno(tok.pos, as_dict=True)
+                        **self.pos_to_lineno_colno(tok.pos, as_dict=True),
                     )
                 return True
 
@@ -2276,7 +2278,7 @@ class LatexWalker(object):
                                 tok.arg,
                                 stop_upon_closing_mathmode,
                             ),
-                            **self.pos_to_lineno_colno(tok.pos, as_dict=True)
+                            **self.pos_to_lineno_colno(tok.pos, as_dict=True),
                         )
                     # all ok, this is a new math mode opening.  Keep an assert
                     # in case we forget to include some math-mode delimiters in
@@ -2291,7 +2293,7 @@ class LatexWalker(object):
                         s=self.s,
                         pos=tok.pos,
                         msg="Unexpected closing math mode: '{}'".format(tok.arg),
-                        **self.pos_to_lineno_colno(tok.pos, as_dict=True)
+                        **self.pos_to_lineno_colno(tok.pos, as_dict=True),
                     )
 
                 # we have encountered a new math inline, parse the math expression
@@ -2306,7 +2308,7 @@ class LatexWalker(object):
                 )
 
                 try:
-                    (mathinline_nodelist, mpos, mlen) = self.get_latex_nodes(
+                    mathinline_nodelist, mpos, mlen = self.get_latex_nodes(
                         p.pos,
                         stop_upon_closing_mathmode=corresponding_closing_mathmode,
                         parsing_state=parsing_state_inner,
@@ -2316,7 +2318,7 @@ class LatexWalker(object):
                         _maketuple(
                             'math mode "{}"'.format(tok.arg),
                             tok.pos,
-                            *self.pos_to_lineno_colno(tok.pos)
+                            *self.pos_to_lineno_colno(tok.pos),
                         )
                     )
                     raise
@@ -2354,7 +2356,7 @@ class LatexWalker(object):
             if tok.tok == "brace_open":
                 # another braced group to read.
                 try:
-                    (groupnode, bpos, blen) = self.get_latex_braced_group(
+                    groupnode, bpos, blen = self.get_latex_braced_group(
                         tok.pos, brace_type=tok.arg, parsing_state=p.parsing_state
                     )
                 # except LatexWalkerEndOfStream as e:
@@ -2376,7 +2378,7 @@ class LatexWalker(object):
             if tok.tok == "begin_environment":
                 # an environment to read.
                 try:
-                    (envnode, epos, elen) = self.get_latex_environment(
+                    envnode, epos, elen = self.get_latex_environment(
                         tok.pos, environmentname=tok.arg, parsing_state=p.parsing_state
                     )
                 except LatexWalkerParseError as e:
@@ -2384,7 +2386,7 @@ class LatexWalker(object):
                         _maketuple(
                             'begin environment "{}"'.format(tok.arg),
                             tok.pos,
-                            *self.pos_to_lineno_colno(tok.pos)
+                            *self.pos_to_lineno_colno(tok.pos),
                         )
                     )
                     raise
@@ -2415,9 +2417,9 @@ class LatexWalker(object):
                     margsresult = (None, tok.pos + tok.len, 0, {})
 
                 if len(margsresult) == 4:
-                    (nodeargd, mapos, malen, mdic) = margsresult
+                    nodeargd, mapos, malen, mdic = margsresult
                 else:
-                    (nodeargd, mapos, malen) = margsresult
+                    nodeargd, mapos, malen = margsresult
                     mdic = {}
 
                 p.pos = mapos + malen
@@ -2473,9 +2475,9 @@ class LatexWalker(object):
                 if res is not None:
                     # specials expects arguments, read them
                     if len(res) == 4:
-                        (nodeargd, mapos, malen, spdic) = res
+                        nodeargd, mapos, malen, spdic = res
                     else:
-                        (nodeargd, mapos, malen) = res
+                        nodeargd, mapos, malen = res
                         spdic = {}
 
                     p.pos = mapos + malen
@@ -2505,7 +2507,7 @@ class LatexWalker(object):
                 s=self.s,
                 pos=p.pos,
                 msg="Unknown token: {!r}".format(tok),
-                **self.pos_to_lineno_colno(p.pos, as_dict=True)
+                **self.pos_to_lineno_colno(p.pos, as_dict=True),
             )
 
         while True:
@@ -2531,7 +2533,7 @@ class LatexWalker(object):
                         msg="Unexpected end of stream, was expecting {}".format(
                             expecting
                         ),
-                        **self.pos_to_lineno_colno(len(self.s), as_dict=True)
+                        **self.pos_to_lineno_colno(len(self.s), as_dict=True),
                     )
                     if self.tolerant_parsing:
                         r_endnow = True
@@ -2651,7 +2653,7 @@ def get_latex_nodes(
     stop_upon_closing_brace=None,
     stop_upon_end_environment=None,
     stop_upon_closing_mathmode=None,
-    **parse_flags
+    **parse_flags,
 ):
     """
     Parses latex content `s`.

great_ai/utilities/external/pylatexenc/latexwalker/__main__.py

@@ -184,7 +184,7 @@ def main(argv=None):
         latex, tolerant_parsing=args.tolerant_parsing, strict_braces=args.strict_braces
     )
 
-    (nodelist, pos, len_) = latexwalker.get_latex_nodes()
+    nodelist, pos, len_ = latexwalker.get_latex_nodes()
 
     if args.output_format == "human":
         print("\n--- NODES ---\n")

great_ai/utilities/external/pylatexenc/macrospec/__init__.py

@@ -34,7 +34,6 @@ macros and environments, specifying how they should be parsed by
    `pylatexenc 2.0`.
 """
 
-
 import sys
 
 if sys.version_info.major > 2:
@@ -154,7 +153,7 @@ class EnvironmentSpec(object):
         environmentname,
         args_parser=MacroStandardArgsParser(),
         is_math_mode=False,
-        **kwargs
+        **kwargs,
     ):
         super(EnvironmentSpec, self).__init__(**kwargs)
         self.environmentname = environmentname
@@ -776,9 +775,9 @@ class LatexContextDb(object):
             new_context.add_context_category(
                 cat,
                 macros=self.d[cat]["macros"].values() if keep_macros else [],
-                environments=self.d[cat]["environments"].values()
+                environments=(
-                if keep_environments
+                    self.d[cat]["environments"].values() if keep_environments else []
-                else [],
+                ),
                 specials=self.d[cat]["specials"].values() if keep_specials else [],
             )
 

great_ai/utilities/external/pylatexenc/macrospec/_argparsers.py

@@ -296,7 +296,7 @@ class MacroStandardArgsParser(object):
 
         for j, argt in enumerate(self.argspec):
             if argt == "{":
-                (node, np, nl) = w.get_latex_expression(
+                node, np, nl = w.get_latex_expression(
                     p, strict_braces=False, parsing_state=get_inner_parsing_state(j)
                 )
                 p = np + nl
@@ -315,7 +315,7 @@ class MacroStandardArgsParser(object):
                 if optarginfotuple is None:
                     argnlist.append(None)
                     continue
-                (node, np, nl) = optarginfotuple
+                node, np, nl = optarginfotuple
                 p = np + nl
                 argnlist.append(node)
 

great_ai/utilities/parallel_map/parallel_map.py

@@ -3,6 +3,7 @@ from typing import (
     Awaitable,
     Callable,
     Iterable,
+    Literal,
     Optional,
     Sequence,
     TypeVar,
@@ -11,7 +12,6 @@ from typing import (
 )
 
 import dill
-from typing_extensions import Literal  # <= Python 3.7
 
 from .get_config import get_config
 from .manage_communication import manage_communication
@@ -31,8 +31,7 @@ def parallel_map(
     chunk_size: Optional[int] = ...,
     concurrency: Optional[int] = ...,
     unordered: bool = ...,
-) -> Iterable[Optional[V]]:
+) -> Iterable[Optional[V]]: ...
-    ...
 
 
 @overload
@@ -44,8 +43,7 @@ def parallel_map(
     ignore_exceptions: Literal[True],
     concurrency: Optional[int] = ...,
     unordered: bool = ...,
-) -> Iterable[Optional[V]]:
+) -> Iterable[Optional[V]]: ...
-    ...
 
 
 @overload
@@ -57,8 +55,7 @@ def parallel_map(
     ignore_exceptions: Literal[False] = ...,
     concurrency: Optional[int] = ...,
     unordered: bool = ...,
-) -> Iterable[V]:
+) -> Iterable[V]: ...
-    ...
 
 
 @overload
@@ -70,8 +67,7 @@ def parallel_map(
     ignore_exceptions: Literal[False] = ...,
     concurrency: Optional[int] = ...,
     unordered: bool = ...,
-) -> Iterable[V]:
+) -> Iterable[V]: ...
-    ...
 
 
 def parallel_map(
@@ -147,7 +143,7 @@ def parallel_map(
     serialized_map_function = dill.dumps(func, byref=True, recurse=False)
 
     processes = [
-        ctx.Process(  # type: ignore
+        ctx.Process(
             name=f"parallel_map_{config.function_name}_{i}",
             target=mapper_function,
             daemon=True,

great_ai/utilities/parallel_map/simple_parallel_map.py

@@ -1,6 +1,6 @@
 from typing import Awaitable, Callable, List, Optional, Sequence, TypeVar, Union
 
-from tqdm.cli import tqdm
+from tqdm import tqdm
 
 from .parallel_map import parallel_map
 

great_ai/utilities/parallel_map/threaded_parallel_map.py

@@ -4,6 +4,7 @@ from typing import (
     Awaitable,
     Callable,
     Iterable,
+    Literal,
     Optional,
     Sequence,
     TypeVar,
@@ -11,8 +12,6 @@ from typing import (
     overload,
 )
 
-from typing_extensions import Literal  # <= Python 3.7
-
 from .get_config import get_config
 from .manage_communication import manage_communication
 from .mapper_function import mapper_function
@@ -30,8 +29,7 @@ def threaded_parallel_map(
     chunk_size: Optional[int] = ...,
     concurrency: Optional[int] = ...,
     unordered: bool = ...,
-) -> Iterable[Optional[V]]:
+) -> Iterable[Optional[V]]: ...
-    ...
 
 
 @overload
@@ -43,8 +41,7 @@ def threaded_parallel_map(
     ignore_exceptions: Literal[True],
     concurrency: Optional[int] = ...,
     unordered: bool = ...,
-) -> Iterable[Optional[V]]:
+) -> Iterable[Optional[V]]: ...
-    ...
 
 
 @overload
@@ -56,8 +53,7 @@ def threaded_parallel_map(
     ignore_exceptions: Literal[False] = ...,
     concurrency: Optional[int] = ...,
     unordered: bool = ...,
-) -> Iterable[V]:
+) -> Iterable[V]: ...
-    ...
 
 
 @overload
@@ -69,8 +65,7 @@ def threaded_parallel_map(
     ignore_exceptions: Literal[False] = ...,
     concurrency: Optional[int] = ...,
     unordered: bool = ...,
-) -> Iterable[V]:
+) -> Iterable[V]: ...
-    ...
 
 
 def threaded_parallel_map(

great_ai/views/operators.py

@@ -1,6 +1,4 @@
-from typing import List
+from typing import List, Literal
-
-from typing_extensions import Literal  # <= Python 3.7
 
 Operator = Literal[">=", "<=", "<", ">", "!=", "=", "contains"]
 

great_ai/views/outputs/classification_output.py

@@ -6,4 +6,4 @@ from ..hashable_base_model import HashableBaseModel
 class ClassificationOutput(HashableBaseModel):
     label: Union[str, int]
     confidence: float
-    explanation: Optional[Any]
+    explanation: Optional[Any] = None

great_ai/views/outputs/regression_output.py

@@ -5,4 +5,4 @@ from ..hashable_base_model import HashableBaseModel
 
 class RegressionOutput(HashableBaseModel):
     value: Union[int, float]
-    explanation: Optional[Any]
+    explanation: Optional[Any] = None

great_ai/views/outputs/sequence_labeling_output.py

@@ -1,6 +1,4 @@
-from typing import Any, List, Optional
+from typing import Any, List, Literal, Optional
-
-from typing_extensions import Literal  # <= Python 3.7
 
 from ..hashable_base_model import HashableBaseModel
 
@@ -9,9 +7,9 @@ class LabeledToken(HashableBaseModel):
     token: str
     tag: Literal["B", "I", "O", "E", "S"]
     confidence: float
-    explanation: Optional[Any]
+    explanation: Optional[Any] = None
 
 
 class SequenceLabelingOutput(HashableBaseModel):
     labeled_tokens: List[LabeledToken]
-    explanation: Optional[Any]
+    explanation: Optional[Any] = None

great_ai/views/query.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 from typing import List, Optional, Sequence
 
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 
 from .filter import Filter
 from .sort_by import SortBy
@@ -15,8 +15,8 @@ class Query(BaseModel):
     until: Optional[datetime] = None
     has_feedback: Optional[bool] = None
 
-    class Config:
+    model_config = ConfigDict(
-        schema_extra = {
+        json_schema_extra={
             "example": {
                 "filter": [
                     {
@@ -33,3 +33,4 @@ class Query(BaseModel):
                 "has_feedback": False,
             }
         }
+    )

great_ai/views/sort_by.py

@@ -1,5 +1,6 @@
+from typing import Literal
+
 from pydantic import BaseModel
-from typing_extensions import Literal  # <= Python 3.7
 
 
 class SortBy(BaseModel):

great_ai/views/trace.py

@@ -1,7 +1,7 @@
 from pprint import pformat
 from typing import Any, Dict, Generic, List, Optional, TypeVar
 
-from pydantic import Extra
+from pydantic import ConfigDict
 
 from .hashable_base_model import HashableBaseModel
 from .model import Model
@@ -9,7 +9,7 @@ from .model import Model
 T = TypeVar("T")
 
 
-class Trace(Generic[T], HashableBaseModel):
+class Trace(HashableBaseModel, Generic[T]):
     """Universal structure for storing prediction traces and training data.
 
     Attributes:
@@ -32,13 +32,12 @@ class Trace(Generic[T], HashableBaseModel):
     original_execution_time_ms: float
     logged_values: Dict[str, Any]
     models: List[Model]
-    exception: Optional[str]
+    exception: Optional[str] = None
-    output: Optional[T]
+    output: Optional[T] = None
     feedback: Any = None
     tags: List[str]
 
-    class Config:
+    model_config = ConfigDict(extra="ignore")
-        extra = Extra.ignore
 
     @property
     def input(self) -> Any:
@@ -79,7 +78,7 @@ class Trace(Generic[T], HashableBaseModel):
     def to_flat_dict(self, include_original: bool = True) -> Dict[str, Any]:
         return {
             **(
-                self.dict()
+                self.model_dump()
                 if include_original
                 else {
                     "trace_id": self.trace_id,
@@ -88,9 +87,11 @@ class Trace(Generic[T], HashableBaseModel):
                 }
             ),
             **{
-                k: v
+                k: (
+                    v
                     if (isinstance(v, float) or isinstance(v, int))
                     else pformat(v, indent=2, compact=True)
+                )
                 for k, v in self.logged_values.items()
             },
             "models_flat": self.models_flat,
@@ -101,6 +102,7 @@ class Trace(Generic[T], HashableBaseModel):
         }
 
     def __repr__(self) -> str:
-        return f"""Trace[{type(self.output).__name__}]({
+        formatted = pformat(self.model_dump(), indent=2, compact=True).replace(
-            pformat(self.dict(), indent=2, compact=True).replace('{ ', '{', 1)
+            "{ ", "{", 1
-        })"""
+        )
+        return f"Trace[{type(self.output).__name__}]({formatted})"

pyproject.toml

@@ -21,51 +21,58 @@ classifiers = [
     "Natural Language :: English",
 ]
 keywords = ["SE4ML", "MLOps", "AI engineering", "general", "robust", "end-to-end", "automated", "trustworthy", "ai", "deployment"]
-requires-python = ">= 3.7"
+requires-python = ">= 3.10"
 dependencies = [
-    "scikit-learn",
+    "scikit-learn >= 1.3, < 2",
-    "matplotlib",
+    "matplotlib >= 3.7, < 4",
-    "numpy",
+    "numpy >= 1.24, < 3",
-    "nbconvert",
+    "nbconvert >= 7.0, < 8",
-    "ipython",
+    "ipython >= 8.0, < 10",
-    "unidecode >= 1.3.0",
+    "unidecode >= 1.3.0, < 2",
-    "syntok >= 1.4.0",
+    "syntok >= 1.4.0, < 2",
-    "langcodes[data] >= 3.3.0",
+    "langcodes[data] >= 3.3.0, < 4",
-    "langdetect >= 1.0.9",
+    "langdetect >= 1.0.9, < 2",
-    "tinydb >= 4.7.0",
+    "tinydb >= 4.7.0, < 5",
-    "boto3 >= 1.23.0",
+    "boto3 >= 1.34, < 2",
-    "plotly >= 5.8.0",
+    "plotly >= 5.20, < 7",
-    "pandas",
+    "pandas >= 2.0, < 4",
-    "dash >= 2.4.0",
+    "dash >= 2.16, < 5",
-    "fastapi >= 0.70.0",
+    "fastapi >= 0.110, < 1",
-    "uvicorn[standard] >= 0.18.0",
+    "uvicorn[standard] >= 0.27, < 1",
-    "watchdog >= 2.1.0",
+    "a2wsgi >= 1.9, < 2",
-    "typeguard >= 2.10.0",
+    "watchdog >= 3.0, < 7",
-    "pymongo >= 4.0.0",
+    "typeguard >= 4.0, < 5",
-    "dill >= 0.3.5.0",
+    "pydantic >= 2.5, < 3",
-    "tqdm",
+    "pymongo >= 4.6, < 5",
-    "httpx >= 0.20.0",
+    "dill >= 0.3.6, < 1",
+    "tqdm >= 4.64, < 5",
+    "httpx >= 0.24, < 1",
 ]
 
 [project.optional-dependencies]
 dev = [
-    "flit",
+    "flit >= 3.9, < 4",
-    "mkdocs",
+    "mkdocs >= 1.5, < 2",
-    "mkdocstrings[python]",
+    "mkdocstrings[python] >= 0.24, < 1",
-    "mkdocs-material",
+    "mkdocs-material >= 9.5, < 10",
-    "mkdocs-jupyter",
+    "mkdocs-jupyter >= 0.24, < 1",
-    "mkdocs-git-revision-date-localized-plugin",
+    "mkdocs-git-revision-date-localized-plugin >= 1.2, < 2",
-    "autoflake",
+    "autoflake >= 2.0, < 3",
-    "isort",
+    "isort >= 5.12, < 7",
-    "black[jupyter]",
+    "black[jupyter] >= 25.1, < 26",
-    "mypy",
+    "mypy >= 1.8, < 2",
-    "flake8",
+    "flake8 >= 7.0, < 8",
-    "tox",
+    "tox >= 4.0, < 5",
-    "pytest",
+    "pytest >= 7.4, < 9",
-    "pytest-cov",
+    "pytest-cov >= 4.1, < 8",
-    "pytest-asyncio",
+    "pytest-asyncio >= 0.23, < 2",
 ]
 
+[tool.pytest.ini_options]
+# `...` in doctests (e.g. third-party exception messages that drift between
+# dependency versions) is treated as a wildcard rather than a literal.
+doctest_optionflags = ["ELLIPSIS"]
+
 [project.urls]
 Documentation = "https://great-ai.scoutinscience.com"
 GitHub = "https://github.com/schmelczer/great-ai"

scripts/check-python.sh

@@ -3,7 +3,7 @@
 set -e
 
 echo "Installing dependencies if necessary"
-python3 -m pip install --upgrade autoflake isort black[jupyter] mypy flake8
+python3 -m pip install autoflake isort black[jupyter] mypy flake8
 
 for dir in "$@"; do
     echo "Checking $dir"
@@ -16,10 +16,10 @@ for dir in "$@"; do
 
     if find . -name "*.py" 2>/dev/null | grep -q .; then
         yes | python3 -m mypy . --install-types > /dev/null || true
-        python3 -m mypy --namespace-packages --ignore-missing-imports --install-types --non-interactive --disallow-untyped-defs --disallow-incomplete-defs --follow-imports=silent --exclude=external/ --exclude=/build/ --pretty .
+        python3 -m mypy --namespace-packages --ignore-missing-imports --install-types --non-interactive --disallow-untyped-defs --disallow-incomplete-defs --follow-imports=silent --exclude=external/ --exclude=/build/ --exclude='(^|/)__main__\.py$' --pretty .
     fi
 
-    python3 -m flake8 . --count --show-source --statistics --exclude=__init__.py,.env,external --ignore=E501,E402,F821,W503,E722,E203
+    python3 -m flake8 . --count --show-source --statistics --exclude=__init__.py,.env,external --extend-ignore=E501,E402,F821,W503,E722,E203,E704
 
     cd -
 done

tests/external/test_internals.py

@@ -24,7 +24,8 @@ class Wrapped:
 
 
 @pytest.mark.asyncio
-async def test_done_callback_cancelled(loop):
+async def test_done_callback_cancelled():
+    loop = asyncio.get_running_loop()
     task = loop.create_future()
     fut = loop.create_future()
 
@@ -38,7 +39,8 @@ async def test_done_callback_cancelled(loop):
 
 
 @pytest.mark.asyncio
-async def test_done_callback_exception(loop):
+async def test_done_callback_exception():
+    loop = asyncio.get_running_loop()
     task = loop.create_future()
     fut = loop.create_future()
 
@@ -60,7 +62,8 @@ async def test_done_callback_exception(loop):
 
 
 @pytest.mark.asyncio
-async def test_done_callback(loop):
+async def test_done_callback():
+    loop = asyncio.get_running_loop()
     task = loop.create_future()
     fut = loop.create_future()
 
@@ -228,7 +231,8 @@ def test_close(loop):
 
 
 @pytest.mark.asyncio
-async def test_wait_closed(loop):
+async def test_wait_closed():
+    loop = asyncio.get_running_loop()
     wrapped = Wrapped()
     wrapped.tasks = set()
 
@@ -238,7 +242,7 @@ async def test_wait_closed(loop):
             return_exceptions=True,
         )
         assert ret == []
-        assert mocked.called_once()
+        mocked.assert_called_once()
 
     asyncio.set_event_loop(loop)
     with mock.patch("great_ai.external.async_lru._close_waited") as mocked:
@@ -247,7 +251,7 @@ async def test_wait_closed(loop):
             return_exceptions=True,
         )
         assert ret == []
-        assert mocked.called_once()
+        mocked.assert_called_once()
     asyncio.set_event_loop(None)
 
     fut = loop.create_future()
@@ -259,7 +263,7 @@ async def test_wait_closed(loop):
             return_exceptions=True,
         )
         assert ret == [None]
-        assert mocked.called_once()
+        mocked.assert_called_once()
 
     exc = ZeroDivisionError()
     fut = loop.create_future()
@@ -271,7 +275,7 @@ async def test_wait_closed(loop):
             return_exceptions=True,
         )
         assert ret == [exc]
-        assert mocked.called_once()
+        mocked.assert_called_once()
 
     fut = loop.create_future()
     fut.set_exception(ZeroDivisionError)
@@ -282,17 +286,21 @@ async def test_wait_closed(loop):
                 wrapped,
                 return_exceptions=False,
             )
-        assert mocked.called_once()
+        # the awaited result raised before _wait_closed could flush its done
+        # callback, so yield once to let the scheduled _close_waited run
+        await asyncio.sleep(0)
+        mocked.assert_called_once()
 
 
 def test_close_waited():
     wrapped = Wrapped()
-    wrapped.cache_clear = partial(_cache_clear, wrapped)
+    # _close_waited calls wrapped.cache_clear(); mock that directly (patching the
+    # module-level _cache_clear would not affect the already-bound partial).
+    wrapped.cache_clear = mock.Mock()
 
-    with mock.patch("great_ai.external.async_lru._cache_clear") as mocked:
     _close_waited(wrapped, None)
 
-        assert mocked.called_once()
+    wrapped.cache_clear.assert_called_once()
 
 
 def test_cache_info():
@@ -339,7 +347,7 @@ def test_cache_hit():
     with mock.patch("great_ai.external.async_lru.__cache_touch") as mocked:
         _cache_hit(wrapped, 1)
 
-        assert mocked.called_once()
+        mocked.assert_called_once()
 
     assert wrapped.hits == 2
 
@@ -357,7 +365,7 @@ def test_cache_miss():
     with mock.patch("great_ai.external.async_lru.__cache_touch") as mocked:
         _cache_miss(wrapped, 1)
 
-        assert mocked.called_once()
+        mocked.assert_called_once()
 
     assert wrapped.misses == 2
 

tests/utilities/test_parallel_map.py

@@ -23,9 +23,7 @@ def test_with_iterable() -> None:
 
     expected = [v**3 for v in range(10)]
 
-    assert (
+    assert list(parallel_map(lambda x: x**3, my_generator(), chunk_size=1)) == expected
-        list(parallel_map(lambda x: x**3, my_generator(), chunk_size=1)) == expected
-    )
 
 
 def test_simple_case_invalid_values() -> None:
@@ -44,14 +42,10 @@ def test_this_process_exception() -> None:
         assert False
 
     with pytest.raises(AssertionError):
-        list(
+        list(parallel_map(lambda v: v**2, my_generator(), concurrency=2, chunk_size=2))
-            parallel_map(lambda v: v**2, my_generator(), concurrency=2, chunk_size=2)
-        )
 
     with pytest.raises(AssertionError):
-        list(
+        list(parallel_map(lambda v: v**2, my_generator(), concurrency=1, chunk_size=2))
-            parallel_map(lambda v: v**2, my_generator(), concurrency=1, chunk_size=2)
-        )
 
 
 def test_ignore_this_process_exception() -> None: