diff --git a/.forgejo/workflows/docker.yml b/.forgejo/workflows/docker.yml deleted file mode 100644 index 2ff9932..0000000 --- a/.forgejo/workflows/docker.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: Publish on DockerHub - -on: - push: - tags: ['*'] - workflow_dispatch: - -jobs: - publish: - runs-on: docker - steps: - - name: Checkout - uses: actions/checkout@v4 - - # The `docker` runner image is node-based and may not ship the docker CLI - # that the docker/* actions below drive; install it idempotently, as the - # sibling repos do for their Docker jobs. - - name: Ensure Docker CLI - run: | - set -eux - if ! command -v docker >/dev/null 2>&1; then - ARCH=$(uname -m) - curl -fsSL --retry 3 --retry-connrefused \ - "https://download.docker.com/linux/static/stable/${ARCH}/docker-27.5.1.tgz" \ - | tar xz --strip-components=1 -C /usr/local/bin docker/docker - fi - docker --version - - # Marketplace actions must be referenced by full URL: this instance's - # DEFAULT_ACTIONS_URL points at code.forgejo.org, so only bare `actions/*` - # names resolve to GitHub automatically. - - name: Set up QEMU - uses: https://github.com/docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: https://github.com/docker/setup-buildx-action@v3 - - - name: Log in to Docker Hub - uses: https://github.com/docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Extract metadata for the Docker image - id: meta - uses: https://github.com/docker/metadata-action@v5 - with: - images: schmelczera/great-ai - - - name: Build and push - uses: https://github.com/docker/build-push-action@v6 - with: - context: . - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - platforms: linux/amd64,linux/arm64 diff --git a/.forgejo/workflows/docs.yml b/.forgejo/workflows/docs.yml deleted file mode 100644 index 973ab3a..0000000 --- a/.forgejo/workflows/docs.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Publish documentation - -on: - push: - branches: [main] - paths: - - 'docs/**' - - 'great_ai/**' - - 'mkdocs.yaml' - - '.forgejo/workflows/docs.yml' - workflow_dispatch: - -concurrency: - group: pages - cancel-in-progress: false - -jobs: - publish: - runs-on: docker - steps: - - uses: actions/checkout@v4 - with: - # The mkdocs git-revision-date plugin reads each page's git history. - fetch-depth: 0 - - - name: Install uv - run: | - curl -LsSf https://astral.sh/uv/install.sh | sh - echo "$HOME/.local/bin" >> "$GITHUB_PATH" - - - name: Set up Python - run: uv python install 3.10 - - - name: Install dependencies - run: | - uv venv --python 3.10 - uv pip install '.[dev]' - - - name: Build documentation - run: | - . .venv/bin/activate - mkdocs build - - # The old workflow ran `mkdocs gh-deploy` (GitHub Pages); on this instance - # built sites are rsynced into the host /pages mount that nginx serves, - # via the shared ci-actions/deploy-pages action — same as photos/reconcile. - - name: Deploy to pages mount - if: github.ref == 'refs/heads/main' - uses: http://forgejo:3000/andras/ci-actions/deploy-pages@main - with: - source: site - target: great-ai diff --git a/.forgejo/workflows/publish.yml b/.forgejo/workflows/publish.yml deleted file mode 100644 index 8042684..0000000 --- a/.forgejo/workflows/publish.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Publish on PyPI - -on: - push: - tags: ['*'] - workflow_dispatch: - -jobs: - publish: - runs-on: docker - steps: - - uses: actions/checkout@v4 - - - name: Install uv - run: | - curl -LsSf https://astral.sh/uv/install.sh | sh - echo "$HOME/.local/bin" >> "$GITHUB_PATH" - - - name: Set up Python - run: uv python install 3.10 - - # Forgejo cannot use PyPI trusted publishing (OIDC), so authenticate with - # an API token, the same way the sibling repos publish their packages. - - name: Build and publish - env: - FLIT_USERNAME: __token__ - FLIT_PASSWORD: ${{ secrets.PYPI_TOKEN }} - run: uvx flit publish diff --git a/.forgejo/workflows/test.yml b/.forgejo/workflows/test.yml deleted file mode 100644 index a76c206..0000000 --- a/.forgejo/workflows/test.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: Check - -on: - push: - branches: [main, dev] - pull_request: - branches: [main, dev] - workflow_dispatch: - -concurrency: - group: ${{ gitea.workflow }}-${{ gitea.ref }} - cancel-in-progress: true - -jobs: - lint: - name: Lint, format & type checks - runs-on: docker - steps: - - uses: actions/checkout@v4 - - - name: Install uv - run: | - curl -LsSf https://astral.sh/uv/install.sh | sh - echo "$HOME/.local/bin" >> "$GITHUB_PATH" - - - name: Set up Python - run: uv python install 3.10 - - - name: Install dependencies - # --seed gives the venv its own pip, which scripts/check-python.sh shells - # out to for its linters; '.[dev]' provides the rest (mypy resolves the - # package's own imports against the installed tree). - run: | - uv venv --seed --python 3.10 - uv pip install '.[dev]' - - - name: Check code and formatting - run: | - . .venv/bin/activate - scripts/check-python.sh great_ai tests - - test: - name: Test on Python ${{ matrix.python-version }} - runs-on: docker - strategy: - fail-fast: false - matrix: - # pyproject declares >= 3.7, but uv's standalone CPython and the current - # dependency floors (scikit-learn/numpy now require >= 3.9) no longer - # build on the older interpreters, so we test the current supported set. - # The windows leg of the old GitHub matrix is dropped: this Forgejo - # instance only has a Linux `docker` runner. - python-version: ['3.10', '3.11', '3.12', '3.13'] - steps: - - uses: actions/checkout@v4 - - - name: Install uv - run: | - curl -LsSf https://astral.sh/uv/install.sh | sh - echo "$HOME/.local/bin" >> "$GITHUB_PATH" - - - name: Set up Python ${{ matrix.python-version }} - run: uv python install ${{ matrix.python-version }} - - - name: Install dependencies - run: | - uv venv --python ${{ matrix.python-version }} - uv pip install '.[dev]' - - - name: Run tests - run: | - . .venv/bin/activate - pytest --doctest-modules --asyncio-mode=strict diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..1230149 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..46492bc --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,38 @@ +name: analyse with CodeQL + +on: + push: + branches: + - main + pull_request: + branches: + - main + schedule: + - cron: '45 13 * * 5' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: + - python + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml new file mode 100644 index 0000000..44cee1c --- /dev/null +++ b/.github/workflows/docker.yaml @@ -0,0 +1,39 @@ +name: publish on DockerHub + +on: + push: + tags: + - '*' + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Log in to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Checkout + uses: actions/checkout@v3 + + - name: Extract metadata for the Docker image + id: meta + uses: docker/metadata-action@v4 + with: + images: schmelczera/great-ai + + - name: Build and push + uses: docker/build-push-action@v4 + with: + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/amd64,linux/arm64 diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml new file mode 100644 index 0000000..1f26dd0 --- /dev/null +++ b/.github/workflows/docs.yaml @@ -0,0 +1,27 @@ +name: publish documentation + +on: + workflow_dispatch: + push: + branches: + - main + - dev + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Python 3.9 + uses: actions/setup-python@v4 + with: + python-version: 3.9 + + - name: Install dependencies + run: pip install --upgrade './[dev]' + + - name: Build documentation + run: mkdocs gh-deploy diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml new file mode 100644 index 0000000..aa13970 --- /dev/null +++ b/.github/workflows/publish.yaml @@ -0,0 +1,26 @@ +name: publish on PyPI + +on: + push: + tags: + - '*' + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Python 3.9 + uses: actions/setup-python@v4 + with: + python-version: 3.9 + + - name: Install Flit + run: pip install --upgrade flit --user + + - name: Build and publish + run: flit publish --setup-py + env: + FLIT_USERNAME: __token__ + FLIT_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..dab6f73 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,96 @@ +name: tests + +on: + workflow_dispatch: + push: + branches: + - main + - dev + +jobs: + test: + name: Build and test on ${{ matrix.operating-system }} with Python ${{ matrix.python-version }} + + strategy: + matrix: + python-version: ["3.7", "3.8", "3.9", "3.10"] + operating-system: [windows-latest, ubuntu-latest] + + runs-on: ${{ matrix.operating-system }} + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip + pip install --upgrade './[dev]' + + - name: Check code and formatting + run: scripts/check-python.sh great_ai tests + + - name: Run tests + run: python3 -m pytest --doctest-modules --cov=. --cov-report=xml --junit-xml pytest.xml --asyncio-mode=strict || true + + - name: Upload results + if: always() + uses: actions/upload-artifact@v3 + with: + name: Unit test coverage (Python ${{ matrix.python-version }} - ${{ matrix.operating-system }}) + path: "*.xml" + + sonar: + name: Analyse Python project with SonarQube + needs: test + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade . + rm -rf build + + - name: Download test results + uses: actions/download-artifact@v3 + with: + path: artifacts + + - uses: sonarsource/sonarqube-scan-action@master + env: + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} + with: + args: > + -Dsonar.projectKey=great-ai + -Dsonar.login=${{ secrets.SONAR_TOKEN }} + -Dsonar.verbose=true + -Dsonar.python.coverage.reportPaths=artifacts/*/coverage.xml + -Dsonar.coverage.exclusions=**/external/**/* + -Dsonar.exclusions=**/external/**/*,artifacts/**/* + + publish-test-results: + name: Publish unit test results + needs: test + runs-on: ubuntu-latest + if: always() + steps: + - name: Download artifacts + uses: actions/download-artifact@v3 + with: + path: artifacts + + - name: Publish results + uses: EnricoMi/publish-unit-test-result-action@v2 + with: + files: artifacts/*/pytest.xml diff --git a/docs/thesis/figures/design-cycle2.drawio.png b/docs/thesis/figures/design-cycle2.drawio.png deleted file mode 100644 index bd6cbf5..0000000 Binary files a/docs/thesis/figures/design-cycle2.drawio.png and /dev/null differ diff --git a/great_ai/context.py b/great_ai/context.py index 24412a5..29327dc 100644 --- a/great_ai/context.py +++ b/great_ai/context.py @@ -4,7 +4,7 @@ from logging import DEBUG, Logger from pathlib import Path from typing import Any, Dict, Optional, Type, Union, cast -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel from great_ai import __version__ @@ -34,7 +34,8 @@ class Context(BaseModel): dashboard_table_size: int route_config: RouteConfig - model_config = ConfigDict(arbitrary_types_allowed=True) + class Config: + arbitrary_types_allowed = True def to_flat_dict(self) -> Dict[str, Any]: return { @@ -136,11 +137,9 @@ def configure( ), is_production=is_production, logger=logger, - should_log_exception_stack=( - not is_production - if should_log_exception_stack is None - else should_log_exception_stack - ), + should_log_exception_stack=not is_production + if should_log_exception_stack is None + else should_log_exception_stack, prediction_cache_size=prediction_cache_size, dashboard_table_size=dashboard_table_size, route_config=route_config, diff --git a/great_ai/deploy/great_ai.py b/great_ai/deploy/great_ai.py index ac14590..9516169 100644 --- a/great_ai/deploy/great_ai.py +++ b/great_ai/deploy/great_ai.py @@ -6,7 +6,6 @@ from typing import ( Callable, Generic, List, - Literal, Optional, Sequence, Tuple, @@ -17,7 +16,8 @@ from typing import ( ) from fastapi import FastAPI -from tqdm import tqdm +from tqdm.cli import tqdm +from typing_extensions import Literal # <= Python 3.7 from ..constants import DASHBOARD_PATH from ..context import get_context @@ -103,13 +103,15 @@ class GreatAI(Generic[T, V]): # Overloaded function signatures 1 and 2 overlap with incompatible return types # https://github.com/python/mypy/issues/12759 func: Callable[..., Awaitable[V]], - ) -> "GreatAI[Awaitable[Trace[V]], V]": ... + ) -> "GreatAI[Awaitable[Trace[V]], V]": + ... @overload @staticmethod def create( func: Callable[..., V], - ) -> "GreatAI[Trace[V], V]": ... + ) -> "GreatAI[Trace[V], V]": + ... @staticmethod def create( @@ -144,7 +146,7 @@ class GreatAI(Generic[T, V]): >>> my_function('3').output Traceback (most recent call last): ... - TypeError: argument a is not of the expected type: ... + TypeError: type of a must be int; got str instead Args: func: The prediction function that needs to be decorated. @@ -168,7 +170,8 @@ class GreatAI(Generic[T, V]): concurrency: Optional[int] = None, unpack_arguments: Literal[True], do_not_persist_traces: bool = ..., - ) -> List[Trace[V]]: ... + ) -> List[Trace[V]]: + ... @overload def process_batch( @@ -178,7 +181,8 @@ class GreatAI(Generic[T, V]): concurrency: Optional[int] = None, unpack_arguments: Literal[False] = ..., do_not_persist_traces: bool = ..., - ) -> List[Trace[V]]: ... + ) -> List[Trace[V]]: + ... def process_batch( self, @@ -230,24 +234,22 @@ class GreatAI(Generic[T, V]): ), ) - # inner/inner_async return the class' T (bound to Trace | Awaitable[Trace]); - # in this method T resolves to Trace[V], but that is not provable to mypy, so - # cast to the concrete shape parallel_map expects. - map_function = cast( - Callable[[Any], Union[Trace[V], Awaitable[Trace[V]]]], - inner_async if get_function_metadata_store(self).is_asynchronous else inner, - ) - return list( tqdm( - parallel_map(map_function, batch, concurrency=concurrency), + parallel_map( + inner_async + if get_function_metadata_store(self).is_asynchronous + else inner, + batch, + concurrency=concurrency, + ), total=len(batch), ) ) @staticmethod def _get_cached_traced_function( - func: Callable[..., Union[V, Awaitable[V]]], + func: Callable[..., Union[V, Awaitable[V]]] ) -> Callable[..., T]: @lru_cache(maxsize=get_context().prediction_cache_size) def func_in_tracing_context_sync( diff --git a/great_ai/deploy/routes/bootstrap_dashboard.py b/great_ai/deploy/routes/bootstrap_dashboard.py index 6bafc5f..2989c95 100644 --- a/great_ai/deploy/routes/bootstrap_dashboard.py +++ b/great_ai/deploy/routes/bootstrap_dashboard.py @@ -1,7 +1,7 @@ from pathlib import Path -from a2wsgi import WSGIMiddleware from fastapi import FastAPI +from fastapi.middleware.wsgi import WSGIMiddleware from fastapi.responses import FileResponse, RedirectResponse from fastapi.staticfiles import StaticFiles @@ -19,10 +19,7 @@ def bootstrap_dashboard(app: FastAPI, function_name: str, documentation: str) -> def get_favicon() -> FileResponse: return FileResponse(PATH / "dashboard/assets/favicon.ico") - # a2wsgi types its WSGI input (Flask) and ASGI output more strictly than - # Starlette's loose WSGIApp/ASGIApp aliases, so mypy flags the mount despite - # this being the canonical, runtime-correct way to bridge the Dash app. - app.mount(DASHBOARD_PATH, WSGIMiddleware(dash_app)) # type: ignore[arg-type] + app.mount(DASHBOARD_PATH, WSGIMiddleware(dash_app)) @app.get("/", include_in_schema=False) def redirect_to_entrypoint() -> RedirectResponse: diff --git a/great_ai/deploy/routes/bootstrap_feedback_endpoints.py b/great_ai/deploy/routes/bootstrap_feedback_endpoints.py index b14df7e..8b219ee 100644 --- a/great_ai/deploy/routes/bootstrap_feedback_endpoints.py +++ b/great_ai/deploy/routes/bootstrap_feedback_endpoints.py @@ -31,7 +31,7 @@ def bootstrap_feedback_endpoints(app: FastAPI) -> None: return trace.feedback @router.delete("/", status_code=status.HTTP_204_NO_CONTENT) - def delete_feedback(trace_id: str) -> Response: + def delete_feedback(trace_id: str) -> Any: trace = get_context().tracing_database.get(trace_id) if trace is None: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND) diff --git a/great_ai/deploy/routes/bootstrap_prediction_endpoint.py b/great_ai/deploy/routes/bootstrap_prediction_endpoint.py index b3787ec..0ed6f5b 100644 --- a/great_ai/deploy/routes/bootstrap_prediction_endpoint.py +++ b/great_ai/deploy/routes/bootstrap_prediction_endpoint.py @@ -22,10 +22,10 @@ def bootstrap_prediction_endpoint( try: if inspect.iscoroutinefunction(func): return await cast(Callable[..., Awaitable[Trace]], func)( - **cast(BaseModel, input_value).model_dump() + **cast(BaseModel, input_value).dict() ) return cast(Callable[..., Trace], func)( - **cast(BaseModel, input_value).model_dump() + **cast(BaseModel, input_value).dict() ) except Exception as e: raise HTTPException( diff --git a/great_ai/deploy/routes/dashboard/create_dash_app.py b/great_ai/deploy/routes/dashboard/create_dash_app.py index e76ab98..2f4bd32 100644 --- a/great_ai/deploy/routes/dashboard/create_dash_app.py +++ b/great_ai/deploy/routes/dashboard/create_dash_app.py @@ -26,9 +26,9 @@ def create_dash_app(function_name: str, version: str, function_docs: str) -> Fla app = Dash( function_name, requests_pathname_prefix=DASHBOARD_PATH + "/", - server=Flask(__name__), # type: ignore[arg-type] + server=Flask(__name__), title=function_name, - update_title=None, # type: ignore[arg-type] + update_title=None, external_stylesheets=[ "/assets/index.css", ], @@ -64,9 +64,9 @@ def create_dash_app(function_name: str, version: str, function_docs: str) -> Fla interval = dcc.Interval( interval=2 * 1000, # in milliseconds n_intervals=0, - max_intervals=( - 1 if get_context().is_production else -1 - ), # will be incremented in production upon each successful request + max_intervals=1 + if get_context().is_production + else -1, # will be incremented in production upon each successful request ) app.layout = html.Main( @@ -159,7 +159,7 @@ def create_dash_app(function_name: str, version: str, function_docs: str) -> Fla take=page_size, conjunctive_filters=non_null_conjunctive_filters, conjunctive_tags=[ONLINE_TAG_NAME], - sort_by=[SortBy.model_validate(s) for s in sort_by], + sort_by=[SortBy.parse_obj(s) for s in sort_by], ) if non_null_conjunctive_filters: @@ -244,7 +244,7 @@ def update_charts( fig.update_layout( margin=dict(l=0, r=0, b=0, t=0, pad=0), ) - execution_time_histogram.figure = fig # type: ignore[attr-defined] + execution_time_histogram.figure = fig parallel_coords_fig = go.Figure( go.Parcoords( diff --git a/great_ai/deploy/routes/dashboard/get_traces_table.py b/great_ai/deploy/routes/dashboard/get_traces_table.py index 1550f48..bcf621e 100644 --- a/great_ai/deploy/routes/dashboard/get_traces_table.py +++ b/great_ai/deploy/routes/dashboard/get_traces_table.py @@ -1,12 +1,10 @@ -from typing import Any - from dash import dash_table from ....context import get_context -def get_traces_table() -> Any: - return dash_table.DataTable( # type: ignore[attr-defined] +def get_traces_table() -> dash_table.DataTable: + return dash_table.DataTable( page_current=0, page_size=get_context().dashboard_table_size, page_action="custom", diff --git a/great_ai/helper/freeze_arguments.py b/great_ai/helper/freeze_arguments.py index e6ff56f..c50b13c 100644 --- a/great_ai/helper/freeze_arguments.py +++ b/great_ai/helper/freeze_arguments.py @@ -1,24 +1,12 @@ import inspect -from functools import lru_cache, wraps -from typing import Any, Callable, Mapping, Sequence, Set, Type, TypeVar, Union, cast +from functools import wraps +from typing import Any, Callable, Mapping, Sequence, Set, TypeVar, Union, cast from pydantic import BaseModel F = TypeVar("F", bound=Callable) -@lru_cache(maxsize=None) -def _hashable_model_type(model_type: Type[BaseModel]) -> Type[BaseModel]: - # The subclass is cached per model type: Pydantic v2's __eq__ requires both - # operands to share a type, so creating a fresh subclass on every freeze() - # call would make even identical models compare unequal. - class HashableValue(model_type): # type: ignore - def __hash__(self) -> int: - return hash(frozenset((k, freeze(v)) for k, v in self.model_dump().items())) - - return HashableValue - - class FrozenDict(dict): def __hash__(self) -> int: # type: ignore return hash(frozenset((k, freeze(v)) for k, v in self.items())) @@ -67,6 +55,11 @@ def freeze(value: Union[Sequence[Any], Mapping[str, Any], Set[Any], BaseModel]) return FrozenSet(value) if isinstance(value, BaseModel): - return _hashable_model_type(type(value))(**value.model_dump()) + + class HashableValue(type(value)): # type: ignore + def __hash__(self) -> int: + return hash(frozenset((k, freeze(v)) for k, v in self.dict().items())) + + return HashableValue(**value.dict()) return value diff --git a/great_ai/large_file/helper/cached_property.py b/great_ai/large_file/helper/cached_property.py index d141d3c..d1b8bb0 100644 --- a/great_ai/large_file/helper/cached_property.py +++ b/great_ai/large_file/helper/cached_property.py @@ -28,9 +28,7 @@ class cached_property: ) try: cache = instance.__dict__ - except ( - AttributeError - ): # not all objects have __dict__ (e.g. class defines slots) + except AttributeError: # not all objects have __dict__ (e.g. class defines slots) msg = ( f"No '__dict__' attribute on {type(instance).__name__!r} " f"instance to cache {self.attrname!r} property." diff --git a/great_ai/large_file/large_file/large_file_base.py b/great_ai/large_file/large_file/large_file_base.py index 92fa5d9..64138c5 100644 --- a/great_ai/large_file/large_file/large_file_base.py +++ b/great_ai/large_file/large_file/large_file_base.py @@ -7,7 +7,9 @@ from abc import ABC, abstractmethod from functools import lru_cache from pathlib import Path from types import TracebackType -from typing import IO, Any, List, Literal, Optional, Type, Union, cast +from typing import IO, Any, List, Optional, Type, Union, cast + +from typing_extensions import Literal # <= Python 3.7 from great_ai.utilities import ConfigFile, get_logger diff --git a/great_ai/large_file/large_file/large_file_mongo.py b/great_ai/large_file/large_file/large_file_mongo.py index 0376ce9..3bed0a1 100644 --- a/great_ai/large_file/large_file/large_file_mongo.py +++ b/great_ai/large_file/large_file/large_file_mongo.py @@ -2,9 +2,8 @@ import re from pathlib import Path from typing import Any, List -from gridfs import DEFAULT_CHUNK_SIZE, GridFSBucket +from gridfs import DEFAULT_CHUNK_SIZE, Database, GridFSBucket from pymongo import MongoClient -from pymongo.database import Database from ...utilities import get_logger from ..helper import DownloadProgressBar, UploadProgressBar, cached_property @@ -61,6 +60,7 @@ class LargeFileMongo(LargeFileBase): ), version=int(f.name.split(MONGO_NAME_VERSION_SEPARATOR)[-1]), remote_path=(f._id, f.length), + origin="mongodb", ) for f in self._client.find( { diff --git a/great_ai/large_file/large_file/large_file_s3.py b/great_ai/large_file/large_file/large_file_s3.py index 3e32a59..119caa8 100644 --- a/great_ai/large_file/large_file/large_file_s3.py +++ b/great_ai/large_file/large_file/large_file_s3.py @@ -100,13 +100,9 @@ class LargeFileS3(LargeFileBase): Bucket=self.bucket_name, Key=remote_path, Filename=str(local_path), - Callback=( - None - if hide_progress - else DownloadProgressBar( - name=str(remote_path), size=size, logger=logger - ) - ), + Callback=None + if hide_progress + else DownloadProgressBar(name=str(remote_path), size=size, logger=logger), ) def _upload(self, local_path: Path, hide_progress: bool) -> None: @@ -117,11 +113,9 @@ class LargeFileS3(LargeFileBase): Filename=str(local_path), Bucket=self.bucket_name, Key=key, - Callback=( - None - if hide_progress - else UploadProgressBar(path=local_path, logger=logger) - ), + Callback=None + if hide_progress + else UploadProgressBar(path=local_path, logger=logger), ) def _delete_old_remote_versions(self) -> None: diff --git a/great_ai/models/use_model.py b/great_ai/models/use_model.py index e18eff8..5fcca36 100644 --- a/great_ai/models/use_model.py +++ b/great_ai/models/use_model.py @@ -1,19 +1,8 @@ from functools import wraps -from typing import ( - Any, - Callable, - Dict, - List, - Literal, - Optional, - Set, - Tuple, - TypeVar, - Union, - cast, -) +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, TypeVar, Union, cast from dill import load +from typing_extensions import Literal # <= Python 3.7 from ..context import get_context from ..helper import get_function_metadata_store diff --git a/great_ai/parameters/parameter.py b/great_ai/parameters/parameter.py index df3a7d4..272ffaf 100644 --- a/great_ai/parameters/parameter.py +++ b/great_ai/parameters/parameter.py @@ -1,7 +1,7 @@ from functools import wraps from typing import Any, Callable, Dict, TypeVar, cast -from typeguard import TypeCheckError, check_type +from typeguard import check_type from ..errors import ArgumentValidationError from ..helper import get_arguments, get_function_metadata_store @@ -32,7 +32,7 @@ def parameter( >>> my_function('3') Traceback (most recent call last): ... - TypeError: argument a is not of the expected type: ... + TypeError: type of a must be int; got str instead >>> @parameter('positive_a', validate=lambda v: v > 0) ... def my_function(positive_a: int): @@ -65,16 +65,13 @@ def parameter( expected_type = func.__annotations__.get(parameter_name) if expected_type is not None: - try: - check_type(argument, expected_type) - except TypeCheckError as error: - raise TypeError( - f"argument {parameter_name} is not of the expected type: {error}" - ) from error + check_type(parameter_name, argument, expected_type) if not validate(argument): raise ArgumentValidationError( - f"Argument {parameter_name} in {func.__name__} did not pass validation" + f"""Argument {parameter_name} in { + func.__name__ + } did not pass validation""" ) context = TracingContext.get_current_tracing_context() diff --git a/great_ai/persistence/mongodb_driver.py b/great_ai/persistence/mongodb_driver.py index 506d74d..f00a6ec 100644 --- a/great_ai/persistence/mongodb_driver.py +++ b/great_ai/persistence/mongodb_driver.py @@ -90,7 +90,7 @@ class MongoDbDriver(TracingDatabaseDriver): value = client[self.mongo_database].traces.find_one(id) if value: - value = Trace.model_validate(value) + value = Trace.parse_obj(value) return value @@ -150,7 +150,7 @@ class MongoDbDriver(TracingDatabaseDriver): ] with client[self.mongo_database].traces.find(**query) as cursor: - documents = [Trace[Any].model_validate(t) for t in cursor] + documents = [Trace[Any].parse_obj(t) for t in cursor] return documents, count def update(self, id: str, new_version: Trace) -> None: diff --git a/great_ai/persistence/parallel_tinydb_driver.py b/great_ai/persistence/parallel_tinydb_driver.py index 6e278a1..c278549 100644 --- a/great_ai/persistence/parallel_tinydb_driver.py +++ b/great_ai/persistence/parallel_tinydb_driver.py @@ -29,16 +29,16 @@ class ParallelTinyDbDriver(TracingDatabaseDriver): path_to_db = Path(DEFAULT_TRACING_DB_FILENAME) def save(self, trace: Trace) -> str: - return self._safe_execute(lambda db: db.insert(trace.model_dump())) + return self._safe_execute(lambda db: db.insert(trace.dict())) def save_batch(self, documents: List[Trace]) -> List[str]: - traces = [d.model_dump() for d in documents] + traces = [d.dict() for d in documents] return self._safe_execute(lambda db: db.insert_multiple(traces)) def get(self, id: str) -> Optional[Trace]: value = self._safe_execute(lambda db: db.get(lambda d: d["trace_id"] == id)) if value: - value = Trace.model_validate(value) + value = Trace.parse_obj(value) return value def query( @@ -51,7 +51,7 @@ class ParallelTinyDbDriver(TracingDatabaseDriver): since: Optional[datetime] = None, until: Optional[datetime] = None, has_feedback: Optional[bool] = None, - sort_by: Sequence[SortBy] = [], + sort_by: Sequence[SortBy] = [] ) -> Tuple[List[Trace], int]: def does_match(d: Dict[str, Any]) -> bool: return ( @@ -67,11 +67,7 @@ class ParallelTinyDbDriver(TracingDatabaseDriver): if not documents: return [], 0 - traces: List[Trace] = [Trace.model_validate(d) for d in documents] - # The DataFrame keeps its default 0..n-1 index, so filtering/sorting below - # preserve the row labels that index back into `traces`. This avoids - # reconstructing Traces from the lossy flattened rows. - df = pd.DataFrame([t.to_flat_dict() for t in traces]) + df = pd.DataFrame([Trace.parse_obj(d).to_flat_dict() for d in documents]) for f in conjunctive_filters: operator = f.operator.lower() @@ -96,13 +92,11 @@ class ParallelTinyDbDriver(TracingDatabaseDriver): count = len(df) result = df.iloc[skip:] if take is None else df.iloc[skip : skip + take] - return [traces[i] for i in result.index], count + return [Trace.parse_obj(trace) for _, trace in result.iterrows()], count def update(self, id: str, new_version: Trace) -> None: self._safe_execute( - lambda db: db.update( - new_version.model_dump(), lambda d: d["trace_id"] == id - ) + lambda db: db.update(new_version.dict(), lambda d: d["trace_id"] == id) ) def delete(self, id: str) -> None: diff --git a/great_ai/persistence/tracing_database_driver.py b/great_ai/persistence/tracing_database_driver.py index 3491237..810aea4 100644 --- a/great_ai/persistence/tracing_database_driver.py +++ b/great_ai/persistence/tracing_database_driver.py @@ -54,7 +54,7 @@ class TracingDatabaseDriver(ABC): until: Optional[datetime] = None, since: Optional[datetime] = None, has_feedback: Optional[bool] = None, - sort_by: Sequence[SortBy] = [], + sort_by: Sequence[SortBy] = [] ) -> Tuple[List[Trace], int]: pass diff --git a/great_ai/remote/call_remote_great_ai_async.py b/great_ai/remote/call_remote_great_ai_async.py index eac83ee..fae1c6b 100644 --- a/great_ai/remote/call_remote_great_ai_async.py +++ b/great_ai/remote/call_remote_great_ai_async.py @@ -64,7 +64,7 @@ async def call_remote_great_ai_async( ) try: if model_class is not None: - trace["output"] = model_class.model_validate(trace["output"]) - return Trace.model_validate(trace) + trace["output"] = model_class.parse_obj(trace["output"]) + return Trace.parse_obj(trace) except Exception: raise RemoteCallError("Could not parse Trace") diff --git a/great_ai/tracing/add_ground_truth.py b/great_ai/tracing/add_ground_truth.py index 15d87a9..4080db2 100644 --- a/great_ai/tracing/add_ground_truth.py +++ b/great_ai/tracing/add_ground_truth.py @@ -1,4 +1,4 @@ -from datetime import datetime, timezone +from datetime import datetime from math import ceil from random import shuffle from typing import Any, Iterable, List, TypeVar, Union, cast @@ -23,7 +23,7 @@ def add_ground_truth( tags: Union[List[str], str] = [], train_split_ratio: float = 1, test_split_ratio: float = 0, - validation_split_ratio: float = 0, + validation_split_ratio: float = 0 ) -> None: """Add training data (with optional train-test splitting). @@ -94,7 +94,7 @@ def add_ground_truth( ) shuffle(split_tags) - created = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + created = datetime.utcnow().isoformat() traces = [ cast( Trace[T], diff --git a/great_ai/tracing/query_ground_truth.py b/great_ai/tracing/query_ground_truth.py index d30739f..81c3c95 100644 --- a/great_ai/tracing/query_ground_truth.py +++ b/great_ai/tracing/query_ground_truth.py @@ -10,7 +10,7 @@ def query_ground_truth( *, since: Optional[datetime] = None, until: Optional[datetime] = None, - return_max_count: Optional[int] = None, + return_max_count: Optional[int] = None ) -> List[Trace]: """Return training samples. diff --git a/great_ai/tracing/tracing_context.py b/great_ai/tracing/tracing_context.py index 9180061..a6a9727 100644 --- a/great_ai/tracing/tracing_context.py +++ b/great_ai/tracing/tracing_context.py @@ -1,10 +1,12 @@ from contextvars import ContextVar -from datetime import datetime, timezone +from datetime import datetime from time import perf_counter from types import TracebackType -from typing import Any, Dict, Generic, List, Literal, Optional, Type, TypeVar, cast +from typing import Any, Dict, Generic, List, Optional, Type, TypeVar, cast from uuid import uuid4 +from typing_extensions import Literal # <= Python 3.7 + from ..constants import DEVELOPMENT_TAG_NAME, ONLINE_TAG_NAME, PRODUCTION_TAG_NAME from ..context import get_context from ..views import Model, Trace @@ -23,7 +25,7 @@ class TracingContext(Generic[T]): self._models: List[Model] = [] self._values: Dict[str, Any] = {} self._trace: Optional[Trace[T]] = None - self._start_datetime = datetime.now(timezone.utc).replace(tzinfo=None) + self._start_datetime = datetime.utcnow() self._start_time = perf_counter() self._name = function_name @@ -49,19 +51,15 @@ class TracingContext(Generic[T]): logged_values=self._values, models=self._models, output=output, - exception=( - None - if exception is None - else f"{type(exception).__name__}: {exception}" - ), + exception=None + if exception is None + else f"{type(exception).__name__}: {exception}", tags=[ self._name, ONLINE_TAG_NAME, - ( - PRODUCTION_TAG_NAME - if get_context().is_production - else DEVELOPMENT_TAG_NAME - ), + PRODUCTION_TAG_NAME + if get_context().is_production + else DEVELOPMENT_TAG_NAME, ], ), ) diff --git a/great_ai/utilities/evaluate_ranking/evaluate_ranking.py b/great_ai/utilities/evaluate_ranking/evaluate_ranking.py index a613c93..da9b381 100644 --- a/great_ai/utilities/evaluate_ranking/evaluate_ranking.py +++ b/great_ai/utilities/evaluate_ranking/evaluate_ranking.py @@ -50,7 +50,7 @@ def evaluate_ranking( fig.patch.set_facecolor("white") ax = plt.axes() else: - ax = axes # type: ignore[assignment] + ax = axes classes = sorted(unique(expected), reverse=reverse_order) str_classes = [str(c) for c in classes] diff --git a/great_ai/utilities/external/pylatexenc/latex2text/__init__.py b/great_ai/utilities/external/pylatexenc/latex2text/__init__.py index 41b23b0..d8ae865 100644 --- a/great_ai/utilities/external/pylatexenc/latex2text/__init__.py +++ b/great_ai/utilities/external/pylatexenc/latex2text/__init__.py @@ -1549,7 +1549,7 @@ def latex2text( "in favor of the `pylatexenc.latex2text.LatexNodes2Text` class.", ) - nodelist, tpos, tlen = latexwalker.get_latex_nodes( + (nodelist, tpos, tlen) = latexwalker.get_latex_nodes( content, keep_inline_math=keep_inline_math, tolerant_parsing=tolerant_parsing ) diff --git a/great_ai/utilities/external/pylatexenc/latex2text/__main__.py b/great_ai/utilities/external/pylatexenc/latex2text/__main__.py index 4f9808f..a19f9b6 100644 --- a/great_ai/utilities/external/pylatexenc/latex2text/__main__.py +++ b/great_ai/utilities/external/pylatexenc/latex2text/__main__.py @@ -289,7 +289,7 @@ def main(argv=None): latex, tolerant_parsing=args.tolerant_parsing, strict_braces=args.strict_braces ) - nodelist, pos, len_ = lw.get_latex_nodes() + (nodelist, pos, len_) = lw.get_latex_nodes() ln2t = LatexNodes2Text( math_mode=args.math_mode, diff --git a/great_ai/utilities/external/pylatexenc/latex2text/_defaultspecs.py b/great_ai/utilities/external/pylatexenc/latex2text/_defaultspecs.py index 86e3ef0..e4a3e76 100644 --- a/great_ai/utilities/external/pylatexenc/latex2text/_defaultspecs.py +++ b/great_ai/utilities/external/pylatexenc/latex2text/_defaultspecs.py @@ -1774,7 +1774,7 @@ def make_accented_char(node, combining, l2tobj): for u in unicode_accents_list: - mname, mcombining = u + (mname, mcombining) = u _latex_specs_base["macros"].append( MacroTextSpec( mname, lambda x, l2tobj, c=mcombining: make_accented_char(x, c, l2tobj) diff --git a/great_ai/utilities/external/pylatexenc/latexencode/_partial_latex_encoder.py b/great_ai/utilities/external/pylatexenc/latexencode/_partial_latex_encoder.py index 6162ad6..393551d 100644 --- a/great_ai/utilities/external/pylatexenc/latexencode/_partial_latex_encoder.py +++ b/great_ai/utilities/external/pylatexenc/latexencode/_partial_latex_encoder.py @@ -71,7 +71,7 @@ class PartialLatexToLatexEncoder(UnicodeToLatexEncoder): # keyword arguments: keep_latex_chars=r"\${}^_", conversion_rules=None, - **kwargs, + **kwargs ): base_conversion_rules = conversion_rules @@ -89,7 +89,7 @@ class PartialLatexToLatexEncoder(UnicodeToLatexEncoder): ) ] + base_conversion_rules, - **kwargs, + **kwargs ) self.keep_latex_chars = keep_latex_chars diff --git a/great_ai/utilities/external/pylatexenc/latexencode/_unicode_to_latex_encoder.py b/great_ai/utilities/external/pylatexenc/latexencode/_unicode_to_latex_encoder.py index af54052..fbc5a47 100644 --- a/great_ai/utilities/external/pylatexenc/latexencode/_unicode_to_latex_encoder.py +++ b/great_ai/utilities/external/pylatexenc/latexencode/_unicode_to_latex_encoder.py @@ -617,7 +617,7 @@ class UnicodeToLatexEncoder(object): res = rulecallable(s, p.pos) if res is None: return None - consumed, repl = res + (consumed, repl) = res self._apply_replacement(p, repl, consumed, rule) return True diff --git a/great_ai/utilities/external/pylatexenc/latexwalker/__init__.py b/great_ai/utilities/external/pylatexenc/latexwalker/__init__.py index 60685e9..b60bd69 100644 --- a/great_ai/utilities/external/pylatexenc/latexwalker/__init__.py +++ b/great_ai/utilities/external/pylatexenc/latexwalker/__init__.py @@ -477,7 +477,7 @@ class LatexNode(object): parsing_state=None, pos=None, len=None, - **kwargs, + **kwargs ): # Important: subclasses must specify a list of fields they set in the @@ -607,7 +607,7 @@ class LatexGroupNode(LatexNode): "nodelist", "delimiters", ), - **kwargs, + **kwargs ) self.nodelist = nodelist self.delimiters = delimiters @@ -639,7 +639,7 @@ class LatexCommentNode(LatexNode): "comment", "comment_post_space", ), - **kwargs, + **kwargs ) self.comment = comment @@ -715,7 +715,7 @@ class LatexMacroNode(LatexNode): super(LatexMacroNode, self).__init__( _fields=("macroname", "nodeargd", "macro_post_space"), _redundant_fields=("nodeoptarg", "nodeargs"), - **kwargs, + **kwargs ) self.macroname = macroname @@ -801,7 +801,7 @@ class LatexEnvironmentNode(LatexNode): "optargs", "args", ), - **kwargs, + **kwargs ) self.environmentname = environmentname @@ -1264,7 +1264,7 @@ class LatexWalker(object): environments=True, keep_inline_math=None, parsing_state=None, - **kwargs, + **kwargs ): r""" Parses the latex content given to the constructor (and stored in `self.s`), @@ -1628,7 +1628,7 @@ class LatexWalker(object): r"Expected expression, got \end", self.s, pos, - **self.pos_to_lineno_colno(pos, as_dict=True), + **self.pos_to_lineno_colno(pos, as_dict=True) ) else: return self._mknodeposlen( @@ -1674,7 +1674,7 @@ class LatexWalker(object): "Expected expression, got closing brace '{}'".format(tok.arg), self.s, pos, - **self.pos_to_lineno_colno(pos, as_dict=True), + **self.pos_to_lineno_colno(pos, as_dict=True) ) return self._mknodeposlen( LatexCharsNode, @@ -1717,7 +1717,7 @@ class LatexWalker(object): "Unknown token type: {}".format(tok.tok), self.s, pos, - **self.pos_to_lineno_colno(pos, as_dict=True), + **self.pos_to_lineno_colno(pos, as_dict=True) ) def get_latex_maybe_optional_arg(self, pos, parsing_state=None): @@ -1821,10 +1821,10 @@ class LatexWalker(object): pos=pos, msg="get_latex_braced_group: not an opening brace/bracket: %s" % (self.s[pos]), - **self.pos_to_lineno_colno(pos, as_dict=True), + **self.pos_to_lineno_colno(pos, as_dict=True) ) - nodelist, npos, nlen = self.get_latex_nodes( + (nodelist, npos, nlen) = self.get_latex_nodes( firsttok.pos + firsttok.len, stop_upon_closing_brace=(brace_type, closing_brace), parsing_state=parsing_state, @@ -1883,14 +1883,12 @@ class LatexWalker(object): pos=pos, msg=r"get_latex_environment: expected \begin{%s}: %s" % ( - ( - environmentname - if environmentname is not None - else "" - ), + environmentname + if environmentname is not None + else "", firsttok.arg, ), - **self.pos_to_lineno_colno(pos, as_dict=True), + **self.pos_to_lineno_colno(pos, as_dict=True) ) if environmentname is None: environmentname = firsttok.arg @@ -1917,9 +1915,9 @@ class LatexWalker(object): argsresult = (None, pos, 0, {}) if len(argsresult) == 4: - argd, apos, alen, adic = argsresult + (argd, apos, alen, adic) = argsresult else: - argd, apos, alen = argsresult + (argd, apos, alen) = argsresult adic = {} pos = apos + alen @@ -1932,7 +1930,7 @@ class LatexWalker(object): math_mode_delimiter="{" + environmentname + "}", ) - nodelist, npos, nlen = self.get_latex_nodes( + (nodelist, npos, nlen) = self.get_latex_nodes( pos, stop_upon_end_environment=environmentname, parsing_state=parsing_state_inner, @@ -1977,7 +1975,7 @@ class LatexWalker(object): s=self.s, pos=tok.pos, msg="End of input while parsing {}".format(what), - **self.pos_to_lineno_colno(tok.pos, as_dict=True), + **self.pos_to_lineno_colno(tok.pos, as_dict=True) ) if getattr(e, "pos", None) is not None and e.lineno is None and e.colno is None: @@ -2229,7 +2227,7 @@ class LatexWalker(object): s=self.s, pos=tok.pos, msg="Unexpected mismatching closing brace: '%s'" % (tok.arg), - **self.pos_to_lineno_colno(tok.pos, as_dict=True), + **self.pos_to_lineno_colno(tok.pos, as_dict=True) ) return True @@ -2241,7 +2239,7 @@ class LatexWalker(object): s=self.s, pos=tok.pos, msg=("Unexpected closing environment: '{}'".format(tok.arg)), - **self.pos_to_lineno_colno(tok.pos, as_dict=True), + **self.pos_to_lineno_colno(tok.pos, as_dict=True) ) elif tok.arg != stop_upon_end_environment: # p.push_lastchars(tok_to_pos_and_chars_from_ppos(tok)) @@ -2254,7 +2252,7 @@ class LatexWalker(object): tok.arg, stop_upon_end_environment ) ), - **self.pos_to_lineno_colno(tok.pos, as_dict=True), + **self.pos_to_lineno_colno(tok.pos, as_dict=True) ) return True @@ -2278,7 +2276,7 @@ class LatexWalker(object): tok.arg, stop_upon_closing_mathmode, ), - **self.pos_to_lineno_colno(tok.pos, as_dict=True), + **self.pos_to_lineno_colno(tok.pos, as_dict=True) ) # all ok, this is a new math mode opening. Keep an assert # in case we forget to include some math-mode delimiters in @@ -2293,7 +2291,7 @@ class LatexWalker(object): s=self.s, pos=tok.pos, msg="Unexpected closing math mode: '{}'".format(tok.arg), - **self.pos_to_lineno_colno(tok.pos, as_dict=True), + **self.pos_to_lineno_colno(tok.pos, as_dict=True) ) # we have encountered a new math inline, parse the math expression @@ -2308,7 +2306,7 @@ class LatexWalker(object): ) try: - mathinline_nodelist, mpos, mlen = self.get_latex_nodes( + (mathinline_nodelist, mpos, mlen) = self.get_latex_nodes( p.pos, stop_upon_closing_mathmode=corresponding_closing_mathmode, parsing_state=parsing_state_inner, @@ -2318,7 +2316,7 @@ class LatexWalker(object): _maketuple( 'math mode "{}"'.format(tok.arg), tok.pos, - *self.pos_to_lineno_colno(tok.pos), + *self.pos_to_lineno_colno(tok.pos) ) ) raise @@ -2356,7 +2354,7 @@ class LatexWalker(object): if tok.tok == "brace_open": # another braced group to read. try: - groupnode, bpos, blen = self.get_latex_braced_group( + (groupnode, bpos, blen) = self.get_latex_braced_group( tok.pos, brace_type=tok.arg, parsing_state=p.parsing_state ) # except LatexWalkerEndOfStream as e: @@ -2378,7 +2376,7 @@ class LatexWalker(object): if tok.tok == "begin_environment": # an environment to read. try: - envnode, epos, elen = self.get_latex_environment( + (envnode, epos, elen) = self.get_latex_environment( tok.pos, environmentname=tok.arg, parsing_state=p.parsing_state ) except LatexWalkerParseError as e: @@ -2386,7 +2384,7 @@ class LatexWalker(object): _maketuple( 'begin environment "{}"'.format(tok.arg), tok.pos, - *self.pos_to_lineno_colno(tok.pos), + *self.pos_to_lineno_colno(tok.pos) ) ) raise @@ -2417,9 +2415,9 @@ class LatexWalker(object): margsresult = (None, tok.pos + tok.len, 0, {}) if len(margsresult) == 4: - nodeargd, mapos, malen, mdic = margsresult + (nodeargd, mapos, malen, mdic) = margsresult else: - nodeargd, mapos, malen = margsresult + (nodeargd, mapos, malen) = margsresult mdic = {} p.pos = mapos + malen @@ -2475,9 +2473,9 @@ class LatexWalker(object): if res is not None: # specials expects arguments, read them if len(res) == 4: - nodeargd, mapos, malen, spdic = res + (nodeargd, mapos, malen, spdic) = res else: - nodeargd, mapos, malen = res + (nodeargd, mapos, malen) = res spdic = {} p.pos = mapos + malen @@ -2507,7 +2505,7 @@ class LatexWalker(object): s=self.s, pos=p.pos, msg="Unknown token: {!r}".format(tok), - **self.pos_to_lineno_colno(p.pos, as_dict=True), + **self.pos_to_lineno_colno(p.pos, as_dict=True) ) while True: @@ -2533,7 +2531,7 @@ class LatexWalker(object): msg="Unexpected end of stream, was expecting {}".format( expecting ), - **self.pos_to_lineno_colno(len(self.s), as_dict=True), + **self.pos_to_lineno_colno(len(self.s), as_dict=True) ) if self.tolerant_parsing: r_endnow = True @@ -2653,7 +2651,7 @@ def get_latex_nodes( stop_upon_closing_brace=None, stop_upon_end_environment=None, stop_upon_closing_mathmode=None, - **parse_flags, + **parse_flags ): """ Parses latex content `s`. diff --git a/great_ai/utilities/external/pylatexenc/latexwalker/__main__.py b/great_ai/utilities/external/pylatexenc/latexwalker/__main__.py index 46eccff..fe95f0b 100644 --- a/great_ai/utilities/external/pylatexenc/latexwalker/__main__.py +++ b/great_ai/utilities/external/pylatexenc/latexwalker/__main__.py @@ -184,7 +184,7 @@ def main(argv=None): latex, tolerant_parsing=args.tolerant_parsing, strict_braces=args.strict_braces ) - nodelist, pos, len_ = latexwalker.get_latex_nodes() + (nodelist, pos, len_) = latexwalker.get_latex_nodes() if args.output_format == "human": print("\n--- NODES ---\n") diff --git a/great_ai/utilities/external/pylatexenc/macrospec/__init__.py b/great_ai/utilities/external/pylatexenc/macrospec/__init__.py index 66ab0de..9975837 100644 --- a/great_ai/utilities/external/pylatexenc/macrospec/__init__.py +++ b/great_ai/utilities/external/pylatexenc/macrospec/__init__.py @@ -34,6 +34,7 @@ macros and environments, specifying how they should be parsed by `pylatexenc 2.0`. """ + import sys if sys.version_info.major > 2: @@ -153,7 +154,7 @@ class EnvironmentSpec(object): environmentname, args_parser=MacroStandardArgsParser(), is_math_mode=False, - **kwargs, + **kwargs ): super(EnvironmentSpec, self).__init__(**kwargs) self.environmentname = environmentname @@ -775,9 +776,9 @@ class LatexContextDb(object): new_context.add_context_category( cat, macros=self.d[cat]["macros"].values() if keep_macros else [], - environments=( - self.d[cat]["environments"].values() if keep_environments else [] - ), + environments=self.d[cat]["environments"].values() + if keep_environments + else [], specials=self.d[cat]["specials"].values() if keep_specials else [], ) diff --git a/great_ai/utilities/external/pylatexenc/macrospec/_argparsers.py b/great_ai/utilities/external/pylatexenc/macrospec/_argparsers.py index abadc40..7fb20e8 100644 --- a/great_ai/utilities/external/pylatexenc/macrospec/_argparsers.py +++ b/great_ai/utilities/external/pylatexenc/macrospec/_argparsers.py @@ -296,7 +296,7 @@ class MacroStandardArgsParser(object): for j, argt in enumerate(self.argspec): if argt == "{": - node, np, nl = w.get_latex_expression( + (node, np, nl) = w.get_latex_expression( p, strict_braces=False, parsing_state=get_inner_parsing_state(j) ) p = np + nl @@ -315,7 +315,7 @@ class MacroStandardArgsParser(object): if optarginfotuple is None: argnlist.append(None) continue - node, np, nl = optarginfotuple + (node, np, nl) = optarginfotuple p = np + nl argnlist.append(node) diff --git a/great_ai/utilities/parallel_map/parallel_map.py b/great_ai/utilities/parallel_map/parallel_map.py index f8c8e1f..bb4dd0d 100644 --- a/great_ai/utilities/parallel_map/parallel_map.py +++ b/great_ai/utilities/parallel_map/parallel_map.py @@ -3,7 +3,6 @@ from typing import ( Awaitable, Callable, Iterable, - Literal, Optional, Sequence, TypeVar, @@ -12,6 +11,7 @@ from typing import ( ) import dill +from typing_extensions import Literal # <= Python 3.7 from .get_config import get_config from .manage_communication import manage_communication @@ -31,7 +31,8 @@ def parallel_map( chunk_size: Optional[int] = ..., concurrency: Optional[int] = ..., unordered: bool = ..., -) -> Iterable[Optional[V]]: ... +) -> Iterable[Optional[V]]: + ... @overload @@ -43,7 +44,8 @@ def parallel_map( ignore_exceptions: Literal[True], concurrency: Optional[int] = ..., unordered: bool = ..., -) -> Iterable[Optional[V]]: ... +) -> Iterable[Optional[V]]: + ... @overload @@ -55,7 +57,8 @@ def parallel_map( ignore_exceptions: Literal[False] = ..., concurrency: Optional[int] = ..., unordered: bool = ..., -) -> Iterable[V]: ... +) -> Iterable[V]: + ... @overload @@ -67,7 +70,8 @@ def parallel_map( ignore_exceptions: Literal[False] = ..., concurrency: Optional[int] = ..., unordered: bool = ..., -) -> Iterable[V]: ... +) -> Iterable[V]: + ... def parallel_map( @@ -143,7 +147,7 @@ def parallel_map( serialized_map_function = dill.dumps(func, byref=True, recurse=False) processes = [ - ctx.Process( + ctx.Process( # type: ignore name=f"parallel_map_{config.function_name}_{i}", target=mapper_function, daemon=True, diff --git a/great_ai/utilities/parallel_map/simple_parallel_map.py b/great_ai/utilities/parallel_map/simple_parallel_map.py index 7a506b3..0981a57 100644 --- a/great_ai/utilities/parallel_map/simple_parallel_map.py +++ b/great_ai/utilities/parallel_map/simple_parallel_map.py @@ -1,6 +1,6 @@ from typing import Awaitable, Callable, List, Optional, Sequence, TypeVar, Union -from tqdm import tqdm +from tqdm.cli import tqdm from .parallel_map import parallel_map diff --git a/great_ai/utilities/parallel_map/threaded_parallel_map.py b/great_ai/utilities/parallel_map/threaded_parallel_map.py index 21d80e5..a751a06 100644 --- a/great_ai/utilities/parallel_map/threaded_parallel_map.py +++ b/great_ai/utilities/parallel_map/threaded_parallel_map.py @@ -4,7 +4,6 @@ from typing import ( Awaitable, Callable, Iterable, - Literal, Optional, Sequence, TypeVar, @@ -12,6 +11,8 @@ from typing import ( overload, ) +from typing_extensions import Literal # <= Python 3.7 + from .get_config import get_config from .manage_communication import manage_communication from .mapper_function import mapper_function @@ -29,7 +30,8 @@ def threaded_parallel_map( chunk_size: Optional[int] = ..., concurrency: Optional[int] = ..., unordered: bool = ..., -) -> Iterable[Optional[V]]: ... +) -> Iterable[Optional[V]]: + ... @overload @@ -41,7 +43,8 @@ def threaded_parallel_map( ignore_exceptions: Literal[True], concurrency: Optional[int] = ..., unordered: bool = ..., -) -> Iterable[Optional[V]]: ... +) -> Iterable[Optional[V]]: + ... @overload @@ -53,7 +56,8 @@ def threaded_parallel_map( ignore_exceptions: Literal[False] = ..., concurrency: Optional[int] = ..., unordered: bool = ..., -) -> Iterable[V]: ... +) -> Iterable[V]: + ... @overload @@ -65,7 +69,8 @@ def threaded_parallel_map( ignore_exceptions: Literal[False] = ..., concurrency: Optional[int] = ..., unordered: bool = ..., -) -> Iterable[V]: ... +) -> Iterable[V]: + ... def threaded_parallel_map( diff --git a/great_ai/views/operators.py b/great_ai/views/operators.py index 071e266..3bd2624 100644 --- a/great_ai/views/operators.py +++ b/great_ai/views/operators.py @@ -1,4 +1,6 @@ -from typing import List, Literal +from typing import List + +from typing_extensions import Literal # <= Python 3.7 Operator = Literal[">=", "<=", "<", ">", "!=", "=", "contains"] diff --git a/great_ai/views/outputs/classification_output.py b/great_ai/views/outputs/classification_output.py index 7e3cccc..f955cc2 100644 --- a/great_ai/views/outputs/classification_output.py +++ b/great_ai/views/outputs/classification_output.py @@ -6,4 +6,4 @@ from ..hashable_base_model import HashableBaseModel class ClassificationOutput(HashableBaseModel): label: Union[str, int] confidence: float - explanation: Optional[Any] = None + explanation: Optional[Any] diff --git a/great_ai/views/outputs/regression_output.py b/great_ai/views/outputs/regression_output.py index 997c92e..96c1daf 100644 --- a/great_ai/views/outputs/regression_output.py +++ b/great_ai/views/outputs/regression_output.py @@ -5,4 +5,4 @@ from ..hashable_base_model import HashableBaseModel class RegressionOutput(HashableBaseModel): value: Union[int, float] - explanation: Optional[Any] = None + explanation: Optional[Any] diff --git a/great_ai/views/outputs/sequence_labeling_output.py b/great_ai/views/outputs/sequence_labeling_output.py index 7dceb8c..a1a156f 100644 --- a/great_ai/views/outputs/sequence_labeling_output.py +++ b/great_ai/views/outputs/sequence_labeling_output.py @@ -1,4 +1,6 @@ -from typing import Any, List, Literal, Optional +from typing import Any, List, Optional + +from typing_extensions import Literal # <= Python 3.7 from ..hashable_base_model import HashableBaseModel @@ -7,9 +9,9 @@ class LabeledToken(HashableBaseModel): token: str tag: Literal["B", "I", "O", "E", "S"] confidence: float - explanation: Optional[Any] = None + explanation: Optional[Any] class SequenceLabelingOutput(HashableBaseModel): labeled_tokens: List[LabeledToken] - explanation: Optional[Any] = None + explanation: Optional[Any] diff --git a/great_ai/views/query.py b/great_ai/views/query.py index ece1d21..7257a7b 100644 --- a/great_ai/views/query.py +++ b/great_ai/views/query.py @@ -1,7 +1,7 @@ from datetime import datetime from typing import List, Optional, Sequence -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel from .filter import Filter from .sort_by import SortBy @@ -15,8 +15,8 @@ class Query(BaseModel): until: Optional[datetime] = None has_feedback: Optional[bool] = None - model_config = ConfigDict( - json_schema_extra={ + class Config: + schema_extra = { "example": { "filter": [ { @@ -33,4 +33,3 @@ class Query(BaseModel): "has_feedback": False, } } - ) diff --git a/great_ai/views/sort_by.py b/great_ai/views/sort_by.py index d0f8eff..aa92fcc 100644 --- a/great_ai/views/sort_by.py +++ b/great_ai/views/sort_by.py @@ -1,6 +1,5 @@ -from typing import Literal - from pydantic import BaseModel +from typing_extensions import Literal # <= Python 3.7 class SortBy(BaseModel): diff --git a/great_ai/views/trace.py b/great_ai/views/trace.py index f7c6328..76f03f5 100644 --- a/great_ai/views/trace.py +++ b/great_ai/views/trace.py @@ -1,7 +1,7 @@ from pprint import pformat from typing import Any, Dict, Generic, List, Optional, TypeVar -from pydantic import ConfigDict +from pydantic import Extra from .hashable_base_model import HashableBaseModel from .model import Model @@ -9,7 +9,7 @@ from .model import Model T = TypeVar("T") -class Trace(HashableBaseModel, Generic[T]): +class Trace(Generic[T], HashableBaseModel): """Universal structure for storing prediction traces and training data. Attributes: @@ -32,12 +32,13 @@ class Trace(HashableBaseModel, Generic[T]): original_execution_time_ms: float logged_values: Dict[str, Any] models: List[Model] - exception: Optional[str] = None - output: Optional[T] = None + exception: Optional[str] + output: Optional[T] feedback: Any = None tags: List[str] - model_config = ConfigDict(extra="ignore") + class Config: + extra = Extra.ignore @property def input(self) -> Any: @@ -78,7 +79,7 @@ class Trace(HashableBaseModel, Generic[T]): def to_flat_dict(self, include_original: bool = True) -> Dict[str, Any]: return { **( - self.model_dump() + self.dict() if include_original else { "trace_id": self.trace_id, @@ -87,11 +88,9 @@ class Trace(HashableBaseModel, Generic[T]): } ), **{ - k: ( - v - if (isinstance(v, float) or isinstance(v, int)) - else pformat(v, indent=2, compact=True) - ) + k: v + if (isinstance(v, float) or isinstance(v, int)) + else pformat(v, indent=2, compact=True) for k, v in self.logged_values.items() }, "models_flat": self.models_flat, @@ -102,7 +101,6 @@ class Trace(HashableBaseModel, Generic[T]): } def __repr__(self) -> str: - formatted = pformat(self.model_dump(), indent=2, compact=True).replace( - "{ ", "{", 1 - ) - return f"Trace[{type(self.output).__name__}]({formatted})" + return f"""Trace[{type(self.output).__name__}]({ + pformat(self.dict(), indent=2, compact=True).replace('{ ', '{', 1) + })""" diff --git a/pyproject.toml b/pyproject.toml index a02460d..d4eaf67 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,58 +21,51 @@ classifiers = [ "Natural Language :: English", ] keywords = ["SE4ML", "MLOps", "AI engineering", "general", "robust", "end-to-end", "automated", "trustworthy", "ai", "deployment"] -requires-python = ">= 3.10" +requires-python = ">= 3.7" dependencies = [ - "scikit-learn >= 1.3, < 2", - "matplotlib >= 3.7, < 4", - "numpy >= 1.24, < 3", - "nbconvert >= 7.0, < 8", - "ipython >= 8.0, < 10", - "unidecode >= 1.3.0, < 2", - "syntok >= 1.4.0, < 2", - "langcodes[data] >= 3.3.0, < 4", - "langdetect >= 1.0.9, < 2", - "tinydb >= 4.7.0, < 5", - "boto3 >= 1.34, < 2", - "plotly >= 5.20, < 7", - "pandas >= 2.0, < 4", - "dash >= 2.16, < 5", - "fastapi >= 0.110, < 1", - "uvicorn[standard] >= 0.27, < 1", - "a2wsgi >= 1.9, < 2", - "watchdog >= 3.0, < 7", - "typeguard >= 4.0, < 5", - "pydantic >= 2.5, < 3", - "pymongo >= 4.6, < 5", - "dill >= 0.3.6, < 1", - "tqdm >= 4.64, < 5", - "httpx >= 0.24, < 1", + "scikit-learn", + "matplotlib", + "numpy", + "nbconvert", + "ipython", + "unidecode >= 1.3.0", + "syntok >= 1.4.0", + "langcodes[data] >= 3.3.0", + "langdetect >= 1.0.9", + "tinydb >= 4.7.0", + "boto3 >= 1.23.0", + "plotly >= 5.8.0", + "pandas", + "dash >= 2.4.0", + "fastapi >= 0.70.0", + "uvicorn[standard] >= 0.18.0", + "watchdog >= 2.1.0", + "typeguard >= 2.10.0", + "pymongo >= 4.0.0", + "dill >= 0.3.5.0", + "tqdm", + "httpx >= 0.20.0", ] [project.optional-dependencies] dev = [ - "flit >= 3.9, < 4", - "mkdocs >= 1.5, < 2", - "mkdocstrings[python] >= 0.24, < 1", - "mkdocs-material >= 9.5, < 10", - "mkdocs-jupyter >= 0.24, < 1", - "mkdocs-git-revision-date-localized-plugin >= 1.2, < 2", - "autoflake >= 2.0, < 3", - "isort >= 5.12, < 7", - "black[jupyter] >= 25.1, < 26", - "mypy >= 1.8, < 2", - "flake8 >= 7.0, < 8", - "tox >= 4.0, < 5", - "pytest >= 7.4, < 9", - "pytest-cov >= 4.1, < 8", - "pytest-asyncio >= 0.23, < 2", + "flit", + "mkdocs", + "mkdocstrings[python]", + "mkdocs-material", + "mkdocs-jupyter", + "mkdocs-git-revision-date-localized-plugin", + "autoflake", + "isort", + "black[jupyter]", + "mypy", + "flake8", + "tox", + "pytest", + "pytest-cov", + "pytest-asyncio", ] -[tool.pytest.ini_options] -# `...` in doctests (e.g. third-party exception messages that drift between -# dependency versions) is treated as a wildcard rather than a literal. -doctest_optionflags = ["ELLIPSIS"] - [project.urls] Documentation = "https://great-ai.scoutinscience.com" GitHub = "https://github.com/schmelczer/great-ai" diff --git a/scripts/check-python.sh b/scripts/check-python.sh index 1b36198..668308c 100755 --- a/scripts/check-python.sh +++ b/scripts/check-python.sh @@ -3,7 +3,7 @@ set -e echo "Installing dependencies if necessary" -python3 -m pip install autoflake isort black[jupyter] mypy flake8 +python3 -m pip install --upgrade autoflake isort black[jupyter] mypy flake8 for dir in "$@"; do echo "Checking $dir" @@ -16,10 +16,10 @@ for dir in "$@"; do if find . -name "*.py" 2>/dev/null | grep -q .; then yes | python3 -m mypy . --install-types > /dev/null || true - python3 -m mypy --namespace-packages --ignore-missing-imports --install-types --non-interactive --disallow-untyped-defs --disallow-incomplete-defs --follow-imports=silent --exclude=external/ --exclude=/build/ --exclude='(^|/)__main__\.py$' --pretty . + python3 -m mypy --namespace-packages --ignore-missing-imports --install-types --non-interactive --disallow-untyped-defs --disallow-incomplete-defs --follow-imports=silent --exclude=external/ --exclude=/build/ --pretty . fi - python3 -m flake8 . --count --show-source --statistics --exclude=__init__.py,.env,external --extend-ignore=E501,E402,F821,W503,E722,E203,E704 + python3 -m flake8 . --count --show-source --statistics --exclude=__init__.py,.env,external --ignore=E501,E402,F821,W503,E722,E203 cd - done diff --git a/tests/external/test_internals.py b/tests/external/test_internals.py index 142c1fa..1003bff 100644 --- a/tests/external/test_internals.py +++ b/tests/external/test_internals.py @@ -24,8 +24,7 @@ class Wrapped: @pytest.mark.asyncio -async def test_done_callback_cancelled(): - loop = asyncio.get_running_loop() +async def test_done_callback_cancelled(loop): task = loop.create_future() fut = loop.create_future() @@ -39,8 +38,7 @@ async def test_done_callback_cancelled(): @pytest.mark.asyncio -async def test_done_callback_exception(): - loop = asyncio.get_running_loop() +async def test_done_callback_exception(loop): task = loop.create_future() fut = loop.create_future() @@ -62,8 +60,7 @@ async def test_done_callback_exception(): @pytest.mark.asyncio -async def test_done_callback(): - loop = asyncio.get_running_loop() +async def test_done_callback(loop): task = loop.create_future() fut = loop.create_future() @@ -231,8 +228,7 @@ def test_close(loop): @pytest.mark.asyncio -async def test_wait_closed(): - loop = asyncio.get_running_loop() +async def test_wait_closed(loop): wrapped = Wrapped() wrapped.tasks = set() @@ -242,7 +238,7 @@ async def test_wait_closed(): return_exceptions=True, ) assert ret == [] - mocked.assert_called_once() + assert mocked.called_once() asyncio.set_event_loop(loop) with mock.patch("great_ai.external.async_lru._close_waited") as mocked: @@ -251,7 +247,7 @@ async def test_wait_closed(): return_exceptions=True, ) assert ret == [] - mocked.assert_called_once() + assert mocked.called_once() asyncio.set_event_loop(None) fut = loop.create_future() @@ -263,7 +259,7 @@ async def test_wait_closed(): return_exceptions=True, ) assert ret == [None] - mocked.assert_called_once() + assert mocked.called_once() exc = ZeroDivisionError() fut = loop.create_future() @@ -275,7 +271,7 @@ async def test_wait_closed(): return_exceptions=True, ) assert ret == [exc] - mocked.assert_called_once() + assert mocked.called_once() fut = loop.create_future() fut.set_exception(ZeroDivisionError) @@ -286,21 +282,17 @@ async def test_wait_closed(): wrapped, return_exceptions=False, ) - # the awaited result raised before _wait_closed could flush its done - # callback, so yield once to let the scheduled _close_waited run - await asyncio.sleep(0) - mocked.assert_called_once() + assert mocked.called_once() def test_close_waited(): wrapped = Wrapped() - # _close_waited calls wrapped.cache_clear(); mock that directly (patching the - # module-level _cache_clear would not affect the already-bound partial). - wrapped.cache_clear = mock.Mock() + wrapped.cache_clear = partial(_cache_clear, wrapped) - _close_waited(wrapped, None) + with mock.patch("great_ai.external.async_lru._cache_clear") as mocked: + _close_waited(wrapped, None) - wrapped.cache_clear.assert_called_once() + assert mocked.called_once() def test_cache_info(): @@ -347,7 +339,7 @@ def test_cache_hit(): with mock.patch("great_ai.external.async_lru.__cache_touch") as mocked: _cache_hit(wrapped, 1) - mocked.assert_called_once() + assert mocked.called_once() assert wrapped.hits == 2 @@ -365,7 +357,7 @@ def test_cache_miss(): with mock.patch("great_ai.external.async_lru.__cache_touch") as mocked: _cache_miss(wrapped, 1) - mocked.assert_called_once() + assert mocked.called_once() assert wrapped.misses == 2 diff --git a/tests/utilities/test_parallel_map.py b/tests/utilities/test_parallel_map.py index 4774f7b..9fe2b8f 100644 --- a/tests/utilities/test_parallel_map.py +++ b/tests/utilities/test_parallel_map.py @@ -23,7 +23,9 @@ def test_with_iterable() -> None: expected = [v**3 for v in range(10)] - assert list(parallel_map(lambda x: x**3, my_generator(), chunk_size=1)) == expected + assert ( + list(parallel_map(lambda x: x**3, my_generator(), chunk_size=1)) == expected + ) def test_simple_case_invalid_values() -> None: @@ -42,10 +44,14 @@ def test_this_process_exception() -> None: assert False with pytest.raises(AssertionError): - list(parallel_map(lambda v: v**2, my_generator(), concurrency=2, chunk_size=2)) + list( + parallel_map(lambda v: v**2, my_generator(), concurrency=2, chunk_size=2) + ) with pytest.raises(AssertionError): - list(parallel_map(lambda v: v**2, my_generator(), concurrency=1, chunk_size=2)) + list( + parallel_map(lambda v: v**2, my_generator(), concurrency=1, chunk_size=2) + ) def test_ignore_this_process_exception() -> None: