Make codebase compatible with Python 3.7
This commit is contained in:
parent
2ad06e8b38
commit
f8db199996
11 changed files with 125 additions and 51 deletions
|
|
@ -1,2 +1,3 @@
|
|||
from .cached_property import cached_property
|
||||
from .human_readable_to_byte import human_readable_to_byte
|
||||
from .progress_bar import DownloadProgressBar, UploadProgressBar
|
||||
|
|
|
|||
52
great_ai/large_file/helper/cached_property.py
Normal file
52
great_ai/large_file/helper/cached_property.py
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
from threading import RLock
|
||||
|
||||
_NOT_FOUND = object()
|
||||
|
||||
|
||||
class cached_property:
|
||||
def __init__(self, func): # type: ignore
|
||||
self.func = func
|
||||
self.attrname = None
|
||||
self.__doc__ = func.__doc__
|
||||
self.lock = RLock()
|
||||
|
||||
def __set_name__(self, owner, name): # type: ignore
|
||||
if self.attrname is None:
|
||||
self.attrname = name
|
||||
elif name != self.attrname:
|
||||
raise TypeError(
|
||||
"Cannot assign the same cached_property to two different names "
|
||||
f"({self.attrname!r} and {name!r})."
|
||||
)
|
||||
|
||||
def __get__(self, instance, owner=None): # type: ignore
|
||||
if instance is None:
|
||||
return self
|
||||
if self.attrname is None:
|
||||
raise TypeError(
|
||||
"Cannot use cached_property instance without calling __set_name__ on it."
|
||||
)
|
||||
try:
|
||||
cache = instance.__dict__
|
||||
except AttributeError: # not all objects have __dict__ (e.g. class defines slots)
|
||||
msg = (
|
||||
f"No '__dict__' attribute on {type(instance).__name__!r} "
|
||||
f"instance to cache {self.attrname!r} property."
|
||||
)
|
||||
raise TypeError(msg) from None
|
||||
val = cache.get(self.attrname, _NOT_FOUND)
|
||||
if val is _NOT_FOUND:
|
||||
with self.lock:
|
||||
# check if another thread filled cache while we awaited lock
|
||||
val = cache.get(self.attrname, _NOT_FOUND)
|
||||
if val is _NOT_FOUND:
|
||||
val = self.func(instance)
|
||||
try:
|
||||
cache[self.attrname] = val
|
||||
except TypeError:
|
||||
msg = (
|
||||
f"The '__dict__' attribute on {type(instance).__name__!r} instance "
|
||||
f"does not support item assignment for caching {self.attrname!r} property."
|
||||
)
|
||||
raise TypeError(msg) from None
|
||||
return val
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
from abc import ABC, abstractmethod
|
||||
from functools import lru_cache
|
||||
|
|
@ -70,7 +71,13 @@ class LargeFileBase(ABC):
|
|||
|
||||
self._buffering = buffering
|
||||
self._encoding = encoding
|
||||
|
||||
if errors is not None and sys.version_info[1] < 8:
|
||||
raise RuntimeError(
|
||||
"The `errors` kwarg is only supported in 3.8 <= Python versions."
|
||||
)
|
||||
self._errors = errors
|
||||
|
||||
self._newline = newline
|
||||
|
||||
LargeFileBase.cache_path.mkdir(parents=True, exist_ok=True)
|
||||
|
|
@ -99,16 +106,20 @@ class LargeFileBase(ABC):
|
|||
cls.initialized = True
|
||||
|
||||
def __enter__(self) -> IO:
|
||||
params = dict(
|
||||
mode=self._mode,
|
||||
buffering=self._buffering,
|
||||
encoding=self._encoding,
|
||||
newline=self._newline,
|
||||
delete=False,
|
||||
prefix="large_file-",
|
||||
)
|
||||
|
||||
if sys.version_info[1] >= 8:
|
||||
params["errors"] = self._errors
|
||||
|
||||
self._file: IO[Any] = (
|
||||
tempfile.NamedTemporaryFile(
|
||||
mode=self._mode,
|
||||
buffering=self._buffering,
|
||||
encoding=self._encoding,
|
||||
newline=self._newline,
|
||||
errors=self._errors,
|
||||
delete=False,
|
||||
prefix="large_file-",
|
||||
)
|
||||
tempfile.NamedTemporaryFile(**params) # type: ignore
|
||||
if "w" in self._mode
|
||||
else open(
|
||||
self.get(),
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import re
|
||||
from functools import cached_property
|
||||
from pathlib import Path
|
||||
from typing import Any, List
|
||||
|
||||
|
|
@ -7,7 +6,7 @@ from gridfs import DEFAULT_CHUNK_SIZE, Database, GridFSBucket
|
|||
from pymongo import MongoClient
|
||||
|
||||
from ...utilities import get_logger
|
||||
from ..helper import DownloadProgressBar, UploadProgressBar
|
||||
from ..helper import DownloadProgressBar, UploadProgressBar, cached_property
|
||||
from ..models import DataInstance
|
||||
from .large_file_base import LargeFileBase
|
||||
|
||||
|
|
|
|||
|
|
@ -1,11 +1,10 @@
|
|||
from functools import cached_property
|
||||
from pathlib import Path
|
||||
from typing import Any, List, Optional
|
||||
|
||||
import boto3
|
||||
|
||||
from ...utilities import get_logger
|
||||
from ..helper import DownloadProgressBar, UploadProgressBar
|
||||
from ..helper import DownloadProgressBar, UploadProgressBar, cached_property
|
||||
from ..models import DataInstance
|
||||
from .large_file_base import LargeFileBase
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue