andras/great-ai
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases 4 Packages Wiki Activity Actions

Add initialized flag

Browse source
This commit is contained in:
Andras Schmelczer 2022-06-03 19:22:02 +02:00
parent a0d3a9cb2a
commit ff431d2af7
6 changed files with 9 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

2
great_ai/src/great_ai/large_file/__init__.py
View file

@ -1 +1 @@
from .large_file import LargeFile, LargeFileS3 from .large_file import LargeFile, LargeFileLocal, LargeFileMongo, LargeFileS3

8
great_ai/src/great_ai/large_file/large_file/large_file.py
View file

@ -2,7 +2,7 @@ import configparser
import os import os
import shutil import shutil
import tempfile import tempfile
from abc import ABC, abstractclassmethod, abstractmethod from abc import ABC, abstractmethod
from pathlib import Path from pathlib import Path
from types import TracebackType from types import TracebackType
from typing import IO, Any, List, Optional, Type, Union, cast from typing import IO, Any, List, Optional, Type, Union, cast
@ -48,6 +48,7 @@ class LargeFile(ABC):
``` ```
""" """
initialized = False
cache_path = Path(".cache") cache_path = Path(".cache")
max_cache_size: Optional[str] = "30GB" max_cache_size: Optional[str] = "30GB"
@ -96,11 +97,11 @@ class LargeFile(ABC):
credentials.default_section credentials.default_section
cls.configure_credentials(**credentials[credentials.default_section]) cls.configure_credentials(**credentials[credentials.default_section])
@abstractclassmethod @classmethod
def configure_credentials( def configure_credentials(
cls, cls,
) -> None: ) -> None:
pass cls.initialized = True
def __enter__(self) -> IO: def __enter__(self) -> IO:
self._file: IO[Any] = ( self._file: IO[Any] = (
@ -236,7 +237,6 @@ class LargeFile(ABC):
), ),
version=int(f.name.split(CACHE_NAME_VERSION_SEPARATOR)[-1]), version=int(f.name.split(CACHE_NAME_VERSION_SEPARATOR)[-1]),
remote_path=f, remote_path=f,
origin="filesystem",
) )
for f in self.cache_path.glob( for f in self.cache_path.glob(
f"{self._name}{CACHE_NAME_VERSION_SEPARATOR}*" f"{self._name}{CACHE_NAME_VERSION_SEPARATOR}*"

1
great_ai/src/great_ai/large_file/large_file/large_file_local.py
View file

@ -33,6 +33,7 @@ class LargeFileLocal(LargeFile):
keep_last_n=keep_last_n, keep_last_n=keep_last_n,
cache_only_mode=True, cache_only_mode=True,
) )
super().configure_credentials()
def _find_remote_instances(self) -> List[DataInstance]: def _find_remote_instances(self) -> List[DataInstance]:
return [] return []

1
great_ai/src/great_ai/large_file/large_file/large_file_mongo.py
View file

@ -32,6 +32,7 @@ class LargeFileMongo(LargeFile):
) -> None: ) -> None:
cls.connection_string = connection_string cls.connection_string = connection_string
cls.database = database cls.database = database
super().configure_credentials()
@cached_property @cached_property
def _client(self) -> GridFSBucket: def _client(self) -> GridFSBucket:

2
great_ai/src/great_ai/large_file/large_file/large_file_s3.py
View file

@ -66,6 +66,7 @@ class LargeFileS3(LargeFile):
cls.secret_access_key = aws_secret_access_key cls.secret_access_key = aws_secret_access_key
cls.bucket_name = large_files_bucket_name cls.bucket_name = large_files_bucket_name
cls.endpoint_url = aws_endpoint_url cls.endpoint_url = aws_endpoint_url
super().configure_credentials()
@cached_property @cached_property
def _client(self) -> boto3.client: def _client(self) -> boto3.client:
@ -99,7 +100,6 @@ class LargeFileS3(LargeFile):
name=o["Key"].split(S3_NAME_VERSION_SEPARATOR)[0], name=o["Key"].split(S3_NAME_VERSION_SEPARATOR)[0],
version=int(o["Key"].split(S3_NAME_VERSION_SEPARATOR)[-1]), version=int(o["Key"].split(S3_NAME_VERSION_SEPARATOR)[-1]),
remote_path=o["Key"], remote_path=o["Key"],
origin="s3",
) )
for o in found_objects["Contents"] for o in found_objects["Contents"]
if o["Key"].split(S3_NAME_VERSION_SEPARATOR)[0] == self._name if o["Key"].split(S3_NAME_VERSION_SEPARATOR)[0] == self._name

3
great_ai/src/great_ai/large_file/models/data_instance.py
View file

@ -1,4 +1,4 @@
from typing import Any, Literal from typing import Any
from pydantic import BaseModel from pydantic import BaseModel
@ -7,4 +7,3 @@ class DataInstance(BaseModel):
name: str name: str
version: int version: int
remote_path: Any remote_path: Any
origin: Literal["filesystem", "mongodb", "s3"]