Add initialized flag

great_ai/src/great_ai/large_file/__init__.py

@@ -1 +1 @@
-from .large_file import LargeFile, LargeFileS3
+from .large_file import LargeFile, LargeFileLocal, LargeFileMongo, LargeFileS3

great_ai/src/great_ai/large_file/large_file/large_file.py

@@ -2,7 +2,7 @@ import configparser
 import os
 import shutil
 import tempfile
-from abc import ABC, abstractclassmethod, abstractmethod
+from abc import ABC, abstractmethod
 from pathlib import Path
 from types import TracebackType
 from typing import IO, Any, List, Optional, Type, Union, cast
@@ -48,6 +48,7 @@ class LargeFile(ABC):
     ```
     """
 
+    initialized = False
     cache_path = Path(".cache")
     max_cache_size: Optional[str] = "30GB"
 
@@ -96,11 +97,11 @@ class LargeFile(ABC):
         credentials.default_section
         cls.configure_credentials(**credentials[credentials.default_section])
 
-    @abstractclassmethod
+    @classmethod
     def configure_credentials(
         cls,
     ) -> None:
-        pass
+        cls.initialized = True
 
     def __enter__(self) -> IO:
         self._file: IO[Any] = (
@@ -236,7 +237,6 @@ class LargeFile(ABC):
                 ),
                 version=int(f.name.split(CACHE_NAME_VERSION_SEPARATOR)[-1]),
                 remote_path=f,
-                origin="filesystem",
             )
             for f in self.cache_path.glob(
                 f"{self._name}{CACHE_NAME_VERSION_SEPARATOR}*"

great_ai/src/great_ai/large_file/large_file/large_file_local.py

@@ -33,6 +33,7 @@ class LargeFileLocal(LargeFile):
             keep_last_n=keep_last_n,
             cache_only_mode=True,
         )
+        super().configure_credentials()
 
     def _find_remote_instances(self) -> List[DataInstance]:
         return []

great_ai/src/great_ai/large_file/large_file/large_file_mongo.py

@@ -32,6 +32,7 @@ class LargeFileMongo(LargeFile):
     ) -> None:
         cls.connection_string = connection_string
         cls.database = database
+        super().configure_credentials()
 
     @cached_property
     def _client(self) -> GridFSBucket:

great_ai/src/great_ai/large_file/large_file/large_file_s3.py

@@ -66,6 +66,7 @@ class LargeFileS3(LargeFile):
         cls.secret_access_key = aws_secret_access_key
         cls.bucket_name = large_files_bucket_name
         cls.endpoint_url = aws_endpoint_url
+        super().configure_credentials()
 
     @cached_property
     def _client(self) -> boto3.client:
@@ -99,7 +100,6 @@ class LargeFileS3(LargeFile):
                     name=o["Key"].split(S3_NAME_VERSION_SEPARATOR)[0],
                     version=int(o["Key"].split(S3_NAME_VERSION_SEPARATOR)[-1]),
                     remote_path=o["Key"],
-                    origin="s3",
                 )
                 for o in found_objects["Contents"]
                 if o["Key"].split(S3_NAME_VERSION_SEPARATOR)[0] == self._name

great_ai/src/great_ai/large_file/models/data_instance.py

@@ -1,4 +1,4 @@
-from typing import Any, Literal
+from typing import Any
 
 from pydantic import BaseModel
 
@@ -7,4 +7,3 @@ class DataInstance(BaseModel):
     name: str
     version: int
     remote_path: Any
-    origin: Literal["filesystem", "mongodb", "s3"]