From d0c5f5b2a46255442e1ac03c07a1e4ae236fcaa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A1s=20Schmelczer?= Date: Sat, 16 Jul 2022 16:22:22 +0200 Subject: [PATCH] Document LargeFiles --- docs/reference/large-file.md | 2 + .../large_file/large_file/large_file_base.py | 91 ++++++++++++------- .../large_file/large_file/large_file_local.py | 25 +++++ .../large_file/large_file/large_file_mongo.py | 8 ++ .../large_file/large_file/large_file_s3.py | 27 +----- 5 files changed, 97 insertions(+), 56 deletions(-) diff --git a/docs/reference/large-file.md b/docs/reference/large-file.md index 31818f1..37b5706 100644 --- a/docs/reference/large-file.md +++ b/docs/reference/large-file.md @@ -4,6 +4,8 @@ from great_ai.large_file import * ``` +For more details about using LargeFiles, check out the [how to guide](/how-to-guides/large-file/). + ::: great_ai.large_file.LargeFileLocal options: show_root_heading: true diff --git a/great_ai/large_file/large_file/large_file_base.py b/great_ai/large_file/large_file/large_file_base.py index 4f412e1..7264520 100644 --- a/great_ai/large_file/large_file/large_file_base.py +++ b/great_ai/large_file/large_file/large_file_base.py @@ -21,31 +21,26 @@ ARCHIVE_EXTENSION = ".tar.gz" class LargeFileBase(ABC): - """ - Store large files remotely. Use local cache for speed up. + """Base for LargeFile implementations with different backends. + + Store large files remotely using the familiar API of `open()`. With built-in + versioning, pruning and local cache. + + By default, files are stored in the ".cache" folder and the least recently used is + deleted after the overall size reaches 30 GBs. Examples: + >>> LargeFileBase.cache_path = Path(".cache") - ``` - with LargeFile("test.txt", "w", keep_last_n=3) as f: - for i in range(1000000): - f.write('test\n') + >>> LargeFileBase.max_cache_size = "30GB" - with LargeFile("test.txt", "r") as f: - print(f.readlines()[0]) - - path_to_cached_text_file = LargeFile("test.txt", version=0).get() - ``` - - By default, files are stored in the ".cache" folder and the - least recently use is deleted after the overall size reaches 30 GBs. - - Change it with the following properties. - - ``` - LargeFile.cache_path = Path(".cache") - LargeFile.max_cache_size = "30GB" - ``` + Attributes: + initialized: Tell whether `configure_credentials` or + `configure_credentials_from_file` has been already called. + cache_path: Storage location for cached files. + max_cache_size: Delete files until the folder at `cache_path` is smaller than + this value. Examples: "5 GB", "10MB", "0.3 TB". Set to `None` for no + automatic cache-pruning. """ initialized = False @@ -86,12 +81,19 @@ class LargeFileBase(ABC): cls, secrets: Union[Path, str, ConfigFile], ) -> None: + """Load file and feed its content to `configure_credentials`. + + Extra keys are ignored. + """ + if not isinstance(secrets, ConfigFile): secrets = ConfigFile(secrets) cls.configure_credentials(**{k.lower(): v for k, v in secrets.items()}) @classmethod def configure_credentials(cls, **kwargs: str) -> None: + """Configure required credentials for the LargeFile backend.""" + cls.initialized = True def __enter__(self) -> IO: @@ -135,16 +137,22 @@ class LargeFileBase(ABC): return False - @property - def _local_name(self) -> str: - return f"{self._name}{CACHE_NAME_VERSION_SEPARATOR}{self.version}" - @property def version(self) -> int: + """Numeric version of the file proxied by this LargeFile instance.""" + return cast(int, self._version) @lru_cache(1) def get(self, hide_progress: bool = False) -> Path: + """Return path to the proxy of a file (or directory). + + If not available in the local cache, an attempt is made to download it. + + Args: + hide_progress: Do not show a progress update after each 10% of progress. + """ + remote_path = next( i.remote_path for i in self._instances if i.version == self._version ) @@ -171,6 +179,14 @@ class LargeFileBase(ABC): return destination def push(self, path: Union[Path, str], hide_progress: bool = False) -> None: + """Upload a file (or directory) as a new version of `key`. + + The file/directory is compressed before upload. + + Args: + hide_progress: Do not show a progress update after each 10% of progress. + """ + if isinstance(path, str): path = Path(path) @@ -208,9 +224,26 @@ class LargeFileBase(ABC): self.clean_up() def delete(self) -> None: + """Delete all versions of the files under this `key`.""" + self._keep_last_n = 0 self._delete_old_remote_versions() + @property + def versions_pretty(self) -> str: + """Formatted string of all available versions.""" + return ", ".join((str(i.version) for i in self._instances)) + + def clean_up(self) -> None: + """Delete local and remote versions according to currently set cache and retention policy.""" + + self._delete_old_remote_versions() + self._prune_cache() + + @property + def _local_name(self) -> str: + return f"{self._name}{CACHE_NAME_VERSION_SEPARATOR}{self.version}" + def _find_instances(self) -> None: if self._cache_only_mode: self._instances = self._find_instances_from_cache() @@ -269,14 +302,6 @@ class LargeFileBase(ABC): else: raise ValueError("Unsupported file mode.") - @property - def versions_pretty(self) -> str: - return ", ".join((str(i.version) for i in self._instances)) - - def clean_up(self) -> None: - self._delete_old_remote_versions() - self._prune_cache() - def _prune_cache(self) -> None: self.cache_path.mkdir(parents=True, exist_ok=True) diff --git a/great_ai/large_file/large_file/large_file_local.py b/great_ai/large_file/large_file/large_file_local.py index 6b31762..c1c5a12 100644 --- a/great_ai/large_file/large_file/large_file_local.py +++ b/great_ai/large_file/large_file/large_file_local.py @@ -9,6 +9,30 @@ logger = get_logger("large_file") class LargeFileLocal(LargeFileBase): + """LargeFile implementation using local filesystem as a backend. + + Store large files remotely using the familiar API of `open()`. With built-in + versioning, pruning and local cache. + + IMPORTANT: If LargeFileLocal.max_cache_size is too small, it won't be enough to + store all your files and they can end up deleted. + + See parent for more details. + + Examples: + >>> LargeFileLocal.cache_path = Path(".cache") + + >>> LargeFileLocal.max_cache_size = "30GB" + + >>> with LargeFileLocal("my_test.txt", "w", keep_last_n=2) as f: + ... f.write('test') + 4 + + >>> with LargeFileLocal("my_test.txt") as f: + ... print(f.read()) + test + """ + def __init__( self, name: str, @@ -40,6 +64,7 @@ class LargeFileLocal(LargeFileBase): def _download( self, remote_path: Any, local_path: Path, hide_progress: bool ) -> None: + # This will never be called because the file must be in the cache raise NotImplementedError() def _upload(self, local_path: Path, hide_progress: bool) -> None: diff --git a/great_ai/large_file/large_file/large_file_mongo.py b/great_ai/large_file/large_file/large_file_mongo.py index 673ac5c..03e9e1e 100644 --- a/great_ai/large_file/large_file/large_file_mongo.py +++ b/great_ai/large_file/large_file/large_file_mongo.py @@ -18,6 +18,14 @@ MONGO_NAME_VERSION_SEPARATOR = "_" class LargeFileMongo(LargeFileBase): + """LargeFile implementation using GridFS (MongoDB) as a backend. + + Store large files remotely using the familiar API of `open()`. With built-in + versioning, pruning and local cache. + + See parent for more details. + """ + mongo_connection_string = None mongo_database = None diff --git a/great_ai/large_file/large_file/large_file_s3.py b/great_ai/large_file/large_file/large_file_s3.py index 0104eba..24dd779 100644 --- a/great_ai/large_file/large_file/large_file_s3.py +++ b/great_ai/large_file/large_file/large_file_s3.py @@ -16,31 +16,12 @@ S3_NAME_VERSION_SEPARATOR = "/" class LargeFileS3(LargeFileBase): - """ - Store large files in S3. Use local cache for speed up. + """LargeFile implementation using S3-compatible storage as a backend. - Examples: + Store large files remotely using the familiar API of `open()`. With built-in + versioning, pruning and local cache. - ``` - with LargeFile("test.txt", "w", keep_last_n=3) as f: - for i in range(1000000): - f.write('test\n') - - with LargeFile("test.txt", "r") as f: - print(f.readlines()[0]) - - path_to_cached_text_file = LargeFile("test.txt", version=0).get() - ``` - - By default, files are stored in the ".cache" folder and the - least recently use is deleted after the overall size reaches 30 GBs. - - Change it with the following properties. - - ``` - LargeFile.cache_path = Path(".cache") - LargeFile.max_cache_size = "30GB" - ``` + See parent for more details. """ region_name = None