Document LargeFiles

docs/reference/large-file.md

@@ -4,6 +4,8 @@
 from great_ai.large_file import *
 ```
 
+For more details about using LargeFiles, check out the [how to guide](/how-to-guides/large-file/).
+
 ::: great_ai.large_file.LargeFileLocal
     options:
         show_root_heading: true

great_ai/large_file/large_file/large_file_base.py

@@ -21,31 +21,26 @@ ARCHIVE_EXTENSION = ".tar.gz"
 
 
 class LargeFileBase(ABC):
-    """
+    """Base for LargeFile implementations with different backends.
-    Store large files remotely. Use local cache for speed up.
+
+    Store large files remotely using the familiar API of `open()`. With built-in
+    versioning, pruning and local cache.
+
+    By default, files are stored in the ".cache" folder and the least recently used is
+    deleted after the overall size reaches 30 GBs.
 
     Examples:
+        >>> LargeFileBase.cache_path = Path(".cache")
 
-    ```
+        >>> LargeFileBase.max_cache_size = "30GB"
-    with LargeFile("test.txt", "w", keep_last_n=3) as f:
-        for i in range(1000000):
-            f.write('test\n')
 
-    with LargeFile("test.txt", "r") as f:
+    Attributes:
-        print(f.readlines()[0])
+        initialized: Tell whether `configure_credentials` or
-
+            `configure_credentials_from_file` has been already called.
-    path_to_cached_text_file = LargeFile("test.txt", version=0).get()
+        cache_path: Storage location for cached files.
-    ```
+        max_cache_size: Delete files until the folder at `cache_path` is smaller than
-
+            this value. Examples: "5 GB", "10MB", "0.3 TB". Set to `None` for no
-    By default, files are stored in the ".cache" folder and the
+            automatic cache-pruning.
-    least recently use is deleted after the overall size reaches 30 GBs.
-
-    Change it with the following properties.
-
-    ```
-    LargeFile.cache_path = Path(".cache")
-    LargeFile.max_cache_size = "30GB"
-    ```
     """
 
     initialized = False
@@ -86,12 +81,19 @@ class LargeFileBase(ABC):
         cls,
         secrets: Union[Path, str, ConfigFile],
     ) -> None:
+        """Load file and feed its content to `configure_credentials`.
+
+        Extra keys are ignored.
+        """
+
         if not isinstance(secrets, ConfigFile):
             secrets = ConfigFile(secrets)
         cls.configure_credentials(**{k.lower(): v for k, v in secrets.items()})
 
     @classmethod
     def configure_credentials(cls, **kwargs: str) -> None:
+        """Configure required credentials for the LargeFile backend."""
+
         cls.initialized = True
 
     def __enter__(self) -> IO:
@@ -135,16 +137,22 @@ class LargeFileBase(ABC):
 
         return False
 
-    @property
-    def _local_name(self) -> str:
-        return f"{self._name}{CACHE_NAME_VERSION_SEPARATOR}{self.version}"
-
     @property
     def version(self) -> int:
+        """Numeric version of the file proxied by this LargeFile instance."""
+
         return cast(int, self._version)
 
     @lru_cache(1)
     def get(self, hide_progress: bool = False) -> Path:
+        """Return path to the proxy of a file (or directory).
+
+        If not available in the local cache, an attempt is made to download it.
+
+        Args:
+            hide_progress: Do not show a progress update after each 10% of progress.
+        """
+
         remote_path = next(
             i.remote_path for i in self._instances if i.version == self._version
         )
@@ -171,6 +179,14 @@ class LargeFileBase(ABC):
         return destination
 
     def push(self, path: Union[Path, str], hide_progress: bool = False) -> None:
+        """Upload a file (or directory) as a new version of `key`.
+
+        The file/directory is compressed before upload.
+
+        Args:
+            hide_progress: Do not show a progress update after each 10% of progress.
+        """
+
         if isinstance(path, str):
             path = Path(path)
 
@@ -208,9 +224,26 @@ class LargeFileBase(ABC):
         self.clean_up()
 
     def delete(self) -> None:
+        """Delete all versions of the files under this `key`."""
+
         self._keep_last_n = 0
         self._delete_old_remote_versions()
 
+    @property
+    def versions_pretty(self) -> str:
+        """Formatted string of all available versions."""
+        return ", ".join((str(i.version) for i in self._instances))
+
+    def clean_up(self) -> None:
+        """Delete local and remote versions according to currently set cache and retention policy."""
+
+        self._delete_old_remote_versions()
+        self._prune_cache()
+
+    @property
+    def _local_name(self) -> str:
+        return f"{self._name}{CACHE_NAME_VERSION_SEPARATOR}{self.version}"
+
     def _find_instances(self) -> None:
         if self._cache_only_mode:
             self._instances = self._find_instances_from_cache()
@@ -269,14 +302,6 @@ class LargeFileBase(ABC):
         else:
             raise ValueError("Unsupported file mode.")
 
-    @property
-    def versions_pretty(self) -> str:
-        return ", ".join((str(i.version) for i in self._instances))
-
-    def clean_up(self) -> None:
-        self._delete_old_remote_versions()
-        self._prune_cache()
-
     def _prune_cache(self) -> None:
         self.cache_path.mkdir(parents=True, exist_ok=True)
 

great_ai/large_file/large_file/large_file_local.py

@@ -9,6 +9,30 @@ logger = get_logger("large_file")
 
 
 class LargeFileLocal(LargeFileBase):
+    """LargeFile implementation using local filesystem as a backend.
+
+    Store large files remotely using the familiar API of `open()`. With built-in
+    versioning, pruning and local cache.
+
+    IMPORTANT: If LargeFileLocal.max_cache_size is too small, it won't be enough to
+    store all your files and they can end up deleted.
+
+    See parent for more details.
+
+    Examples:
+        >>> LargeFileLocal.cache_path = Path(".cache")
+
+        >>> LargeFileLocal.max_cache_size = "30GB"
+
+        >>> with LargeFileLocal("my_test.txt", "w", keep_last_n=2) as f:
+        ...     f.write('test')
+        4
+
+        >>> with LargeFileLocal("my_test.txt") as f:
+        ...     print(f.read())
+        test
+    """
+
     def __init__(
         self,
         name: str,
@@ -40,6 +64,7 @@ class LargeFileLocal(LargeFileBase):
     def _download(
         self, remote_path: Any, local_path: Path, hide_progress: bool
     ) -> None:
+        # This will never be called because the file must be in the cache
         raise NotImplementedError()
 
     def _upload(self, local_path: Path, hide_progress: bool) -> None:

great_ai/large_file/large_file/large_file_mongo.py

@@ -18,6 +18,14 @@ MONGO_NAME_VERSION_SEPARATOR = "_"
 
 
 class LargeFileMongo(LargeFileBase):
+    """LargeFile implementation using GridFS (MongoDB) as a backend.
+
+    Store large files remotely using the familiar API of `open()`. With built-in
+    versioning, pruning and local cache.
+
+    See parent for more details.
+    """
+
     mongo_connection_string = None
     mongo_database = None
 

great_ai/large_file/large_file/large_file_s3.py

@@ -16,31 +16,12 @@ S3_NAME_VERSION_SEPARATOR = "/"
 
 
 class LargeFileS3(LargeFileBase):
-    """
+    """LargeFile implementation using S3-compatible storage as a backend.
-    Store large files in S3. Use local cache for speed up.
 
-    Examples:
+    Store large files remotely using the familiar API of `open()`. With built-in
+    versioning, pruning and local cache.
 
-    ```
+    See parent for more details.
-    with LargeFile("test.txt", "w", keep_last_n=3) as f:
-        for i in range(1000000):
-            f.write('test\n')
-
-    with LargeFile("test.txt", "r") as f:
-        print(f.readlines()[0])
-
-    path_to_cached_text_file = LargeFile("test.txt", version=0).get()
-    ```
-
-    By default, files are stored in the ".cache" folder and the
-    least recently use is deleted after the overall size reaches 30 GBs.
-
-    Change it with the following properties.
-
-    ```
-    LargeFile.cache_path = Path(".cache")
-    LargeFile.max_cache_size = "30GB"
-    ```
     """
 
     region_name = None