Move files

great_ai/large_file/helper/__init__.py

@@ -0,0 +1,2 @@
+from .human_readable_to_byte import human_readable_to_byte
+from .progress_bar import DownloadProgressBar, UploadProgressBar

great_ai/large_file/helper/bytes_to_megabytes.py

@@ -0,0 +1,2 @@
+def bytes_to_megabytes(bytes: int) -> str:
+    return f"{round(bytes / 1000 / 1000, 2):.2f}"

great_ai/large_file/helper/human_readable_to_byte.py

@@ -0,0 +1,31 @@
+import re
+
+
+def human_readable_to_byte(size: str) -> int:
+    """Case is ignored, kb, kB, Kb, and KB are all treated as kilobyte."""
+
+    if size.strip() == "0":
+        return 0
+
+    possible_units = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
+    units_re = "|".join(possible_units)
+    regex = re.compile(
+        rf"""
+        \s*                         # trim
+        (?P<scalar>\d+(.\d+)?)      # get scalar, it might be a float
+        \s*                         # ignore optional whitespace
+        (?P<unit>{units_re})        # capture the unit
+    """,
+        flags=re.VERBOSE | re.IGNORECASE,
+    )
+
+    match = regex.match(size)
+    if not match:
+        raise ValueError(f'Could not find values in "{size}"')
+
+    results = match.groupdict()
+
+    scalar = float(results["scalar"])
+    idx = possible_units.index(results["unit"].upper())
+    factor = 1024**idx
+    return round(scalar * factor)

great_ai/large_file/helper/progress_bar.py

@@ -0,0 +1,51 @@
+import os
+import threading
+from logging import Logger
+from pathlib import Path
+
+from .bytes_to_megabytes import bytes_to_megabytes
+
+
+class ProgressBar:
+    min_progress_percentage_change = 10
+
+    def __init__(self, file_size: int, logger: Logger, prefix: str):
+        self._file_size = file_size
+        self._logger = logger
+        self._prefix = prefix
+
+        self._last_percentage: float = 0
+        self._seen_so_far = 0
+        self._lock = threading.Lock()
+
+    def __call__(self, bytes_amount: int) -> None:
+        with self._lock:
+            self._seen_so_far += bytes_amount
+            percentage = (self._seen_so_far / float(self._file_size)) * 100
+
+            if (
+                percentage != 100
+                and percentage - self._last_percentage
+                < self.min_progress_percentage_change
+            ):
+                return
+
+            self._last_percentage += self.min_progress_percentage_change
+
+            file_size_mb = bytes_to_megabytes(self._file_size)
+            seen_so_far_mb = bytes_to_megabytes(self._seen_so_far)
+            progress = seen_so_far_mb.rjust(len(file_size_mb))
+            self._logger.info(
+                f"{self._prefix} {progress}/{file_size_mb} MB ({percentage:.1f}%)"
+            )
+
+
+class DownloadProgressBar(ProgressBar):
+    def __init__(self, name: str, size: int, logger: Logger):
+        super().__init__(file_size=size, logger=logger, prefix=f"Downloading {name}")
+
+
+class UploadProgressBar(ProgressBar):
+    def __init__(self, path: Path, logger: Logger):
+        size = os.path.getsize(path)
+        super().__init__(file_size=size, logger=logger, prefix=f"Uploading {path.name}")