andras/great-ai
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases 4 Packages Wiki Activity Actions

Fix key sorting

Browse source
This commit is contained in:
Andras Schmelczer 2022-05-27 12:47:16 +02:00
parent bd41a32d09
commit 8729ce1362
1 changed files with 11 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

21
great_ai/src/great_ai/open_s3/large_file.py
View file

@ -157,7 +157,7 @@ class LargeFile:
@property @property
def version_ids(self) -> List[int]: def version_ids(self) -> List[int]:
return sorted(self._get_version_from_key(key) for key in self._versions) return [self._get_version_from_key(key) for key in self._versions]
@property @property
def version(self) -> int: def version(self) -> int:
@ -271,30 +271,31 @@ class LargeFile:
def _find_versions(self) -> None: def _find_versions(self) -> None:
if self._offline_mode: if self._offline_mode:
self._fetch_versions_from_cache() versions: Union[List[str], List[Path]] = self._fetch_versions_from_cache()
else: else:
self._create_client() self._create_client()
self._fetch_versions_from_s3() versions = self._fetch_versions_from_s3()
self._versions = sorted(versions, key=self._get_version_from_key)
if self._versions: if self._versions:
logger.info(f"Found versions: {self.version_ids}") logger.info(f"Found versions: {self.version_ids}")
else: else:
logger.info("No versions found") logger.info("No versions found")
def _fetch_versions_from_cache(self) -> None: def _fetch_versions_from_cache(self) -> List[Path]:
logger.info(f"Fetching offline versions of {self._name}") logger.info(f"Fetching offline versions of {self._name}")
self._versions = [ return list(self.cache_path.glob(f"{self._local_name}-*"))
path for path in self.cache_path.glob(f"{self._local_name}-*")
]
def _fetch_versions_from_s3(self) -> None: def _fetch_versions_from_s3(self) -> List[str]:
logger.info(f"Fetching online versions of {self._name}") logger.info(f"Fetching online versions of {self._name}")
found_objects = self._client.list_objects_v2( found_objects = self._client.list_objects_v2(
Bucket=self.bucket_name, Prefix=self._name Bucket=self.bucket_name, Prefix=self._name
) )
self._versions = ( return (
sorted(o["Key"] for o in found_objects["Contents"]) [o["Key"] for o in found_objects["Contents"]]
if "Contents" in found_objects if "Contents" in found_objects
else [] else []
) )