SPlit up

pipeline/download/map_assets.py

@@ -3,6 +3,7 @@ import base64
 import json
 import re
 import sys
+import time
 import urllib.request
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from io import BytesIO
@@ -120,18 +121,29 @@ def collect_twemoji_codes() -> list[str]:
     return sorted({f"{ord(e[0]):x}" for e in emojis})
 
 
+DOWNLOAD_ATTEMPTS = 3
+RETRY_BACKOFF_S = 2.0
+
+
 def download_file(url: str, dest: Path) -> tuple[bool, str]:
-    """Download a single file. Returns (success, url)."""
+    """Download a single file, retrying transient errors. Returns (success, url)."""
     dest.parent.mkdir(parents=True, exist_ok=True)
-    try:
-        urllib.request.urlretrieve(url, dest)
-        return True, url
-    except urllib.error.HTTPError as e:
-        print(f"  {e.code} {url}", file=sys.stderr)
-        return False, url
-    except Exception as e:
-        print(f"  ERROR {url}: {e}", file=sys.stderr)
-        return False, url
+    for attempt in range(DOWNLOAD_ATTEMPTS):
+        if attempt:
+            time.sleep(RETRY_BACKOFF_S * 2 ** (attempt - 1))
+        try:
+            urllib.request.urlretrieve(url, dest)
+            return True, url
+        except urllib.error.HTTPError as e:
+            # 4xx is a permanent answer (bad glyph range / missing emoji);
+            # retrying won't change it.
+            if 400 <= e.code < 500:
+                print(f"  {e.code} {url}", file=sys.stderr)
+                return False, url
+            print(f"  {e.code} {url} (attempt {attempt + 1})", file=sys.stderr)
+        except Exception as e:
+            print(f"  ERROR {url}: {e} (attempt {attempt + 1})", file=sys.stderr)
+    return False, url
 
 
 def download_text(url: str) -> str:
@@ -389,37 +401,38 @@ def main():
         url = f"{POI_ICON_BASE}/{icon_path}"
         tasks.append((url, poi_icons_dir / icon_path))
 
-    # Skip already-downloaded files
-    remaining = [(url, dest) for url, dest in tasks]
-
-    print(f"Downloading {len(remaining) + len(DERIVED_POI_ICON_PATHS)} assets")
+    print(f"Downloading {len(tasks) + len(DERIVED_POI_ICON_PATHS)} assets")
 
     ok = 0
-    fail = 0
+    failed_urls: list[str] = []
     with ThreadPoolExecutor(max_workers=20) as pool:
-        futures = {
-            pool.submit(download_file, url, dest): url for url, dest in remaining
-        }
+        futures = {pool.submit(download_file, url, dest): url for url, dest in tasks}
         for future in as_completed(futures):
             success, url = future.result()
             if success:
                 ok += 1
             else:
-                fail += 1
+                failed_urls.append(url)
 
     for kind, source_path, dest_path in DERIVED_POI_ICON_PATHS:
-        success, _url = download_derived_poi_icon(
+        success, url = download_derived_poi_icon(
             kind, source_path, poi_icons_dir / dest_path
         )
         if success:
             ok += 1
         else:
-            fail += 1
+            failed_urls.append(url)
 
     crop_poi_svg_icons(poi_icons_dir)
     inject_townhall_sprite(sprites_dir)
 
-    print(f"Done: {ok} downloaded, {fail} failed")
+    print(f"Done: {ok} downloaded, {len(failed_urls)} failed")
+    if failed_urls:
+        # A partial asset bundle (missing font ranges, sprites, icons) renders
+        # broken labels at runtime but would otherwise satisfy the make stamp.
+        for url in failed_urls:
+            print(f"  missing: {url}", file=sys.stderr)
+        sys.exit(1)
 
 
 if __name__ == "__main__":