Extract common download utils

pipeline/download/deprivation_data.py

@@ -1,29 +1,13 @@
 import argparse
-import httpx
+import tempfile
 import polars as pl
 from pathlib import Path
-import tempfile
+
+from pipeline.utils import download
 
 URL = "https://assets.publishing.service.gov.uk/media/691ded34513046b952c500bd/File_5_IoD2025_Scores_for_the_Indices_of_Deprivation.xlsx"
 
 
-def download_file(url: str, output_path: Path) -> None:
-    with httpx.stream("GET", url, follow_redirects=True, timeout=60) as response:
-        response.raise_for_status()
-        total = int(response.headers.get("content-length", 0))
-        downloaded = 0
-        with open(output_path, "wb") as f:
-            for chunk in response.iter_bytes(chunk_size=8192):
-                f.write(chunk)
-                downloaded += len(chunk)
-                if total:
-                    print(
-                        f"\rDownloaded {downloaded / 1024 / 1024:.1f} MB / {total / 1024 / 1024:.1f} MB",
-                        end="",
-                    )
-    print(f"\nSaved to {output_path}")
-
-
 def convert_to_parquet(xlsx_path: Path, parquet_path: Path) -> None:
     print("Reading Excel file (sheet 2)...")
 
@@ -51,7 +35,7 @@ def main() -> None:
 
     with tempfile.TemporaryDirectory() as cache_dir:
         xlsx_path = Path(cache_dir) / "IoD2025_Scores.xlsx"
-        download_file(URL, xlsx_path)
+        download(URL, xlsx_path, timeout=60)
         convert_to_parquet(xlsx_path, args.output)