Move pipelines

download_deprivation_data.py

@@ -1,64 +0,0 @@
-#!/usr/bin/env python3
-"""Download IoD2025 Deprivation Scores and convert to Parquet."""
-
-import httpx
-import polars as pl
-from pathlib import Path
-
-URL = "https://assets.publishing.service.gov.uk/media/691ded34513046b952c500bd/File_5_IoD2025_Scores_for_the_Indices_of_Deprivation.xlsx"
-
-BASE_DATA_PATH = Path("./data_sources")
-BASE_DATA_PATH.mkdir(exist_ok=True)
-XLSX_PATH = BASE_DATA_PATH / "IoD2025_Scores.xlsx"
-PARQUET_PATH = BASE_DATA_PATH / "IoD2025_Scores.parquet"
-
-
-def download_file(url: str, output_path: Path) -> None:
-    """Download file from URL."""
-    print(f"Downloading from {url}...")
-    with httpx.stream("GET", url, follow_redirects=True, timeout=60) as response:
-        response.raise_for_status()
-        total = int(response.headers.get("content-length", 0))
-        downloaded = 0
-        with open(output_path, "wb") as f:
-            for chunk in response.iter_bytes(chunk_size=8192):
-                f.write(chunk)
-                downloaded += len(chunk)
-                if total:
-                    print(
-                        f"\rDownloaded {downloaded / 1024 / 1024:.1f} MB / {total / 1024 / 1024:.1f} MB",
-                        end="",
-                    )
-    print(f"\nSaved to {output_path}")
-
-
-def convert_to_parquet(xlsx_path: Path, parquet_path: Path) -> None:
-    """Convert Excel sheet 2 to Parquet."""
-    print("Reading Excel file (sheet 2)...")
-
-    # Read the 2nd sheet (index 1) - IoD2025 Scores
-    df = pl.read_excel(
-        xlsx_path,
-        sheet_id=2,  # 1-indexed, so 2 = second sheet
-    )
-
-    print(f"Shape: {df.shape}")
-    print(f"Columns: {df.columns}")
-
-    df.write_parquet(parquet_path, compression="zstd")
-    print(f"Saved to {parquet_path}")
-    print(f"Excel size: {xlsx_path.stat().st_size / 1024 / 1024:.1f} MB")
-    print(f"Parquet size: {parquet_path.stat().st_size / 1024 / 1024:.1f} MB")
-
-
-def main() -> None:
-    if not XLSX_PATH.exists():
-        download_file(URL, XLSX_PATH)
-    else:
-        print(f"Excel file already exists at {XLSX_PATH}, skipping download")
-
-    convert_to_parquet(XLSX_PATH, PARQUET_PATH)
-
-
-if __name__ == "__main__":
-    main()