import argparse import tempfile import polars as pl from pathlib import Path from pipeline.local_temp import local_tmp_dir from pipeline.utils import download URL = "https://assets.publishing.service.gov.uk/media/691ded34513046b952c500bd/File_5_IoD2025_Scores_for_the_Indices_of_Deprivation.xlsx" def convert_to_parquet(xlsx_path: Path, parquet_path: Path) -> None: print("Reading Excel file (sheet 2)...") # Read the 2nd sheet (index 1) - IoD2025 Scores df = pl.read_excel( xlsx_path, sheet_id=2, # 1-indexed, so 2 = second sheet ) print(f"Shape: {df.shape}") print(f"Columns: {df.columns}") df.write_parquet(parquet_path, compression="zstd") print(f"Saved to {parquet_path}") def main() -> None: parser = argparse.ArgumentParser( description="Download and convert Index of Deprivation data" ) parser.add_argument( "--output", type=Path, required=True, help="Output parquet file path" ) args = parser.parse_args() with tempfile.TemporaryDirectory(dir=local_tmp_dir()) as cache_dir: xlsx_path = Path(cache_dir) / "IoD2025_Scores.xlsx" download(URL, xlsx_path, timeout=60) convert_to_parquet(xlsx_path, args.output) if __name__ == "__main__": main()