59 lines
1.8 KiB
Python
59 lines
1.8 KiB
Python
import argparse
|
|
import httpx
|
|
import polars as pl
|
|
from pathlib import Path
|
|
import tempfile
|
|
|
|
URL = "https://assets.publishing.service.gov.uk/media/691ded34513046b952c500bd/File_5_IoD2025_Scores_for_the_Indices_of_Deprivation.xlsx"
|
|
|
|
|
|
def download_file(url: str, output_path: Path) -> None:
|
|
with httpx.stream("GET", url, follow_redirects=True, timeout=60) as response:
|
|
response.raise_for_status()
|
|
total = int(response.headers.get("content-length", 0))
|
|
downloaded = 0
|
|
with open(output_path, "wb") as f:
|
|
for chunk in response.iter_bytes(chunk_size=8192):
|
|
f.write(chunk)
|
|
downloaded += len(chunk)
|
|
if total:
|
|
print(
|
|
f"\rDownloaded {downloaded / 1024 / 1024:.1f} MB / {total / 1024 / 1024:.1f} MB",
|
|
end="",
|
|
)
|
|
print(f"\nSaved to {output_path}")
|
|
|
|
|
|
def convert_to_parquet(xlsx_path: Path, parquet_path: Path) -> None:
|
|
print("Reading Excel file (sheet 2)...")
|
|
|
|
# Read the 2nd sheet (index 1) - IoD2025 Scores
|
|
df = pl.read_excel(
|
|
xlsx_path,
|
|
sheet_id=2, # 1-indexed, so 2 = second sheet
|
|
)
|
|
|
|
print(f"Shape: {df.shape}")
|
|
print(f"Columns: {df.columns}")
|
|
|
|
df.write_parquet(parquet_path, compression="zstd")
|
|
print(f"Saved to {parquet_path}")
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(
|
|
description="Download and convert Index of Deprivation data"
|
|
)
|
|
parser.add_argument(
|
|
"--output", type=Path, required=True, help="Output parquet file path"
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
with tempfile.TemporaryDirectory() as cache_dir:
|
|
xlsx_path = Path(cache_dir) / "IoD2025_Scores.xlsx"
|
|
download_file(URL, xlsx_path)
|
|
convert_to_parquet(xlsx_path, args.output)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|