andras/perfect-postcode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

Stop doing manual caching

Browse source
This commit is contained in:
Andras Schmelczer 2026-01-31 10:18:31 +00:00
parent 5c39f31283
commit 0cea9b873c
6 changed files with 82 additions and 191 deletions
Download patch file Download diff file Expand all files Collapse all files

21
pipeline/download/deprivation_data.py
View file

@ -1,14 +1,11 @@
import argparse
import httpx
import polars as pl
from pathlib import Path
import tempfile
URL = "https://assets.publishing.service.gov.uk/media/691ded34513046b952c500bd/File_5_IoD2025_Scores_for_the_Indices_of_Deprivation.xlsx"
BASE_DATA_PATH = Path("./data_sources")
BASE_DATA_PATH.mkdir(exist_ok=True)
XLSX_PATH = BASE_DATA_PATH / "IoD2025_Scores.xlsx"
PARQUET_PATH = BASE_DATA_PATH / "IoD2025_Scores.parquet"
def download_file(url: str, output_path: Path) -> None:
with httpx.stream("GET", url, follow_redirects=True, timeout=60) as response:
@ -41,17 +38,17 @@ def convert_to_parquet(xlsx_path: Path, parquet_path: Path) -> None:
df.write_parquet(parquet_path, compression="zstd")
print(f"Saved to {parquet_path}")
print(f"Excel size: {xlsx_path.stat().st_size / 1024 / 1024:.1f} MB")
print(f"Parquet size: {parquet_path.stat().st_size / 1024 / 1024:.1f} MB")
def main() -> None:
if not XLSX_PATH.exists():
download_file(URL, XLSX_PATH)
else:
print(f"Excel file already exists at {XLSX_PATH}, skipping download")
parser = argparse.ArgumentParser(description="Download and convert Index of Deprivation data")
parser.add_argument("--output", type=Path, required=True, help="Output parquet file path")
args = parser.parse_args()
convert_to_parquet(XLSX_PATH, PARQUET_PATH)
with tempfile.TemporaryDirectory() as cache_dir:
xlsx_path = Path(cache_dir) / "IoD2025_Scores.xlsx"
download_file(URL, xlsx_path)
convert_to_parquet(xlsx_path, args.output)
if __name__ == "__main__":