add deprerivation (affluency) data

README.md

@@ -45,12 +45,27 @@ task prepare
 
 ## Data Sources
 
-- [Price Paid](https://www.gov.uk/government/collections/price-paid-data)
+- [Price Paid](https://www.gov.uk/government/statistical-data-sets/price-paid-data-downloads)
 
 - [English Indices of Deprevation 2025](https://www.gov.uk/government/statistics/english-indices-of-deprivation-2025)
+  - The English Indices of Deprivation (IoD25) measure relative levels of deprivation in 33,755 small areas or neighbourhoods, called Lower-layer Super Output Areas (LSOAs), in England.
 
 - [Population by Ethnicity and Region 2021](https://www.ethnicity-facts-figures.service.gov.uk/uk-population-by-ethnicity/national-and-regional-populations/regional-ethnic-diversity/latest/#download-the-data)
 
 - [Crime](https://data.police.uk/data/)
 
 - [Postcode -> GPS](https://www.arcgis.com/sharing/rest/content/items/077631e063eb4e1ab43575d01381ec33/data)
+
+Nice to haves?
+
+- <https://www.gov.uk/government/statistics/english-indices-of-deprivation-2025> - file 8!
+
+## Backend Data Sources
+
+- [UK Regions](https://www.ons.gov.uk/methodology/geography/ukgeographies/statisticalgeographies)
+![alt text](image.png)
+  - [Lower Level Super Output Area (LSOAs)](https://communitiesopendata-communities.hub.arcgis.com/datasets/4da63019f25546aa92a922a5ea682950_0/explore?location=51.506508%2C-0.041229%2C13.82)
+  - [Local Authority (Lower Tier)](https://communitiesopendata-communities.hub.arcgis.com/datasets/f3954cc3ded54a08b6fffbb361f5ee76_0/explore?location=52.522271%2C-2.489913%2C7.17)
+  - [Local Autheority (Upper Tier)](https://communitiesopendata-communities.hub.arcgis.com/datasets/6e8edb2974da4834bbafa09644a5b02d_0/explore?location=52.684195%2C-2.489482%2C7.17)
+- [Open Geography](https://geoportal.statistics.gov.uk/)
+- [CommunitiesOpenData](https://communitiesopendata-communities.hub.arcgis.com/)

download_deprivation_data.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+"""Download IoD2025 Deprivation Scores and convert to Parquet."""
+
+import httpx
+import polars as pl
+from pathlib import Path
+
+URL = "https://assets.publishing.service.gov.uk/media/691ded34513046b952c500bd/File_5_IoD2025_Scores_for_the_Indices_of_Deprivation.xlsx"
+
+BASE_DATA_PATH = Path("./data_sources")
+BASE_DATA_PATH.mkdir(exist_ok=True)
+XLSX_PATH = BASE_DATA_PATH / "IoD2025_Scores.xlsx"
+PARQUET_PATH = BASE_DATA_PATH / "IoD2025_Scores.parquet"
+
+
+def download_file(url: str, output_path: Path) -> None:
+    """Download file from URL."""
+    print(f"Downloading from {url}...")
+    with httpx.stream("GET", url, follow_redirects=True, timeout=60) as response:
+        response.raise_for_status()
+        total = int(response.headers.get("content-length", 0))
+        downloaded = 0
+        with open(output_path, "wb") as f:
+            for chunk in response.iter_bytes(chunk_size=8192):
+                f.write(chunk)
+                downloaded += len(chunk)
+                if total:
+                    print(f"\rDownloaded {downloaded / 1024 / 1024:.1f} MB / {total / 1024 / 1024:.1f} MB", end="")
+    print(f"\nSaved to {output_path}")
+
+
+def convert_to_parquet(xlsx_path: Path, parquet_path: Path) -> None:
+    """Convert Excel sheet 2 to Parquet."""
+    print("Reading Excel file (sheet 2)...")
+
+    # Read the 2nd sheet (index 1) - IoD2025 Scores
+    df = pl.read_excel(
+        xlsx_path,
+        sheet_id=2,  # 1-indexed, so 2 = second sheet
+    )
+
+    print(f"Shape: {df.shape}")
+    print(f"Columns: {df.columns}")
+
+    df.write_parquet(parquet_path, compression="zstd")
+    print(f"Saved to {parquet_path}")
+    print(f"Excel size: {xlsx_path.stat().st_size / 1024 / 1024:.1f} MB")
+    print(f"Parquet size: {parquet_path.stat().st_size / 1024 / 1024:.1f} MB")
+
+
+def main() -> None:
+    if not XLSX_PATH.exists():
+        download_file(URL, XLSX_PATH)
+    else:
+        print(f"Excel file already exists at {XLSX_PATH}, skipping download")
+
+    convert_to_parquet(XLSX_PATH, PARQUET_PATH)
+
+
+if __name__ == "__main__":
+    main()