53 lines
1.9 KiB
Python
53 lines
1.9 KiB
Python
import argparse
|
|
import tempfile
|
|
import polars as pl
|
|
from pathlib import Path
|
|
|
|
from pipeline.utils import download, extract_zip
|
|
|
|
URL = "https://www.arcgis.com/sharing/rest/content/items/36b718ad00de49afb9ad364f8b815b9e/data"
|
|
|
|
|
|
def convert_to_parquet(data_path: Path, parquet_path: Path) -> None:
|
|
# Classification code columns (ruc21ind, oac11ind, imd20ind) look numeric
|
|
# in early rows but contain string codes like "UN1" (Unclassified) later
|
|
# on. Force them to String to avoid mid-stream dtype inference failures.
|
|
# Note: NSPL renames these year suffixes as new releases roll in (e.g.
|
|
# Feb 2026 bumped oac from oac21ind → oac11ind, imd from imd19ind →
|
|
# imd20ind), so keep this dict in sync with the current CSV headers —
|
|
# polars silently ignores overrides for missing columns, masking drift.
|
|
df = pl.scan_csv(
|
|
data_path / "Data/NSPL_FEB_2026_UK.csv",
|
|
try_parse_dates=True,
|
|
schema_overrides={
|
|
"ruc21ind": pl.String,
|
|
"oac11ind": pl.String,
|
|
"imd20ind": pl.String,
|
|
},
|
|
)
|
|
print(f"Columns: {df.collect_schema().names()}")
|
|
parquet_path.parent.mkdir(parents=True, exist_ok=True)
|
|
df.sink_parquet(parquet_path, compression="zstd")
|
|
print(f"Saved to {parquet_path}")
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(
|
|
description="Download and convert ArcGIS postcode data"
|
|
)
|
|
parser.add_argument(
|
|
"--output", type=Path, required=True, help="Output parquet file path"
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
with tempfile.TemporaryDirectory() as cache_dir:
|
|
download_path = Path(cache_dir) / "arcgis_data.zip"
|
|
extract_path = Path(cache_dir) / "arcgis_extracted"
|
|
|
|
download(URL, download_path)
|
|
extract_zip(download_path, extract_path)
|
|
convert_to_parquet(extract_path, args.output)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|