72 lines
2.1 KiB
Python
72 lines
2.1 KiB
Python
import argparse
|
|
import tempfile
|
|
import zipfile
|
|
import httpx
|
|
import polars as pl
|
|
from pathlib import Path
|
|
from tqdm import tqdm
|
|
|
|
URL = "https://www.arcgis.com/sharing/rest/content/items/077631e063eb4e1ab43575d01381ec33/data"
|
|
|
|
|
|
def download_with_progress(url: str, output_path: Path) -> None:
|
|
with httpx.stream(
|
|
"GET",
|
|
url,
|
|
follow_redirects=True,
|
|
timeout=httpx.Timeout(30.0, read=None),
|
|
) as response:
|
|
response.raise_for_status() # pyright: ignore[reportUnusedCallResult]
|
|
total = int(response.headers.get("content-length", 0))
|
|
|
|
with (
|
|
open(output_path, "wb") as f,
|
|
tqdm(
|
|
total=total,
|
|
unit="B",
|
|
unit_scale=True,
|
|
unit_divisor=1024,
|
|
desc="Downloading",
|
|
) as pbar,
|
|
):
|
|
for chunk in response.iter_bytes(chunk_size=8192):
|
|
f.write(chunk)
|
|
pbar.update(len(chunk))
|
|
return
|
|
|
|
|
|
def extract_zip(zip_path: Path, extract_path: Path) -> None:
|
|
extract_path.mkdir(exist_ok=True)
|
|
|
|
with zipfile.ZipFile(zip_path, "r") as zf:
|
|
zf.extractall(extract_path)
|
|
|
|
|
|
def convert_to_parquet(data_path: Path, parquet_path: Path) -> None:
|
|
df = pl.scan_csv(data_path / "Data/NSPL_MAY_2025_UK.csv", try_parse_dates=True)
|
|
print(f"Columns: {df.collect_schema().names()}")
|
|
parquet_path.parent.mkdir(parents=True, exist_ok=True)
|
|
df.sink_parquet(parquet_path, compression="zstd")
|
|
print(f"Saved to {parquet_path}")
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(
|
|
description="Download and convert ArcGIS postcode data"
|
|
)
|
|
parser.add_argument(
|
|
"--output", type=Path, required=True, help="Output parquet file path"
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
with tempfile.TemporaryDirectory() as cache_dir:
|
|
download_path = Path(cache_dir) / "arcgis_data.zip"
|
|
extract_path = Path(cache_dir) / "arcgis_extracted"
|
|
|
|
download_with_progress(URL, download_path)
|
|
extract_zip(download_path, extract_path)
|
|
convert_to_parquet(extract_path, args.output)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|