SPlit up

pipeline/download/lsoa_children.py

@@ -10,21 +10,19 @@ of the 0-4, 10-14 and 15-19 bands (one fifth per single year of age).
 """
 
 import argparse
-from io import BytesIO
 from pathlib import Path
 
-import httpx
 import polars as pl
 
+from pipeline.utils import ENGLAND_LSOA_COUNT_2021, download_nomis_csv
+
 # NOMIS API: Census 2021 TS007A (age, five-year bands) by LSOA 2021 (TYPE151).
 # c2021_age_19 codes: 1 = 0-4, 2 = 5-9, 3 = 10-14, 4 = 15-19.
-# NOMIS paginates at 25,000 rows by default, so we paginate with recordoffset.
 BASE_URL = (
     "https://www.nomisweb.co.uk/api/v01/dataset/NM_2020_1.data.csv"
     "?date=latest&geography=TYPE151&measures=20100&c2021_age_19=1,2,3,4"
     "&select=GEOGRAPHY_CODE,C2021_AGE_19,OBS_VALUE"
 )
-PAGE_SIZE = 25000
 
 AGE_BAND_COLUMNS = {
     1: "aged_0_4",
@@ -36,24 +34,7 @@ AGE_BAND_COLUMNS = {
 
 def download_and_convert(output_path: Path) -> None:
     print("Downloading Census 2021 LSOA age bands from NOMIS...")
-    frames = []
-    offset = 0
-    while True:
-        url = f"{BASE_URL}&recordoffset={offset}"
-        response = httpx.get(url, follow_redirects=True, timeout=120)
-        response.raise_for_status()
-        if len(response.content) == 0:
-            break
-        chunk = pl.read_csv(BytesIO(response.content))
-        if chunk.height == 0:
-            break
-        frames.append(chunk)
-        print(f"  Fetched {chunk.height} rows (offset={offset})")
-        if chunk.height < PAGE_SIZE:
-            break
-        offset += PAGE_SIZE
-
-    df = pl.concat(frames)
+    df = download_nomis_csv(BASE_URL)
     print(f"Total rows: {df.height}")
 
     result = (
@@ -70,6 +51,11 @@ def download_and_convert(output_path: Path) -> None:
         raise ValueError(f"NOMIS response missing age bands: {missing}")
 
     print(f"England LSOAs: {result.height}")
+    if result.height != ENGLAND_LSOA_COUNT_2021:
+        raise ValueError(
+            f"Expected {ENGLAND_LSOA_COUNT_2021} England LSOAs, "
+            f"got {result.height}: truncated NOMIS download?"
+        )
     for name in AGE_BAND_COLUMNS.values():
         print(f"  {name}: total {result[name].sum():,}")