lgtm

pipeline/download/development_sites.py

@@ -0,0 +1,302 @@
+"""Download planned/pipeline development sites for the "new developments" layer.
+
+This is the forward-looking "where new homes are coming" signal that complements
+the EPC new-build + Land Registry "first sale" delivery signals already in the
+pipeline. Two national, Open Government Licence v3.0 sources are merged into a
+single coordinate-keyed parquet that the Rust server serves as a map layer:
+
+  - MHCLG Brownfield Land register (statutory LPA brownfield registers, normalised)
+    https://www.planning.data.gov.uk/dataset/brownfield-land
+  - Homes England Land Hub (public land being disposed for development)
+    https://www.gov.uk/government/publications/homes-england-land-hub
+
+Output schema (one row per site):
+    lat, lon, source, name, min_dwellings, max_dwellings, planning_status,
+    permission_type, permission_date, hectares, local_authority, url
+"""
+
+import argparse
+import math
+import re
+import tempfile
+from pathlib import Path
+
+import httpx
+import polars as pl
+from shapely.geometry import shape
+
+from pipeline.local_temp import local_tmp_dir
+from pipeline.utils.download import download
+
+BROWNFIELD_URL = "https://files.planning.data.gov.uk/dataset/brownfield-land.parquet"
+HOMES_ENGLAND_URL = (
+    "https://services-eu1.arcgis.com/yo0w4PgP4XL49bfF/arcgis/rest/services/"
+    "Homes_England_Land_Hub_Sites/FeatureServer/0/query"
+)
+
+# Output column order; doubles as the polars schema so empty inputs still produce
+# a well-typed frame. The Rust loader (data/developments.rs) reads these names.
+SCHEMA: dict[str, pl.DataType] = {
+    "lat": pl.Float64,
+    "lon": pl.Float64,
+    "source": pl.String,
+    "name": pl.String,
+    "min_dwellings": pl.Int32,
+    "max_dwellings": pl.Int32,
+    "planning_status": pl.String,
+    "permission_type": pl.String,
+    "permission_date": pl.String,
+    "hectares": pl.Float64,
+    "local_authority": pl.String,
+    "url": pl.String,
+}
+
+_COORD = r"-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?"
+_POINT_RE = re.compile(rf"POINT\s*\(\s*({_COORD})\s+({_COORD})\s*\)", re.IGNORECASE)
+
+# 1 acre = 0.404686 hectares. The Homes England feed reports area in acres.
+_ACRES_TO_HECTARES = 0.404686
+
+# Great Britain bounding box, used to reject mis-ordered or projected coordinates
+# (e.g. a WKT written "lat lon", or stray British National Grid eastings).
+_LON_MIN, _LON_MAX = -9.0, 2.1
+_LAT_MIN, _LAT_MAX = 49.0, 61.1
+
+
+def _first(row: dict, *keys: str):
+    """First present, non-None value among ``keys`` (case-sensitive)."""
+    for key in keys:
+        if key in row and row[key] is not None:
+            return row[key]
+    return None
+
+
+def _clean_str(value) -> str | None:
+    if value is None:
+        return None
+    text = str(value).strip()
+    return text or None
+
+
+def _to_int(value) -> int | None:
+    if value is None:
+        return None
+    try:
+        number = float(value)
+    except (TypeError, ValueError):
+        return None
+    if not math.isfinite(number):  # NaN or +/-inf (int(inf) raises OverflowError)
+        return None
+    return int(number)
+
+
+def _to_float(value) -> float | None:
+    if value is None:
+        return None
+    try:
+        number = float(value)
+    except (TypeError, ValueError):
+        return None
+    if not math.isfinite(number):  # NaN or +/-inf
+        return None
+    return number
+
+
+def _valid_lonlat(lon: float, lat: float) -> bool:
+    return _LON_MIN <= lon <= _LON_MAX and _LAT_MIN <= lat <= _LAT_MAX
+
+
+def _point_lonlat(row: dict) -> tuple[float, float] | None:
+    """Extract (lon, lat) from a brownfield row's WKT point or lat/lon columns."""
+    wkt = _clean_str(_first(row, "point"))
+    if wkt:
+        match = _POINT_RE.match(wkt)
+        if match:
+            lon, lat = float(match.group(1)), float(match.group(2))
+            if _valid_lonlat(lon, lat):
+                return lon, lat
+    lon = _to_float(_first(row, "longitude", "long", "lng"))
+    lat = _to_float(_first(row, "latitude", "lat"))
+    if lon is not None and lat is not None and _valid_lonlat(lon, lat):
+        return lon, lat
+    return None
+
+
+def _geometry_centroid(geom) -> tuple[float, float] | None:
+    """Centroid (lon, lat) of a GeoJSON geometry already in WGS84."""
+    if not geom:
+        return None
+    try:
+        centroid = shape(geom).centroid
+    except (ValueError, TypeError, AttributeError):
+        return None
+    if centroid.is_empty:
+        return None
+    lon, lat = float(centroid.x), float(centroid.y)
+    return (lon, lat) if _valid_lonlat(lon, lat) else None
+
+
+def _has_dwellings(min_d: int | None, max_d: int | None) -> bool:
+    return (min_d is not None and min_d > 0) or (max_d is not None and max_d > 0)
+
+
+def _brownfield_url(entity) -> str | None:
+    """Canonical per-site page on the Planning Data platform.
+
+    The register's own ``site-plan-url`` is unreliable — many LPAs point every
+    row at a single generic register landing page — so we link to the stable,
+    always-per-site entity page instead.
+    """
+    entity_id = _to_int(entity)
+    if entity_id is None:
+        return None
+    return f"https://www.planning.data.gov.uk/entity/{entity_id}"
+
+
+def _is_residential(use: str | None, capacity: int | None) -> bool:
+    """Keep Homes England sites that will deliver homes."""
+    if capacity is not None and capacity > 0:
+        return True
+    if use is None:
+        return False
+    lowered = use.lower()
+    return any(token in lowered for token in ("resid", "housing", "mixed", "dwelling"))
+
+
+def _frame_from_rows(rows: list[dict]) -> pl.DataFrame:
+    return pl.DataFrame(rows, schema=SCHEMA, orient="row")
+
+
+def brownfield_to_frame(raw: pl.DataFrame) -> pl.DataFrame:
+    """Normalise the brownfield-land register to the unified development schema.
+
+    Drops sites that have left the register (a non-empty ``end-date`` means the
+    site was built out or withdrawn) and sites with no residential capacity, so
+    the layer shows pipeline housing rather than every parcel ever registered.
+    """
+    rows: list[dict] = []
+    for row in raw.iter_rows(named=True):
+        lonlat = _point_lonlat(row)
+        if lonlat is None:
+            continue
+        if _clean_str(_first(row, "end-date", "end_date")):
+            continue
+        min_d = _to_int(_first(row, "minimum-net-dwellings", "minimum_net_dwellings"))
+        max_d = _to_int(_first(row, "maximum-net-dwellings", "maximum_net_dwellings"))
+        if not _has_dwellings(min_d, max_d):
+            continue
+        lon, lat = lonlat
+        rows.append(
+            {
+                "lat": lat,
+                "lon": lon,
+                "source": "brownfield",
+                "name": _clean_str(_first(row, "site-address", "name")),
+                "min_dwellings": min_d,
+                "max_dwellings": max_d,
+                "planning_status": _clean_str(
+                    _first(row, "planning-permission-status")
+                ),
+                "permission_type": _clean_str(_first(row, "planning-permission-type")),
+                "permission_date": _clean_str(_first(row, "planning-permission-date")),
+                "hectares": _to_float(_first(row, "hectares")),
+                "local_authority": _clean_str(_first(row, "organisation")),
+                "url": _brownfield_url(_first(row, "entity")),
+            }
+        )
+    return _frame_from_rows(rows)
+
+
+def homes_england_to_frame(features: list[dict]) -> pl.DataFrame:
+    """Normalise Homes England Land Hub GeoJSON features to the unified schema."""
+    rows: list[dict] = []
+    for feature in features:
+        props = feature.get("properties") or {}
+        lonlat = _geometry_centroid(feature.get("geometry"))
+        if lonlat is None:
+            continue
+        capacity = _to_int(
+            _first(props, "Housing_Capacity", "HousingCapacity", "Units", "Homes")
+        )
+        use = _clean_str(_first(props, "Proposed_Use"))
+        if not _is_residential(use, capacity):
+            continue
+        lon, lat = lonlat
+        # The feed reports area in acres (`Gross_Area__Acres_`); convert to hectares
+        # so the column is consistent with the brownfield register.
+        acres = _to_float(_first(props, "Gross_Area__Acres_"))
+        hectares = round(acres * _ACRES_TO_HECTARES, 4) if acres is not None else None
+        rows.append(
+            {
+                "lat": lat,
+                "lon": lon,
+                "source": "homes-england",
+                "name": _clean_str(_first(props, "Parcel_Name", "Site_Reference")),
+                "min_dwellings": None,
+                "max_dwellings": capacity,
+                "planning_status": _clean_str(_first(props, "Planning_Status")),
+                "permission_type": None,
+                "permission_date": None,
+                "hectares": hectares,
+                "local_authority": _clean_str(_first(props, "Local_Authority")),
+                # The Land Hub exposes no stable per-site page; leave the link unset
+                # rather than pointing at a generic landing page.
+                "url": None,
+            }
+        )
+    return _frame_from_rows(rows)
+
+
+def _fetch_homes_england() -> list[dict]:
+    params = {
+        "where": "1=1",
+        "outFields": "*",
+        "outSR": "4326",
+        "f": "geojson",
+        "resultRecordCount": "2000",
+    }
+    response = httpx.get(
+        HOMES_ENGLAND_URL, params=params, follow_redirects=True, timeout=120
+    )
+    response.raise_for_status()
+    payload = response.json()
+    if payload.get("exceededTransferLimit"):
+        print(
+            "  WARNING: Homes England query hit the transfer limit; "
+            "only the first page of sites was fetched."
+        )
+    return payload.get("features", []) or []
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Download brownfield + Homes England development sites"
+    )
+    parser.add_argument(
+        "--output", type=Path, required=True, help="Output parquet file path"
+    )
+    args = parser.parse_args()
+
+    with tempfile.TemporaryDirectory(dir=local_tmp_dir()) as cache_dir:
+        brownfield_path = Path(cache_dir) / "brownfield-land.parquet"
+        print("Downloading MHCLG brownfield land register...")
+        download(BROWNFIELD_URL, brownfield_path)
+        brownfield = brownfield_to_frame(pl.read_parquet(brownfield_path))
+        print(f"  Brownfield: {brownfield.height} residential sites")
+
+        print("Downloading Homes England Land Hub...")
+        homes_england = homes_england_to_frame(_fetch_homes_england())
+        print(f"  Homes England: {homes_england.height} residential sites")
+
+    combined = pl.concat([brownfield, homes_england]).filter(
+        pl.col("lat").is_not_null() & pl.col("lon").is_not_null()
+    )
+
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    combined.write_parquet(args.output, compression="zstd")
+    size_kb = args.output.stat().st_size / 1024
+    print(f"Saved {combined.height} development sites to {args.output} ({size_kb:.0f} KB)")
+
+
+if __name__ == "__main__":
+    main()