perfect-postcode/pipeline/download/development_sites.py

"""Download planned/pipeline development sites for the "new developments" layer.

This is the forward-looking "where new homes are coming" signal that complements
the EPC new-build + Land Registry "first sale" delivery signals already in the
pipeline. Two national, Open Government Licence v3.0 sources are merged into a
single coordinate-keyed parquet that the Rust server serves as a map layer:

  - MHCLG Brownfield Land register (statutory LPA brownfield registers, normalised)
    https://www.planning.data.gov.uk/dataset/brownfield-land
  - Homes England Land Hub (public land being disposed for development)
    https://www.gov.uk/government/publications/homes-england-land-hub

Output schema (one row per site):
    lat, lon, source, name, min_dwellings, max_dwellings, planning_status,
    permission_type, permission_date, hectares, local_authority, url
"""

import argparse
import math
import re
import tempfile
from pathlib import Path

import httpx
import polars as pl
from shapely.geometry import shape

from pipeline.local_temp import local_tmp_dir
from pipeline.utils.download import download

BROWNFIELD_URL = "https://files.planning.data.gov.uk/dataset/brownfield-land.parquet"
HOMES_ENGLAND_URL = (
    "https://services-eu1.arcgis.com/yo0w4PgP4XL49bfF/arcgis/rest/services/"
    "Homes_England_Land_Hub_Sites/FeatureServer/0/query"
)

# Output column order; doubles as the polars schema so empty inputs still produce
# a well-typed frame. The Rust loader (data/developments.rs) reads these names.
SCHEMA: dict[str, pl.DataType] = {
    "lat": pl.Float64,
    "lon": pl.Float64,
    "source": pl.String,
    "name": pl.String,
    "min_dwellings": pl.Int32,
    "max_dwellings": pl.Int32,
    "planning_status": pl.String,
    "permission_type": pl.String,
    "permission_date": pl.String,
    "hectares": pl.Float64,
    "local_authority": pl.String,
    "url": pl.String,
}

_COORD = r"-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?"
_POINT_RE = re.compile(rf"POINT\s*\(\s*({_COORD})\s+({_COORD})\s*\)", re.IGNORECASE)

# 1 acre = 0.404686 hectares. The Homes England feed reports area in acres.
_ACRES_TO_HECTARES = 0.404686

# Great Britain bounding box, used to reject mis-ordered or projected coordinates
# (e.g. a WKT written "lat lon", or stray British National Grid eastings).
_LON_MIN, _LON_MAX = -9.0, 2.1
_LAT_MIN, _LAT_MAX = 49.0, 61.1


def _first(row: dict, *keys: str):
    """First present, non-None value among ``keys`` (case-sensitive)."""
    for key in keys:
        if key in row and row[key] is not None:
            return row[key]
    return None


def _clean_str(value) -> str | None:
    if value is None:
        return None
    text = str(value).strip()
    return text or None


def _to_int(value) -> int | None:
    if value is None:
        return None
    try:
        number = float(value)
    except (TypeError, ValueError):
        return None
    if not math.isfinite(number):  # NaN or +/-inf (int(inf) raises OverflowError)
        return None
    return int(number)


def _to_float(value) -> float | None:
    if value is None:
        return None
    try:
        number = float(value)
    except (TypeError, ValueError):
        return None
    if not math.isfinite(number):  # NaN or +/-inf
        return None
    return number


def _valid_lonlat(lon: float, lat: float) -> bool:
    return _LON_MIN <= lon <= _LON_MAX and _LAT_MIN <= lat <= _LAT_MAX


def _point_lonlat(row: dict) -> tuple[float, float] | None:
    """Extract (lon, lat) from a brownfield row's WKT point or lat/lon columns."""
    wkt = _clean_str(_first(row, "point"))
    if wkt:
        match = _POINT_RE.match(wkt)
        if match:
            lon, lat = float(match.group(1)), float(match.group(2))
            if _valid_lonlat(lon, lat):
                return lon, lat
    lon = _to_float(_first(row, "longitude", "long", "lng"))
    lat = _to_float(_first(row, "latitude", "lat"))
    if lon is not None and lat is not None and _valid_lonlat(lon, lat):
        return lon, lat
    return None


def _geometry_centroid(geom) -> tuple[float, float] | None:
    """Centroid (lon, lat) of a GeoJSON geometry already in WGS84."""
    if not geom:
        return None
    try:
        centroid = shape(geom).centroid
    except (ValueError, TypeError, AttributeError):
        return None
    if centroid.is_empty:
        return None
    lon, lat = float(centroid.x), float(centroid.y)
    return (lon, lat) if _valid_lonlat(lon, lat) else None


def _has_dwellings(min_d: int | None, max_d: int | None) -> bool:
    return (min_d is not None and min_d > 0) or (max_d is not None and max_d > 0)


def _brownfield_url(entity) -> str | None:
    """Canonical per-site page on the Planning Data platform.

    The register's own ``site-plan-url`` is unreliable — many LPAs point every
    row at a single generic register landing page — so we link to the stable,
    always-per-site entity page instead.
    """
    entity_id = _to_int(entity)
    if entity_id is None:
        return None
    return f"https://www.planning.data.gov.uk/entity/{entity_id}"


def _is_residential(use: str | None, capacity: int | None) -> bool:
    """Keep Homes England sites that will deliver homes."""
    if capacity is not None and capacity > 0:
        return True
    if use is None:
        return False
    lowered = use.lower()
    return any(token in lowered for token in ("resid", "housing", "mixed", "dwelling"))


def _frame_from_rows(rows: list[dict]) -> pl.DataFrame:
    return pl.DataFrame(rows, schema=SCHEMA, orient="row")


def brownfield_to_frame(raw: pl.DataFrame) -> pl.DataFrame:
    """Normalise the brownfield-land register to the unified development schema.

    Drops sites that have left the register (a non-empty ``end-date`` means the
    site was built out or withdrawn) and sites with no residential capacity, so
    the layer shows pipeline housing rather than every parcel ever registered.
    """
    rows: list[dict] = []
    for row in raw.iter_rows(named=True):
        lonlat = _point_lonlat(row)
        if lonlat is None:
            continue
        if _clean_str(_first(row, "end-date", "end_date")):
            continue
        min_d = _to_int(_first(row, "minimum-net-dwellings", "minimum_net_dwellings"))
        max_d = _to_int(_first(row, "maximum-net-dwellings", "maximum_net_dwellings"))
        if not _has_dwellings(min_d, max_d):
            continue
        lon, lat = lonlat
        rows.append(
            {
                "lat": lat,
                "lon": lon,
                "source": "brownfield",
                "name": _clean_str(_first(row, "site-address", "name")),
                "min_dwellings": min_d,
                "max_dwellings": max_d,
                "planning_status": _clean_str(
                    _first(row, "planning-permission-status")
                ),
                "permission_type": _clean_str(_first(row, "planning-permission-type")),
                "permission_date": _clean_str(_first(row, "planning-permission-date")),
                "hectares": _to_float(_first(row, "hectares")),
                "local_authority": _clean_str(_first(row, "organisation")),
                "url": _brownfield_url(_first(row, "entity")),
            }
        )
    return _frame_from_rows(rows)


def homes_england_to_frame(features: list[dict]) -> pl.DataFrame:
    """Normalise Homes England Land Hub GeoJSON features to the unified schema."""
    rows: list[dict] = []
    for feature in features:
        props = feature.get("properties") or {}
        lonlat = _geometry_centroid(feature.get("geometry"))
        if lonlat is None:
            continue
        capacity = _to_int(
            _first(props, "Housing_Capacity", "HousingCapacity", "Units", "Homes")
        )
        use = _clean_str(_first(props, "Proposed_Use"))
        if not _is_residential(use, capacity):
            continue
        lon, lat = lonlat
        # The feed reports area in acres (`Gross_Area__Acres_`); convert to hectares
        # so the column is consistent with the brownfield register.
        acres = _to_float(_first(props, "Gross_Area__Acres_"))
        hectares = round(acres * _ACRES_TO_HECTARES, 4) if acres is not None else None
        rows.append(
            {
                "lat": lat,
                "lon": lon,
                "source": "homes-england",
                "name": _clean_str(_first(props, "Parcel_Name", "Site_Reference")),
                "min_dwellings": None,
                "max_dwellings": capacity,
                "planning_status": _clean_str(_first(props, "Planning_Status")),
                "permission_type": None,
                "permission_date": None,
                "hectares": hectares,
                "local_authority": _clean_str(_first(props, "Local_Authority")),
                # The Land Hub exposes no stable per-site page; leave the link unset
                # rather than pointing at a generic landing page.
                "url": None,
            }
        )
    return _frame_from_rows(rows)


def _fetch_homes_england() -> list[dict]:
    params = {
        "where": "1=1",
        "outFields": "*",
        "outSR": "4326",
        "f": "geojson",
        "resultRecordCount": "2000",
    }
    response = httpx.get(
        HOMES_ENGLAND_URL, params=params, follow_redirects=True, timeout=120
    )
    response.raise_for_status()
    payload = response.json()
    if payload.get("exceededTransferLimit"):
        print(
            "  WARNING: Homes England query hit the transfer limit; "
            "only the first page of sites was fetched."
        )
    return payload.get("features", []) or []


def main() -> None:
    parser = argparse.ArgumentParser(
        description="Download brownfield + Homes England development sites"
    )
    parser.add_argument(
        "--output", type=Path, required=True, help="Output parquet file path"
    )
    args = parser.parse_args()

    with tempfile.TemporaryDirectory(dir=local_tmp_dir()) as cache_dir:
        brownfield_path = Path(cache_dir) / "brownfield-land.parquet"
        print("Downloading MHCLG brownfield land register...")
        download(BROWNFIELD_URL, brownfield_path)
        brownfield = brownfield_to_frame(pl.read_parquet(brownfield_path))
        print(f"  Brownfield: {brownfield.height} residential sites")

        print("Downloading Homes England Land Hub...")
        homes_england = homes_england_to_frame(_fetch_homes_england())
        print(f"  Homes England: {homes_england.height} residential sites")

    combined = pl.concat([brownfield, homes_england]).filter(
        pl.col("lat").is_not_null() & pl.col("lon").is_not_null()
    )

    args.output.parent.mkdir(parents=True, exist_ok=True)
    combined.write_parquet(args.output, compression="zstd")
    size_kb = args.output.stat().st_size / 1024
    print(f"Saved {combined.height} development sites to {args.output} ({size_kb:.0f} KB)")


if __name__ == "__main__":
    main()